scrapy / scrapy
1
"""
2
Download timeout middleware
3

4
See documentation in docs/topics/downloader-middleware.rst
5
"""
6

7 7
from scrapy import signals
8

9

10 7
class DownloadTimeoutMiddleware:
11

12 7
    def __init__(self, timeout=180):
13 7
        self._timeout = timeout
14

15 7
    @classmethod
16 4
    def from_crawler(cls, crawler):
17 7
        o = cls(crawler.settings.getfloat('DOWNLOAD_TIMEOUT'))
18 7
        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
19 7
        return o
20

21 7
    def spider_opened(self, spider):
22 7
        self._timeout = getattr(spider, 'download_timeout', self._timeout)
23

24 7
    def process_request(self, request, spider):
25 7
        if self._timeout:
26 7
            request.meta.setdefault('download_timeout', self._timeout)

Read our documentation on viewing source code .

Loading