TeamHG-Memex / scrapy-rotating-proxies
Showing 1 of 1 files from the diff.

@@ -75,6 +75,9 @@
Loading
75 75
        self.max_proxies_to_try = max_proxies_to_try
76 76
        self.stats = crawler.stats
77 77
78 +
        self.log_task = None
79 +
        self.reanimate_task = None
80 +
78 81
    @classmethod
79 82
    def from_crawler(cls, crawler):
80 83
        s = crawler.settings
@@ -102,10 +105,13 @@
Loading
102 105
        return mw
103 106
104 107
    def engine_started(self):
105 -
        self.log_task = task.LoopingCall(self.log_stats)
106 -
        self.log_task.start(self.logstats_interval, now=True)
107 -
        self.reanimate_task = task.LoopingCall(self.reanimate_proxies)
108 -
        self.reanimate_task.start(self.reanimate_interval, now=False)
108 +
        if self.logstats_interval:
109 +
            self.log_task = task.LoopingCall(self.log_stats)
110 +
            self.log_task.start(self.logstats_interval, now=True)
111 +
112 +
        if self.reanimate_interval:
113 +
            self.reanimate_task = task.LoopingCall(self.reanimate_proxies)
114 +
            self.reanimate_task.start(self.reanimate_interval, now=False)
109 115
110 116
    def reanimate_proxies(self):
111 117
        n_reanimated = self.proxies.reanimate()
@@ -114,9 +120,10 @@
Loading
114 120
                         n_reanimated)
115 121
116 122
    def engine_stopped(self):
117 -
        if self.log_task.running:
123 +
        if self.log_task and self.log_task.running:
118 124
            self.log_task.stop()
119 -
        if self.reanimate_task.running:
125 +
126 +
        if self.reanimate_task and self.reanimate_task.running:
120 127
            self.reanimate_task.stop()
121 128
122 129
    def process_request(self, request, spider):
@@ -220,19 +227,19 @@
Loading
220 227
221 228
    By default, client is considered banned if a request failed, and alive
222 229
    if a response was received. You can override ban detection method by
223 -
    passing a path to a custom BanDectionPolicy in 
230 +
    passing a path to a custom BanDectionPolicy in
224 231
    ``ROTATING_PROXY_BAN_POLICY``, e.g.::
225 -
      
232 +
226 233
    ROTATING_PROXY_BAN_POLICY = 'myproject.policy.MyBanPolicy'
227 -
    
228 -
    The policy must be a class with ``response_is_ban``  
229 -
    and ``exception_is_ban`` methods. These methods can return True 
234 +
235 +
    The policy must be a class with ``response_is_ban``
236 +
    and ``exception_is_ban`` methods. These methods can return True
230 237
    (ban detected), False (not a ban) or None (unknown). It can be convenient
231 238
    to subclass and modify default BanDetectionPolicy::
232 -
        
239 +
233 240
        # myproject/policy.py
234 241
        from rotating_proxies.policy import BanDetectionPolicy
235 -
        
242 +
236 243
        class MyPolicy(BanDetectionPolicy):
237 244
            def response_is_ban(self, request, response):
238 245
                # use default rules, but also consider HTTP 200 responses
@@ -240,12 +247,12 @@
Loading
240 247
                ban = super(MyPolicy, self).response_is_ban(request, response)
241 248
                ban = ban or b'captcha' in response.body
242 249
                return ban
243 -
                
250 +
244 251
            def exception_is_ban(self, request, exception):
245 252
                # override method completely: don't take exceptions in account
246 253
                return None
247 -
        
248 -
    Instead of creating a policy you can also implement ``response_is_ban`` 
254 +
255 +
    Instead of creating a policy you can also implement ``response_is_ban``
249 256
    and ``exception_is_ban`` methods as spider methods, for example::
250 257
251 258
        class MySpider(scrapy.Spider):
@@ -256,7 +263,7 @@
Loading
256 263
257 264
            def exception_is_ban(self, request, exception):
258 265
                return None
259 -
     
266 +
260 267
    """
261 268
    def __init__(self, stats, policy):
262 269
        self.stats = stats
Files Coverage
rotating_proxies 50.83%
Project Totals (5 files) 50.83%
54.3
TRAVIS_PYTHON_VERSION=3.6
TRAVIS_OS_NAME=linux
TOXENV=py36
54.1
TRAVIS_PYTHON_VERSION=2.7
TRAVIS_OS_NAME=linux
TOXENV=py27
54.2
TRAVIS_PYTHON_VERSION=3.5
TRAVIS_OS_NAME=linux
TOXENV=py35
1
comment:
2
  layout: "header, diff, tree"
3

4
coverage:
5
  status:
6
    project: false
Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file. The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files. The size and color of each slice is representing the number of statements and the coverage, respectively.
Grid
Each block represents a single file in the project. The size and color of each block is represented by the number of statements and the coverage, respectively.
Loading