scrapy / scrapy
Showing 1 of 2 files from the diff.
Other files ignored by Codecov

@@ -178,8 +178,8 @@
Loading
178 178
179 179
        if isinstance(request, RequestList):
180 180
            d = DeferredList([self._download(r, self.spider) for r in request.requests])
181 -
            # FIXME: handle other cases (exceptions, redirects, etc)
182 -
            d.addBoth(lambda result: ResponseList(request_list=request, responses=[t[1] for t in result]))
181 +
            d.addBoth(self._handle_requestlist_result)
182 +
            d.addBoth(lambda result: ResponseList(request_list=request, responses=result))
183 183
        else:
184 184
            d = self._download(request, self.spider)
185 185
        d.addBoth(self._handle_downloader_output, request)
@@ -221,6 +221,33 @@
Loading
221 221
        )
222 222
        return d
223 223
224 +
    def _handle_requestlist_result(self, result: list, processed: Optional[list] = None) -> Union[DeferredList, list]:
225 +
        """If necessary, re-schedule pending requests within the results of a RequestList (scheduler is bypassed)"""
226 +
        assert self.spider is not None  # typing
227 +
228 +
        def remove_pending_from_slot(result: list, processed: list, pending: list) -> list:
229 +
            assert self.slot is not None  # typing
230 +
            for p in pending:
231 +
                self.slot.remove_request(p)
232 +
            return result
233 +
234 +
        processed = processed or []
235 +
        pending = []
236 +
        for _, r in result:
237 +
            if isinstance(r, Request):
238 +
                pending.append(r)
239 +
            else:
240 +
                processed.append(r)
241 +
        if pending:
242 +
            d = DeferredList([self._download(p, self.spider) for p in pending])
243 +
            d.addBoth(self._handle_requestlist_result, processed=processed)
244 +
            d.addBoth(remove_pending_from_slot, processed=processed, pending=pending)
245 +
            d.addErrback(lambda f: logger.info('Error while removing request from slot',
246 +
                                               exc_info=failure_to_exc_info(f),
247 +
                                               extra={'spider': self.spider}))
248 +
            return d
249 +
        return processed
250 +
224 251
    def spider_is_idle(self, spider: Optional[Spider] = None) -> bool:
225 252
        if spider is not None:
226 253
            warnings.warn(
@@ -240,7 +267,7 @@
Loading
240 267
            return False
241 268
        return True
242 269
243 -
    def crawl(self, request: Request, spider: Optional[Spider] = None) -> None:
270 +
    def crawl(self, request: Union[Request, RequestList], spider: Optional[Spider] = None) -> None:
244 271
        """Inject the request into the spider <-> downloader pipeline"""
245 272
        if spider is not None:
246 273
            warnings.warn(
Files Coverage
scrapy 87.40%
Project Totals (162 files) 87.40%
Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file. The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files. The size and color of each slice is representing the number of statements and the coverage, respectively.
Grid
Each block represents a single file in the project. The size and color of each block is represented by the number of statements and the coverage, respectively.
Loading