#4897 headers_received signal

Open Eugenio Lacuesta elacuesta
Coverage Reach
utils/python.py utils/misc.py utils/conf.py utils/iterators.py utils/log.py utils/defer.py utils/datatypes.py utils/deprecate.py utils/test.py utils/console.py utils/project.py utils/reactor.py utils/curl.py utils/url.py utils/signal.py utils/request.py utils/response.py utils/ssl.py utils/testproc.py utils/reqser.py utils/trackref.py utils/spider.py utils/benchserver.py utils/testsite.py utils/serialize.py utils/decorators.py utils/display.py utils/sitemap.py utils/gz.py utils/ftp.py utils/engine.py utils/boto.py utils/template.py utils/ossignal.py utils/versions.py utils/httpobj.py utils/job.py utils/asyncgen.py utils/py36.py core/downloader/handlers/http11.py core/downloader/handlers/ftp.py core/downloader/handlers/__init__.py core/downloader/handlers/s3.py core/downloader/handlers/http10.py core/downloader/handlers/datauri.py core/downloader/handlers/file.py core/downloader/handlers/http.py core/downloader/__init__.py core/downloader/webclient.py core/downloader/middleware.py core/downloader/tls.py core/downloader/contextfactory.py core/engine.py core/scraper.py core/scheduler.py core/spidermw.py extensions/feedexport.py extensions/httpcache.py extensions/memusage.py extensions/telnet.py extensions/throttle.py extensions/closespider.py extensions/debug.py extensions/logstats.py extensions/corestats.py extensions/spiderstate.py extensions/statsmailer.py extensions/memdebug.py commands/parse.py commands/genspider.py commands/startproject.py commands/__init__.py commands/check.py commands/runspider.py commands/fetch.py commands/shell.py commands/bench.py commands/settings.py commands/edit.py commands/crawl.py commands/version.py commands/view.py commands/list.py http/request/form.py http/request/__init__.py http/request/json_request.py http/request/rpc.py http/response/text.py http/response/__init__.py http/response/html.py http/response/xml.py http/cookies.py http/headers.py http/__init__.py http/common.py downloadermiddlewares/httpcache.py downloadermiddlewares/cookies.py downloadermiddlewares/robotstxt.py downloadermiddlewares/redirect.py downloadermiddlewares/decompression.py downloadermiddlewares/httpcompression.py downloadermiddlewares/httpproxy.py downloadermiddlewares/retry.py downloadermiddlewares/ajaxcrawl.py downloadermiddlewares/stats.py downloadermiddlewares/httpauth.py downloadermiddlewares/useragent.py downloadermiddlewares/downloadtimeout.py downloadermiddlewares/defaultheaders.py pipelines/files.py pipelines/media.py pipelines/images.py pipelines/__init__.py settings/__init__.py settings/default_settings.py spidermiddlewares/referer.py spidermiddlewares/offsite.py spidermiddlewares/depth.py spidermiddlewares/httperror.py spidermiddlewares/urllength.py spiders/crawl.py spiders/__init__.py spiders/feed.py spiders/sitemap.py spiders/init.py exporters.py contracts/__init__.py contracts/default.py crawler.py linkextractors/lxmlhtml.py linkextractors/__init__.py shell.py cmdline.py pqueues.py robotstxt.py mail.py item.py resolver.py responsetypes.py squeues.py dupefilters.py middleware.py statscollectors.py spiderloader.py logformatter.py selector/unified.py selector/__init__.py exceptions.py loader/__init__.py loader/processors.py loader/common.py signals.py signalmanager.py __init__.py link.py extension.py interfaces.py __main__.py

No flags found

Use flags to group coverage reports by test type, project and/or folders.
Then setup custom commit statuses and notifications for each flag.

e.g., #unittest #integration

#production #enterprise

#frontend #backend

Learn more about Codecov Flags here.


@@ -379,6 +379,25 @@
Loading
379 379
        return result
380 380
381 381
    def _cb_bodyready(self, txresponse, request):
382 +
        headers_received_result = self._crawler.signals.send_catch_log(
383 +
            signal=signals.headers_received,
384 +
            headers=Headers(txresponse.headers.getAllRawHeaders()),
385 +
            request=request,
386 +
            spider=self._crawler.spider,
387 +
        )
388 +
        for handler, result in headers_received_result:
389 +
            if isinstance(result, Failure) and isinstance(result.value, StopDownload):
390 +
                logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
391 +
                             {"request": request, "handler": handler.__qualname__})
392 +
                return {
393 +
                    "txresponse": txresponse,
394 +
                    "body": b"",
395 +
                    "flags": ["download_stopped"],
396 +
                    "certificate": None,
397 +
                    "ip_address": None,
398 +
                    "failure": result if result.value.fail else None,
399 +
                }
400 +
382 401
        # deliverBody hangs for responses without body
383 402
        if txresponse.length == 0:
384 403
            return {

@@ -17,6 +17,7 @@
Loading
17 17
request_left_downloader = object()
18 18
response_received = object()
19 19
response_downloaded = object()
20 +
headers_received = object()
20 21
bytes_received = object()
21 22
item_scraped = object()
22 23
item_dropped = object()

Learn more Showing 1 files with coverage changes found.

Changes in scrapy/core/downloader/__init__.py
-2
+1
+1
Loading file...
Files Coverage
scrapy -0.01% 87.88%
Project Totals (158 files) 87.88%
Loading