Commit ⋅ scrapy/scrapy


from scrapy.http import Headers
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.responsetypes import responsetypes



@@ -110,7 +110,7 @@

        status = int(self.status)
        headers = Headers(self.response_headers)
        respcls = responsetypes.from_args(headers=headers, url=self._url)
        return respcls(url=self._url, status=status, headers=headers, body=body)
        return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))

    def _set_connection_attributes(self, request):
        parsed = urlparse_cached(request)

@@ -434,6 +434,11 @@

    def _cb_bodydone(self, result, request, url):
        headers = Headers(result["txresponse"].headers.getAllRawHeaders())
        respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
        try:
            version = result["txresponse"].version
            protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}"
        except (AttributeError, TypeError, IndexError):
            protocol = None
        response = respcls(
            url=url,
            status=int(result["txresponse"].code),

@@ -442,6 +447,7 @@

            flags=result["flags"],
            certificate=result["certificate"],
            ip_address=result["ip_address"],
            protocol=protocol,
        )
        if result.get("failure"):
            result["failure"].value.response = response

@@ -160,7 +160,7 @@

    multiple = ele.multiple
    if v is None and not multiple:
        # Match browser behaviour on simple select tag without options selected
        # And for select tags wihout options
        # And for select tags without options
        o = ele.value_options
        return (n, o[0]) if o else (None, None)
    elif v is not None and multiple:

@@ -1,5 +1,4 @@

import os
import pickle
import warnings

from importlib import import_module

@@ -68,18 +67,10 @@

    if settings_module_path:
        settings.setmodule(settings_module_path, priority='project')

    pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
    if pickled_settings:
        warnings.warn("Use of environment variable "
                      "'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE' "
                      "is deprecated.", ScrapyDeprecationWarning)
        settings.setdict(pickle.loads(pickled_settings), priority='project')

    scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if
                      k.startswith('SCRAPY_')}
    valid_envvars = {
        'CHECK',
        'PICKLED_SETTINGS_TO_OVERRIDE',
        'PROJECT',
        'PYTHON_SHELL',
        'SETTINGS_MODULE',

@@ -17,8 +17,18 @@


class Response(object_ref):

    def __init__(self, url, status=200, headers=None, body=b'', flags=None,
                 request=None, certificate=None, ip_address=None):
    def __init__(
        self,
        url,
        status=200,
        headers=None,
        body=b"",
        flags=None,
        request=None,
        certificate=None,
        ip_address=None,
        protocol=None,
    ):
        self.headers = Headers(headers or {})
        self.status = int(status)
        self._set_body(body)

@@ -27,6 +37,7 @@

        self.flags = [] if flags is None else list(flags)
        self.certificate = certificate
        self.ip_address = ip_address
        self.protocol = protocol

    @property
    def cb_kwargs(self):

@@ -89,8 +100,9 @@

        """Create a new Response with the same attributes except for those
        given new values.
        """
        for x in ['url', 'status', 'headers', 'body',
                  'request', 'flags', 'certificate', 'ip_address']:
        for x in [
            "url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol",
        ]:
            kwargs.setdefault(x, getattr(self, x))
        cls = kwargs.pop('cls', self.__class__)
        return cls(*args, **kwargs)

Files		•	•	•	Coverage
scrapy	9,719	8,537	254	928	87.84%
Project Totals (158 files)	9,719	8,537	254	928	87.84%

View CI Build Download

comment:
  layout: "header, diff, tree"

coverage:
  status:
    project: false

Read the docs

Interact now

The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file. The size and color of each slice is representing the number of statements and the coverage, respectively.

SVG

The top section represents the entire project. Proceeding with folders and finally individual files. The size and color of each slice is representing the number of statements and the coverage, respectively.

SVG

Each block represents a single file in the project. The size and color of each block is represented by the number of statements and the coverage, respectively.

SVG

Learn more about the coverage sunburst graph

110	110		status = int(self.status)
111	111		headers = Headers(self.response_headers)
112	112		respcls = responsetypes.from_args(headers=headers, url=self._url)
113		-	return respcls(url=self._url, status=status, headers=headers, body=body)
	113	+	return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
114	114
115	115		def _set_connection_attributes(self, request):
116	116		parsed = urlparse_cached(request)

442	447		flags=result["flags"],
443	448		certificate=result["certificate"],
444	449		ip_address=result["ip_address"],
	450	+	protocol=protocol,
445	451		)
446	452		if result.get("failure"):
447	453		result["failure"].value.response = response

160	160		multiple = ele.multiple
161	161		if v is None and not multiple:
162	162		# Match browser behaviour on simple select tag without options selected
163		-	# And for select tags wihout options
	163	+	# And for select tags without options
164	164		o = ele.value_options
165	165		return (n, o[0]) if o else (None, None)
166	166		elif v is not None and multiple:

Navigation	Overlay
`t` Navigate files	`h` Toggle hits
`y` Change url to tip of branch	`m` Toggle misses
`b / v` Jump to prev/next hit line	`p` Toggle partial
`z / x` Jump to prev/next missed or partial line	`1..9` Toggle flags
`shift + o` Open current page in GitHub	`a` Toggle all on
`/ or ?` Show keyboard shortcuts dialog	`c` Toggle context lines or commits

7	7
8	8		from scrapy.http import Headers
9	9		from scrapy.utils.httpobj import urlparse_cached
10		-	from scrapy.utils.python import to_bytes
	10	+	from scrapy.utils.python import to_bytes, to_unicode
11	11		from scrapy.responsetypes import responsetypes
12	12
13	13

434	434		def _cb_bodydone(self, result, request, url):
435	435		headers = Headers(result["txresponse"].headers.getAllRawHeaders())
436	436		respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
	437	+	try:
	438	+	version = result["txresponse"].version
	439	+	protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}"
	440	+	except (AttributeError, TypeError, IndexError):
	441	+	protocol = None
437	442		response = respcls(
438	443		url=url,
439	444		status=int(result["txresponse"].code),

1	1		import os
2		-	import pickle
3	2		import warnings
4	3
5	4		from importlib import import_module

68	67		if settings_module_path:
69	68		settings.setmodule(settings_module_path, priority='project')
70	69
71		-	pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
72		-	if pickled_settings:
73		-	warnings.warn("Use of environment variable "
74		-	"'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE' "
75		-	"is deprecated.", ScrapyDeprecationWarning)
76		-	settings.setdict(pickle.loads(pickled_settings), priority='project')
77		-
78	70		scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if
79	71		k.startswith('SCRAPY_')}
80	72		valid_envvars = {
81	73		'CHECK',
82		-	'PICKLED_SETTINGS_TO_OVERRIDE',
83	74		'PROJECT',
84	75		'PYTHON_SHELL',
85	76		'SETTINGS_MODULE',

17	17
18	18		class Response(object_ref):
19	19
20		-	def __init__(self, url, status=200, headers=None, body=b'', flags=None,
21		-	request=None, certificate=None, ip_address=None):
	20	+	def __init__(
	21	+	self,
	22	+	url,
	23	+	status=200,
	24	+	headers=None,
	25	+	body=b"",
	26	+	flags=None,
	27	+	request=None,
	28	+	certificate=None,
	29	+	ip_address=None,
	30	+	protocol=None,
	31	+	):
22	32		self.headers = Headers(headers or {})
23	33		self.status = int(status)
24	34		self._set_body(body)

27	37		self.flags = [] if flags is None else list(flags)
28	38		self.certificate = certificate
29	39		self.ip_address = ip_address
	40	+	self.protocol = protocol
30	41
31	42		@property
32	43		def cb_kwargs(self):

89	100		"""Create a new Response with the same attributes except for those
90	101		given new values.
91	102		"""
92		-	for x in ['url', 'status', 'headers', 'body',
93		-	'request', 'flags', 'certificate', 'ip_address']:
	103	+	for x in [
	104	+	"url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol",
	105	+	]:
94	106		kwargs.setdefault(x, getattr(self, x))
95	107		cls = kwargs.pop('cls', self.__class__)
96	108		return cls(args, *kwargs)

1	comment:
2	layout: "header, diff, tree"
3
4	coverage:
5	status:
6	project: false