scrapy / scrapy

@@ -7,7 +7,7 @@
Loading
7 7
from contextlib import suppress
8 8
from io import BytesIO
9 9
from time import time
10 -
from urllib.parse import urldefrag
10 +
from urllib.parse import urldefrag, urlunparse
11 11
12 12
from twisted.internet import defer, protocol, ssl
13 13
from twisted.internet.endpoints import TCP4ClientEndpoint
@@ -276,7 +276,7 @@
Loading
276 276
        bindaddress = request.meta.get('bindaddress') or self._bindAddress
277 277
        proxy = request.meta.get('proxy')
278 278
        if proxy:
279 -
            _, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
279 +
            proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
280 280
            scheme = _parse(request.url)[0]
281 281
            proxyHost = to_unicode(proxyHost)
282 282
            omitConnectTunnel = b'noconnect' in proxyParams
@@ -301,9 +301,13 @@
Loading
301 301
                    pool=self._pool,
302 302
                )
303 303
            else:
304 +
                proxyScheme = proxyScheme or b'http'
305 +
                proxyHost = to_bytes(proxyHost, encoding='ascii')
306 +
                proxyPort = to_bytes(str(proxyPort), encoding='ascii')
307 +
                proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
304 308
                return self._ProxyAgent(
305 309
                    reactor=reactor,
306 -
                    proxyURI=to_bytes(proxy, encoding='ascii'),
310 +
                    proxyURI=to_bytes(proxyURI, encoding='ascii'),
307 311
                    connectTimeout=timeout,
308 312
                    bindAddress=bindaddress,
309 313
                    pool=self._pool,

@@ -1,3 +1,4 @@
Loading
1 +
import re
1 2
from time import time
2 3
from urllib.parse import urlparse, urlunparse, urldefrag
3 4
@@ -32,6 +33,8 @@
Loading
32 33
    and is ascii-only.
33 34
    """
34 35
    url = url.strip()
36 +
    if not re.match(r'^\w+://', url):
37 +
        url = '//' + url
35 38
    parsed = urlparse(url)
36 39
    return _parsed_url_args(parsed)
37 40
Files Coverage
scrapy 88.20%
Project Totals (162 files) 88.20%
Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file. The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files. The size and color of each slice is representing the number of statements and the coverage, respectively.
Grid
Each block represents a single file in the project. The size and color of each block is represented by the number of statements and the coverage, respectively.
Loading