scrapy / scrapy
1
"""Helper functions for scrapy.http objects (Request, Response)"""
2

3 7
from typing import Union
4 7
from urllib.parse import urlparse, ParseResult
5 7
from weakref import WeakKeyDictionary
6

7 7
from scrapy.http import Request, Response
8

9

10 7
_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary()
11

12

13 7
def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult:
14
    """Return urlparse.urlparse caching the result, where the argument can be a
15
    Request or Response object
16
    """
17 7
    if request_or_response not in _urlparse_cache:
18 7
        _urlparse_cache[request_or_response] = urlparse(request_or_response.url)
19 7
    return _urlparse_cache[request_or_response]

Read our documentation on viewing source code .

Loading