scrapy / scrapy
1
"""
2
Helper functions for serializing (and deserializing) requests.
3
"""
4 7
import inspect
5

6 7
from scrapy.http import Request
7 7
from scrapy.utils.python import to_unicode
8 7
from scrapy.utils.misc import load_object
9

10

11 7
def request_to_dict(request, spider=None):
12
    """Convert Request object to a dict.
13

14
    If a spider is given, it will try to find out the name of the spider method
15
    used in the callback and store that as the callback.
16
    """
17 7
    cb = request.callback
18 7
    if callable(cb):
19 7
        cb = _find_method(spider, cb)
20 7
    eb = request.errback
21 7
    if callable(eb):
22 7
        eb = _find_method(spider, eb)
23 7
    d = {
24
        'url': to_unicode(request.url),  # urls should be safe (safe_string_url)
25
        'callback': cb,
26
        'errback': eb,
27
        'method': request.method,
28
        'headers': dict(request.headers),
29
        'body': request.body,
30
        'cookies': request.cookies,
31
        'meta': request.meta,
32
        '_encoding': request._encoding,
33
        'priority': request.priority,
34
        'dont_filter': request.dont_filter,
35
        'flags': request.flags,
36
        'cb_kwargs': request.cb_kwargs,
37
    }
38 7
    if type(request) is not Request:
39 7
        d['_class'] = request.__module__ + '.' + request.__class__.__name__
40 7
    return d
41

42

43 7
def request_from_dict(d, spider=None):
44
    """Create Request object from a dict.
45

46
    If a spider is given, it will try to resolve the callbacks looking at the
47
    spider for methods with the same name.
48
    """
49 7
    cb = d['callback']
50 7
    if cb and spider:
51 7
        cb = _get_method(spider, cb)
52 7
    eb = d['errback']
53 7
    if eb and spider:
54 7
        eb = _get_method(spider, eb)
55 7
    request_cls = load_object(d['_class']) if '_class' in d else Request
56 7
    return request_cls(
57
        url=to_unicode(d['url']),
58
        callback=cb,
59
        errback=eb,
60
        method=d['method'],
61
        headers=d['headers'],
62
        body=d['body'],
63
        cookies=d['cookies'],
64
        meta=d['meta'],
65
        encoding=d['_encoding'],
66
        priority=d['priority'],
67
        dont_filter=d['dont_filter'],
68
        flags=d.get('flags'),
69
        cb_kwargs=d.get('cb_kwargs'),
70
    )
71

72

73 7
def _find_method(obj, func):
74
    # Only instance methods contain ``__func__``
75 7
    if obj and hasattr(func, '__func__'):
76 7
        members = inspect.getmembers(obj, predicate=inspect.ismethod)
77 7
        for name, obj_func in members:
78
            # We need to use __func__ to access the original
79
            # function object because instance method objects
80
            # are generated each time attribute is retrieved from
81
            # instance.
82
            #
83
            # Reference: The standard type hierarchy
84
            # https://docs.python.org/3/reference/datamodel.html
85 7
            if obj_func.__func__ is func.__func__:
86 7
                return name
87 7
    raise ValueError(f"Function {func} is not an instance method in: {obj}")
88

89

90 7
def _get_method(obj, name):
91 7
    name = str(name)
92 7
    try:
93 7
        return getattr(obj, name)
94 0
    except AttributeError:
95 0
        raise ValueError(f"Method {name!r} not found in: {obj}")

Read our documentation on viewing source code .

Loading