scrapy / scrapy
1 7
from w3lib.http import headers_dict_to_raw
2 7
from scrapy.utils.datatypes import CaselessDict
3 7
from scrapy.utils.python import to_unicode
4

5

6 7
class Headers(CaselessDict):
7
    """Case insensitive http headers dictionary"""
8

9 7
    def __init__(self, seq=None, encoding='utf-8'):
10 7
        self.encoding = encoding
11 7
        super().__init__(seq)
12

13 7
    def normkey(self, key):
14
        """Normalize key to bytes"""
15 7
        return self._tobytes(key.title())
16

17 7
    def normvalue(self, value):
18
        """Normalize values to bytes"""
19 7
        if value is None:
20 7
            value = []
21 7
        elif isinstance(value, (str, bytes)):
22 7
            value = [value]
23 7
        elif not hasattr(value, '__iter__'):
24 7
            value = [value]
25

26 7
        return [self._tobytes(x) for x in value]
27

28 7
    def _tobytes(self, x):
29 7
        if isinstance(x, bytes):
30 7
            return x
31 7
        elif isinstance(x, str):
32 7
            return x.encode(self.encoding)
33 7
        elif isinstance(x, int):
34 7
            return str(x).encode(self.encoding)
35
        else:
36 7
            raise TypeError(f'Unsupported value type: {type(x)}')
37

38 7
    def __getitem__(self, key):
39 7
        try:
40 7
            return super().__getitem__(key)[-1]
41 7
        except IndexError:
42 0
            return None
43

44 7
    def get(self, key, def_val=None):
45 7
        try:
46 7
            return super().get(key, def_val)[-1]
47 7
        except IndexError:
48 7
            return None
49

50 7
    def getlist(self, key, def_val=None):
51 7
        try:
52 7
            return super().__getitem__(key)
53 7
        except KeyError:
54 7
            if def_val is not None:
55 7
                return self.normvalue(def_val)
56 7
            return []
57

58 7
    def setlist(self, key, list_):
59 7
        self[key] = list_
60

61 7
    def setlistdefault(self, key, default_list=()):
62 7
        return self.setdefault(key, default_list)
63

64 7
    def appendlist(self, key, value):
65 7
        lst = self.getlist(key)
66 7
        lst.extend(self.normvalue(value))
67 7
        self[key] = lst
68

69 7
    def items(self):
70 7
        return ((k, self.getlist(k)) for k in self.keys())
71

72 7
    def values(self):
73 7
        return [self[k] for k in self.keys()]
74

75 7
    def to_string(self):
76 7
        return headers_dict_to_raw(self)
77

78 7
    def to_unicode_dict(self):
79
        """ Return headers as a CaselessDict with unicode keys
80
        and unicode values. Multiple values are joined with ','.
81
        """
82 7
        return CaselessDict(
83
            (to_unicode(key, encoding=self.encoding),
84
             to_unicode(b','.join(value), encoding=self.encoding))
85
            for key, value in self.items())
86

87 7
    def __copy__(self):
88 7
        return self.__class__(self)
89 7
    copy = __copy__

Read our documentation on viewing source code .

Loading