scrapy / scrapy
1
"""
2
This module contains data types used by Scrapy which are not included in the
3
Python Standard Library.
4

5
This module must not depend on any module outside the Standard Library.
6
"""
7

8 7
import collections
9 7
import weakref
10 7
from collections.abc import Mapping
11

12

13 7
class CaselessDict(dict):
14

15 7
    __slots__ = ()
16

17 7
    def __init__(self, seq=None):
18 7
        super().__init__()
19 7
        if seq:
20 7
            self.update(seq)
21

22 7
    def __getitem__(self, key):
23 7
        return dict.__getitem__(self, self.normkey(key))
24

25 7
    def __setitem__(self, key, value):
26 7
        dict.__setitem__(self, self.normkey(key), self.normvalue(value))
27

28 7
    def __delitem__(self, key):
29 7
        dict.__delitem__(self, self.normkey(key))
30

31 7
    def __contains__(self, key):
32 7
        return dict.__contains__(self, self.normkey(key))
33 7
    has_key = __contains__
34

35 7
    def __copy__(self):
36 7
        return self.__class__(self)
37 7
    copy = __copy__
38

39 7
    def normkey(self, key):
40
        """Method to normalize dictionary key access"""
41 7
        return key.lower()
42

43 7
    def normvalue(self, value):
44
        """Method to normalize values prior to be setted"""
45 7
        return value
46

47 7
    def get(self, key, def_val=None):
48 7
        return dict.get(self, self.normkey(key), self.normvalue(def_val))
49

50 7
    def setdefault(self, key, def_val=None):
51 7
        return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
52

53 7
    def update(self, seq):
54 7
        seq = seq.items() if isinstance(seq, Mapping) else seq
55 7
        iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
56 7
        super().update(iseq)
57

58 7
    @classmethod
59 7
    def fromkeys(cls, keys, value=None):
60 7
        return cls((k, value) for k in keys)
61

62 7
    def pop(self, key, *args):
63 7
        return dict.pop(self, self.normkey(key), *args)
64

65

66 7
class LocalCache(collections.OrderedDict):
67
    """Dictionary with a finite number of keys.
68

69
    Older items expires first.
70
    """
71

72 7
    def __init__(self, limit=None):
73 7
        super().__init__()
74 7
        self.limit = limit
75

76 7
    def __setitem__(self, key, value):
77 7
        if self.limit:
78 7
            while len(self) >= self.limit:
79 7
                self.popitem(last=False)
80 7
        super().__setitem__(key, value)
81

82

83 7
class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
84
    """
85
    A weakref.WeakKeyDictionary implementation that uses LocalCache as its
86
    underlying data structure, making it ordered and capable of being size-limited.
87

88
    Useful for memoization, while avoiding keeping received
89
    arguments in memory only because of the cached references.
90

91
    Note: like LocalCache and unlike weakref.WeakKeyDictionary,
92
    it cannot be instantiated with an initial dictionary.
93
    """
94

95 7
    def __init__(self, limit=None):
96 7
        super().__init__()
97 7
        self.data = LocalCache(limit=limit)
98

99 7
    def __setitem__(self, key, value):
100 7
        try:
101 7
            super().__setitem__(key, value)
102 7
        except TypeError:
103 7
            pass  # key is not weak-referenceable, skip caching
104

105 7
    def __getitem__(self, key):
106 7
        try:
107 7
            return super().__getitem__(key)
108 7
        except (TypeError, KeyError):
109 7
            return None  # key is either not weak-referenceable or not cached
110

111

112 7
class SequenceExclude:
113
    """Object to test if an item is NOT within some sequence."""
114

115 7
    def __init__(self, seq):
116 7
        self.seq = seq
117

118 7
    def __contains__(self, item):
119 7
        return item not in self.seq

Read our documentation on viewing source code .

Loading