scrapy / scrapy
1
"""
2
Scrapy extension for collecting scraping stats
3
"""
4 7
import pprint
5 7
import logging
6

7 7
logger = logging.getLogger(__name__)
8

9

10 7
class StatsCollector:
11

12 7
    def __init__(self, crawler):
13 7
        self._dump = crawler.settings.getbool('STATS_DUMP')
14 7
        self._stats = {}
15

16 7
    def get_value(self, key, default=None, spider=None):
17 7
        return self._stats.get(key, default)
18

19 7
    def get_stats(self, spider=None):
20 7
        return self._stats
21

22 7
    def set_value(self, key, value, spider=None):
23 7
        self._stats[key] = value
24

25 7
    def set_stats(self, stats, spider=None):
26 0
        self._stats = stats
27

28 7
    def inc_value(self, key, count=1, start=0, spider=None):
29 7
        d = self._stats
30 7
        d[key] = d.setdefault(key, start) + count
31

32 7
    def max_value(self, key, value, spider=None):
33 7
        self._stats[key] = max(self._stats.setdefault(key, value), value)
34

35 7
    def min_value(self, key, value, spider=None):
36 7
        self._stats[key] = min(self._stats.setdefault(key, value), value)
37

38 7
    def clear_stats(self, spider=None):
39 0
        self._stats.clear()
40

41 7
    def open_spider(self, spider):
42 7
        pass
43

44 7
    def close_spider(self, spider, reason):
45 7
        if self._dump:
46 7
            logger.info("Dumping Scrapy stats:\n" + pprint.pformat(self._stats),
47
                        extra={'spider': spider})
48 7
        self._persist_stats(self._stats, spider)
49

50 7
    def _persist_stats(self, stats, spider):
51 7
        pass
52

53

54 7
class MemoryStatsCollector(StatsCollector):
55

56 7
    def __init__(self, crawler):
57 7
        super().__init__(crawler)
58 7
        self.spider_stats = {}
59

60 7
    def _persist_stats(self, stats, spider):
61 7
        self.spider_stats[spider.name] = stats
62

63

64 7
class DummyStatsCollector(StatsCollector):
65

66 7
    def get_value(self, key, default=None, spider=None):
67 7
        return default
68

69 7
    def set_value(self, key, value, spider=None):
70 7
        pass
71

72 7
    def set_stats(self, stats, spider=None):
73 0
        pass
74

75 7
    def inc_value(self, key, count=1, start=0, spider=None):
76 7
        pass
77

78 7
    def max_value(self, key, value, spider=None):
79 7
        pass
80

81 7
    def min_value(self, key, value, spider=None):
82 7
        pass

Read our documentation on viewing source code .

Loading