scrapy / scrapy
1
"""
2
StatsMailer extension sends an email when a spider finishes scraping.
3

4
Use STATSMAILER_RCPTS setting to enable and give the recipient mail address
5
"""
6

7 0
from scrapy import signals
8 0
from scrapy.mail import MailSender
9 0
from scrapy.exceptions import NotConfigured
10

11 0
class StatsMailer:
12

13 0
    def __init__(self, stats, recipients, mail):
14 0
        self.stats = stats
15 0
        self.recipients = recipients
16 0
        self.mail = mail
17

18 0
    @classmethod
19 0
    def from_crawler(cls, crawler):
20 0
        recipients = crawler.settings.getlist("STATSMAILER_RCPTS")
21 7
        if not recipients:
22 0
            raise NotConfigured
23 0
        mail = MailSender.from_settings(crawler.settings)
24 0
        o = cls(crawler.stats, recipients, mail)
25 0
        crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
26 0
        return o
27

28 0
    def spider_closed(self, spider):
29 0
        spider_stats = self.stats.get_stats(spider)
30 0
        body = "Global stats\n\n"
31 7
        body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items())
32 0
        body += f"\n\n{spider.name} stats\n\n"
33 7
        body += "\n".join(f"{k:<50} : {v}" for k, v in spider_stats.items())
34 0
        return self.mail.send(self.recipients, f"Scrapy stats for: {spider.name}", body)

Read our documentation on viewing source code .

Loading