Develop

src/pipelines/ClosingSpiderPipeline.py

+from scrapy.mail import MailSender
+from scrapy.conf import settings
+
+
+class ClosingSpiderPipeline(object):
+
+    def close_spider(self, spider):
+        self._send_email(spider)
+        self._save_statistics(spider)
+
+    def _send_email(self, spider):
+        spider.logger.info('Sending email...')
+
+        mailer = MailSender.from_settings(settings)
+        to = [settings.get('MAIL_FROM')]
+        subject = 'Parser name:{} finished'.format(spider.name)
+        body = subject + ':\n\n'
+        body += '\n'.join(['{}: {}'.format(k, v) for k, v in spider.crawler.stats.get_stats().items()])
+
+        mailer.send(to=to, subject=subject, body=body)
+
+    def _save_statistics(self, spider):
+        stats = spider.crawler.stats.get_stats()
+        with open('../statistics.txt', 'w') as f:
+            for k, v in stats.items():
+                f.writelines('{}: {}\n'.format(k, v))