fix error for send mail

src/pipelines/ClosingSpiderPipeline.py → src/pipelines/StatsMailer.py

@@ -2,25 +2,13 @@ from scrapy.mail import MailSender
 from scrapy.conf import settings
 
 
-class ClosingSpiderPipeline(object):
+class StatsMailer(object):
+    name = 'StatsMailer'
 
     def close_spider(self, spider):
-        self._send_email(spider)
-        self._save_statistics(spider)
-
-    def _send_email(self, spider):
-        spider.logger.info('Sending email...')
-
         mailer = MailSender.from_settings(settings)
         to = [settings.get('MAIL_FROM')]
         subject = 'Parser name:{} finished'.format(spider.name)
         body = subject + ':\n\n'
         body += '\n'.join(['{}: {}'.format(k, v) for k, v in spider.crawler.stats.get_stats().items()])
-
-        mailer.send(to=to, subject=subject, body=body)
-
-    def _save_statistics(self, spider):
-        stats = spider.crawler.stats.get_stats()
-        with open('../statistics.txt', 'w') as f:
-            for k, v in stats.items():
-                f.writelines('{}: {}\n'.format(k, v))
+        return mailer.send(to=to, subject=subject, body=body)

src/pipelines/StatsWriter.py

+
+
+class StatsWriter(object):
+    name = 'StatsWriter'
+
+    def close_spider(self, spider):
+        stats = spider.crawler.stats.get_stats()
+        with open('../statistics.txt', 'w') as f:
+            for k, v in stats.items():
+                f.writelines('{}: {}\n'.format(k, v))

src/settings.py

@@ -71,7 +71,9 @@ ROBOTSTXT_OBEY = True
 IMAGES_STORE = '../image'
 ITEM_PIPELINES = {
     'pipelines.DatabasePipeline.DatabasePipeline': 101,
-    'pipelines.ClosingSpiderPipeline.ClosingSpiderPipeline': 1000,
+    'pipelines.DatabaseWriter.DatabaseWriter': 101,
+    'pipelines.StatsMailer.StatsMailer': 999,
+    'pipelines.StatsWriter.StatsWriter': 1000,
     'scrapy.pipelines.images.ImagesPipeline': 1,
 }