ClosingSpiderPipeline.py

from scrapy.mail import MailSender
from scrapy.conf import settings


class ClosingSpiderPipeline(object):

    def close_spider(self, spider):
        self._send_email(spider)
        self._save_statistics(spider)

    def _send_email(self, spider):
        spider.logger.info('Sending email...')

        mailer = MailSender.from_settings(settings)
        to = [settings.get('MAIL_FROM')]
        subject = 'Parser name:{} finished'.format(spider.name)
        body = subject + ':\n\n'
        body += '\n'.join(['{}: {}'.format(k, v) for k, v in spider.crawler.stats.get_stats().items()])

        mailer.send(to=to, subject=subject, body=body)

    def _save_statistics(self, spider):
        stats = spider.crawler.stats.get_stats()
        with open('../statistics.txt', 'w') as f:
            for k, v in stats.items():
                f.writelines('{}: {}\n'.format(k, v))