Commit 2aaa0d64 authored by Kyryll Parolis's avatar Kyryll Parolis
Browse files

Finished writing middleware.

parent b4b27d9a
......@@ -4,10 +4,15 @@
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
import scrapy
from dotenv import load_dotenv
from scrapy import signals
import os
from scrapy.mail import MailSender
import settings
class BuildzoomParserSpiderMiddleware:
# Not all methods need to be defined. If a method is not defined,
......@@ -73,14 +78,18 @@ class BuildzoomParserDownloaderMiddleware:
def process_request(self, request, spider):
request.meta['proxy'] = os.getenv('http_proxy')
def process_response(self, request, response: scrapy.http.Response, spider):
mailer = MailSender().from_settings(settings)
if response.status == 407:
mailer.send(subject="407 status code <buildzoom_parser>",
body="Proxy authentication required.",
cc=["another@example.com"])
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
elif response.status == 429:
mailer.send(subject="429 status code <buildzoom_parser>",
body="Too many requests.",
cc=["another@example.com"])
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
......
......@@ -96,3 +96,8 @@ DOWNLOADER_MIDDLEWARES = {
# HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
MAIL_TO = os.getenv("MAIL_TO")
MAIL_HOST = os.getenv("MAIL_HOST")
MAIL_USER = os.getenv("MAIL_USER")
MAIL_PASS = os.getenv("MAIL_PASS")
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment