Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
Kyryll Parolis
buildzoom_parser
Commits
e33ea827
Commit
e33ea827
authored
Jun 22, 2020
by
Kyryll Parolis
Browse files
Added mail notification.
parent
066a12e8
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
11 additions
and
10 deletions
+11
-10
.gitignore
.gitignore
+1
-0
src/middlewares/middlewares.py
src/middlewares/middlewares.py
+10
-10
No files found.
.gitignore
View file @
e33ea827
...
@@ -8,3 +8,4 @@ src/pipelines/__pycache__
...
@@ -8,3 +8,4 @@ src/pipelines/__pycache__
src/spiders/__pycache__
src/spiders/__pycache__
src/alembic/versions/__pycache__
src/alembic/versions/__pycache__
src/alembic/__pycache__
src/alembic/__pycache__
log.txt
src/middlewares/middlewares.py
View file @
e33ea827
...
@@ -4,14 +4,11 @@
...
@@ -4,14 +4,11 @@
#
#
# See documentation in:
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
import
scrapy
from
dotenv
import
load_dotenv
from
dotenv
import
load_dotenv
from
scrapy
import
signals
from
scrapy
import
signals
import
os
from
scrapy.mail
import
MailSender
from
scrapy.mail
import
MailSender
from
scrapy.utils.project
import
get_project_settings
import
settings
class
BuildzoomParserSpiderMiddleware
:
class
BuildzoomParserSpiderMiddleware
:
...
@@ -76,17 +73,20 @@ class BuildzoomParserDownloaderMiddleware:
...
@@ -76,17 +73,20 @@ class BuildzoomParserDownloaderMiddleware:
return
s
return
s
def
process_request
(
self
,
request
,
spider
):
def
process_request
(
self
,
request
,
spider
):
request
.
meta
[
'proxy'
]
=
os
.
getenv
(
'http_proxy'
)
# request.meta['proxy'] = os.getenv('http_proxy')
pass
def
process_response
(
self
,
request
,
response
:
scrapy
.
http
.
Response
,
spider
):
def
process_response
(
self
,
request
,
response
,
spider
):
mailer
=
MailSender
().
from_settings
(
settings
)
mailer
=
MailSender
(
mailfrom
=
'kyryllpar@gmail.com'
,
smtphost
=
'smtp.gmail.com'
,
smtpport
=
587
,
smtpuser
=
'kyryllpar@gmail.com'
,
smtppass
=
'Bc279811'
)
if
response
.
status
==
407
:
if
response
.
status
==
407
:
mailer
.
send
(
subject
=
"407 status code <buildzoom_parser>"
,
mailer
.
send
(
to
=
[
"kyryllpar@gmail.com"
],
subject
=
"407 status code <buildzoom_parser>"
,
body
=
"Proxy authentication required."
,
body
=
"Proxy authentication required."
,
cc
=
[
"another@example.com"
])
cc
=
[
"another@example.com"
])
elif
response
.
status
==
429
:
elif
response
.
status
==
429
:
mailer
.
send
(
subject
=
"429 status code <buildzoom_parser>"
,
mailer
.
send
(
to
=
[
"kyryllpar@gmail.com"
],
subject
=
"429 status code <buildzoom_parser>"
,
body
=
"Too many requests."
,
body
=
"Too many requests."
,
cc
=
[
"another@example.com"
])
cc
=
[
"another@example.com"
])
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment