From d7290700b71975a294d634a1c319e4d6806371c2 Mon Sep 17 00:00:00 2001
From: Gusev Anton <gusev_aa@groupbwt.com>
Date: Fri, 14 Jun 2019 15:07:56 +0300
Subject: [PATCH] add alternative database connection using mysqlclient
---
requirments.txt | 1 +
src/database/AltConnector.py | 21 +++++++++++++++++++++
src/database/connectors/AltBookConnector.py | 21 +++++++++++++++++++++
src/spiders/books_spider.py | 3 ++-
4 files changed, 45 insertions(+), 1 deletion(-)
create mode 100644 src/database/AltConnector.py
create mode 100644 src/database/connectors/AltBookConnector.py
diff --git a/requirments.txt b/requirments.txt
index 00f7343..041bd3a 100644
--- a/requirments.txt
+++ b/requirments.txt
@@ -12,6 +12,7 @@ incremental==17.5.0
lxml==4.3.4
Mako==1.0.12
MarkupSafe==1.1.1
+mysqlclient==1.3.12
parsel==1.5.1
Pillow==6.0.0
pyasn1==0.4.5
diff --git a/src/database/AltConnector.py b/src/database/AltConnector.py
new file mode 100644
index 0000000..c84e990
--- /dev/null
+++ b/src/database/AltConnector.py
@@ -0,0 +1,21 @@
+from scrapy.conf import settings
+from utils.Singleton import Singleton
+import MySQLdb as sql
+import os
+
+
+class AltConnector(metaclass=Singleton):
+ __connection = None
+
+ @staticmethod
+ def get_connection():
+ if not AltConnector.__connection:
+ # TODO cant use db-port
+ AltConnector.__connection = sql.connect(
+ os.getenv('DB_HOST'),
+ os.getenv('DB_USERNAME'),
+ os.getenv('DB_PASSWORD'),
+ os.getenv('DB_DATABASE'),
+ charset='utf8',
+ port=int(os.getenv('DB_PORT')))
+ return AltConnector.__connection
diff --git a/src/database/connectors/AltBookConnector.py b/src/database/connectors/AltBookConnector.py
new file mode 100644
index 0000000..3fdadc3
--- /dev/null
+++ b/src/database/connectors/AltBookConnector.py
@@ -0,0 +1,21 @@
+from ..AltConnector import AltConnector
+from database.models.Book import Book
+
+
+class AltBookConnector(AltConnector):
+ def save_book(self, item):
+ conn = self.get_connection()
+ cursor = conn.cursor()
+
+ # filter item keys in table field
+ real_keys = Book.__table__.columns.keys()
+ real_field = {k: v for k, v in item.items() if k in real_keys}
+ real_field.update({'image_path': item['images'][0]['path']})
+
+ sql = 'INSERT INTO books({keys}) VALUES({values}) ;'.format(
+ keys=', '.join(real_field.keys()),
+ values=', '.join(list('%s' for key in real_field.keys()))
+ )
+
+ cursor.execute(sql, real_field.values())
+ conn.commit()
diff --git a/src/spiders/books_spider.py b/src/spiders/books_spider.py
index e14ce74..4462ac1 100644
--- a/src/spiders/books_spider.py
+++ b/src/spiders/books_spider.py
@@ -2,6 +2,7 @@ from scrapy import Spider
import re
from items.BookItem import BookItem
from database.connectors.BookConnector import BookConnector
+from database.connectors.AltBookConnector import AltBookConnector
class BooksSpider(Spider, BookConnector):
@@ -56,4 +57,4 @@ class BooksSpider(Spider, BookConnector):
if 'Four' in class_all:
return 4
if 'Five' in class_all:
- return 5
\ No newline at end of file
+ return 5
--
GitLab