Merge branch 'develop' into 'master'

Develop

See merge request !2

requirments.txt

@@ -12,6 +12,7 @@ incremental==17.5.0
 lxml==4.3.4
 Mako==1.0.12
 MarkupSafe==1.1.1
+mysqlclient==1.3.12
 parsel==1.5.1
 Pillow==6.0.0
 pyasn1==0.4.5

src/database/AltConnector.py

+from scrapy.conf import settings
+from utils.Singleton import Singleton
+import MySQLdb as sql
+import os
+
+
+class AltConnector(metaclass=Singleton):
+    __connection = None
+
+    @staticmethod
+    def get_connection():
+        if not AltConnector.__connection:
+            # TODO cant use db-port
+            AltConnector.__connection = sql.connect(
+                os.getenv('DB_HOST'),
+                os.getenv('DB_USERNAME'),
+                os.getenv('DB_PASSWORD'),
+                os.getenv('DB_DATABASE'),
+                charset='utf8',
+                port=int(os.getenv('DB_PORT')))
+        return AltConnector.__connection

src/database/connectors/AltBookConnector.py

+from ..AltConnector import AltConnector
+from database.models.Book import Book
+
+
+class AltBookConnector(AltConnector):
+    def save_book(self, item):
+        conn = self.get_connection()
+        cursor = conn.cursor()
+
+        # filter item keys in table field
+        real_keys = Book.__table__.columns.keys()
+        real_field = {k: v for k, v in item.items() if k in real_keys}
+        real_field.update({'image_path': item['images'][0]['path']})
+
+        sql = 'INSERT INTO books({keys}) VALUES({values}) ;'.format(
+            keys=', '.join(real_field.keys()),
+            values=', '.join(list('%s' for key in real_field.keys()))
+        )
+
+        cursor.execute(sql, real_field.values())
+        conn.commit()

src/spiders/books_spider.py

@@ -2,6 +2,7 @@ from scrapy import Spider
 import re
 from items.BookItem import BookItem
 from database.connectors.BookConnector import BookConnector
+from database.connectors.AltBookConnector import AltBookConnector
 
 
 class BooksSpider(Spider, BookConnector):
@@ -56,4 +57,4 @@ class BooksSpider(Spider, BookConnector):
         if 'Four' in class_all:
             return 4
         if 'Five' in class_all:
-            return 5
\ No newline at end of file
+            return 5