diff --git a/requirments.txt b/requirments.txt index 00f7343300a4d6eafb565c57b224953cd7e1b779..041bd3a7c68c487243b49221b2a11bc6d2c3a251 100644 --- a/requirments.txt +++ b/requirments.txt @@ -12,6 +12,7 @@ incremental==17.5.0 lxml==4.3.4 Mako==1.0.12 MarkupSafe==1.1.1 +mysqlclient==1.3.12 parsel==1.5.1 Pillow==6.0.0 pyasn1==0.4.5 diff --git a/src/database/AltConnector.py b/src/database/AltConnector.py new file mode 100644 index 0000000000000000000000000000000000000000..c84e990f7dff42a52385c462ec43b97150964d2a --- /dev/null +++ b/src/database/AltConnector.py @@ -0,0 +1,21 @@ +from scrapy.conf import settings +from utils.Singleton import Singleton +import MySQLdb as sql +import os + + +class AltConnector(metaclass=Singleton): + __connection = None + + @staticmethod + def get_connection(): + if not AltConnector.__connection: + # TODO cant use db-port + AltConnector.__connection = sql.connect( + os.getenv('DB_HOST'), + os.getenv('DB_USERNAME'), + os.getenv('DB_PASSWORD'), + os.getenv('DB_DATABASE'), + charset='utf8', + port=int(os.getenv('DB_PORT'))) + return AltConnector.__connection diff --git a/src/database/connectors/AltBookConnector.py b/src/database/connectors/AltBookConnector.py new file mode 100644 index 0000000000000000000000000000000000000000..3fdadc3ef4a35e93bc275fa92e24f75e36697875 --- /dev/null +++ b/src/database/connectors/AltBookConnector.py @@ -0,0 +1,21 @@ +from ..AltConnector import AltConnector +from database.models.Book import Book + + +class AltBookConnector(AltConnector): + def save_book(self, item): + conn = self.get_connection() + cursor = conn.cursor() + + # filter item keys in table field + real_keys = Book.__table__.columns.keys() + real_field = {k: v for k, v in item.items() if k in real_keys} + real_field.update({'image_path': item['images'][0]['path']}) + + sql = 'INSERT INTO books({keys}) VALUES({values}) ;'.format( + keys=', '.join(real_field.keys()), + values=', '.join(list('%s' for key in real_field.keys())) + ) + + cursor.execute(sql, real_field.values()) + conn.commit() diff --git a/src/spiders/books_spider.py b/src/spiders/books_spider.py index e14ce74d2cd4020291a64f31c70a444e0caf5547..4462ac1cfac4f280d548317ff0d1ade21060b326 100644 --- a/src/spiders/books_spider.py +++ b/src/spiders/books_spider.py @@ -2,6 +2,7 @@ from scrapy import Spider import re from items.BookItem import BookItem from database.connectors.BookConnector import BookConnector +from database.connectors.AltBookConnector import AltBookConnector class BooksSpider(Spider, BookConnector): @@ -56,4 +57,4 @@ class BooksSpider(Spider, BookConnector): if 'Four' in class_all: return 4 if 'Five' in class_all: - return 5 \ No newline at end of file + return 5