Skip to content
Snippets Groups Projects
Commit d7290700 authored by Anton Gusev's avatar Anton Gusev
Browse files

add alternative database connection using mysqlclient

parent eb39af84
No related branches found
No related tags found
4 merge requests!5Develop,!4Develop,!3Develop,!2Develop
...@@ -12,6 +12,7 @@ incremental==17.5.0 ...@@ -12,6 +12,7 @@ incremental==17.5.0
lxml==4.3.4 lxml==4.3.4
Mako==1.0.12 Mako==1.0.12
MarkupSafe==1.1.1 MarkupSafe==1.1.1
mysqlclient==1.3.12
parsel==1.5.1 parsel==1.5.1
Pillow==6.0.0 Pillow==6.0.0
pyasn1==0.4.5 pyasn1==0.4.5
......
from scrapy.conf import settings
from utils.Singleton import Singleton
import MySQLdb as sql
import os
class AltConnector(metaclass=Singleton):
__connection = None
@staticmethod
def get_connection():
if not AltConnector.__connection:
# TODO cant use db-port
AltConnector.__connection = sql.connect(
os.getenv('DB_HOST'),
os.getenv('DB_USERNAME'),
os.getenv('DB_PASSWORD'),
os.getenv('DB_DATABASE'),
charset='utf8',
port=int(os.getenv('DB_PORT')))
return AltConnector.__connection
from ..AltConnector import AltConnector
from database.models.Book import Book
class AltBookConnector(AltConnector):
def save_book(self, item):
conn = self.get_connection()
cursor = conn.cursor()
# filter item keys in table field
real_keys = Book.__table__.columns.keys()
real_field = {k: v for k, v in item.items() if k in real_keys}
real_field.update({'image_path': item['images'][0]['path']})
sql = 'INSERT INTO books({keys}) VALUES({values}) ;'.format(
keys=', '.join(real_field.keys()),
values=', '.join(list('%s' for key in real_field.keys()))
)
cursor.execute(sql, real_field.values())
conn.commit()
...@@ -2,6 +2,7 @@ from scrapy import Spider ...@@ -2,6 +2,7 @@ from scrapy import Spider
import re import re
from items.BookItem import BookItem from items.BookItem import BookItem
from database.connectors.BookConnector import BookConnector from database.connectors.BookConnector import BookConnector
from database.connectors.AltBookConnector import AltBookConnector
class BooksSpider(Spider, BookConnector): class BooksSpider(Spider, BookConnector):
...@@ -56,4 +57,4 @@ class BooksSpider(Spider, BookConnector): ...@@ -56,4 +57,4 @@ class BooksSpider(Spider, BookConnector):
if 'Four' in class_all: if 'Four' in class_all:
return 4 return 4
if 'Five' in class_all: if 'Five' in class_all:
return 5 return 5
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment