Skip to content
Snippets Groups Projects
Commit c70deaf5 authored by Anton Gusev's avatar Anton Gusev
Browse files

add class for connect to database

parent 7b3f7efd
No related branches found
No related tags found
5 merge requests!5Develop,!4Develop,!3Develop,!2Develop,!1Develop
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from scrapy.conf import settings
class Connector:
connection = None
engine = create_engine(settings['CONNECTION_STRING'])
Session = sessionmaker(bind=engine)
@staticmethod
def get_session():
return Connector.Session()
@staticmethod
def get_connection():
if not Connector.connection:
Connector.connection = Connector.engine.connect()
return Connector.connection
from ..Connector import Connector
from ..models.Book import Book
class BookConnector(Connector):
def save_book(self, item):
session = self.get_session()
book = Book()
book.update(item)
book.image_path = item['images'][0]['path']
session.add(book)
session.commit()
\ No newline at end of file
class DatabasePipeline:
name = 'DatabasePipeline'
def process_item(self, item, spider):
spider.save_book(item)
return item
import scrapy
import re
from items.BookItem import BookItem
from database.connectors.BookConnector import BookConnector
import logging
class BooksSpider(scrapy.Spider):
class BooksSpider(BookConnector, scrapy.Spider):
name = 'books'
start_urls = [
'http://books.toscrape.com/catalogue/page-1.html'
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment