From c85dce46d4f557475dd2111a8bb394d87b905c36 Mon Sep 17 00:00:00 2001
From: Gusev Anton <gusev_aa@groupbwt.com>
Date: Tue, 11 Jun 2019 19:17:57 +0300
Subject: [PATCH] change project structure
---
.gitignore | 2 +
.../tutorial/__init__.py => sql/.gitignore | 0
src/.env.example | 1 +
src/__init__.py | 0
.../tutorial/items.py => items/BookItem.py} | 6 --
.../tutorial/spiders => items}/__init__.py | 0
src/middlewares/__init__.py | 4 +
.../tutorial => middlewares}/middlewares.py | 0
src/pipelines/__init__.py | 4 +
.../tutorial => pipelines}/pipelines.py | 0
src/{tutorial => }/scrapy.cfg | 2 +-
src/{tutorial/tutorial => }/settings.py | 4 +-
src/spiders/__init__.py | 4 +
.../tutorial => }/spiders/books_spider.py | 3 +-
.../tutorial => }/spiders/quotes_spider.py | 0
src/tutorial/alembic.ini | 74 ------------------
src/tutorial/alembic/README | 1 -
src/tutorial/alembic/env.py | 75 -------------------
src/tutorial/alembic/script.py.mako | 24 ------
19 files changed, 20 insertions(+), 184 deletions(-)
rename src/tutorial/tutorial/__init__.py => sql/.gitignore (100%)
create mode 100644 src/.env.example
create mode 100644 src/__init__.py
rename src/{tutorial/tutorial/items.py => items/BookItem.py} (82%)
rename src/{tutorial/tutorial/spiders => items}/__init__.py (100%)
create mode 100644 src/middlewares/__init__.py
rename src/{tutorial/tutorial => middlewares}/middlewares.py (100%)
create mode 100644 src/pipelines/__init__.py
rename src/{tutorial/tutorial => pipelines}/pipelines.py (100%)
rename src/{tutorial => }/scrapy.cfg (89%)
rename src/{tutorial/tutorial => }/settings.py (97%)
create mode 100644 src/spiders/__init__.py
rename src/{tutorial/tutorial => }/spiders/books_spider.py (98%)
rename src/{tutorial/tutorial => }/spiders/quotes_spider.py (100%)
delete mode 100644 src/tutorial/alembic.ini
delete mode 100644 src/tutorial/alembic/README
delete mode 100644 src/tutorial/alembic/env.py
delete mode 100644 src/tutorial/alembic/script.py.mako
diff --git a/.gitignore b/.gitignore
index 2d56f81..3216ae5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,6 @@
.*
!.gitignore
+!.env.example
*.pyc
+__pycache__
*.json
\ No newline at end of file
diff --git a/src/tutorial/tutorial/__init__.py b/sql/.gitignore
similarity index 100%
rename from src/tutorial/tutorial/__init__.py
rename to sql/.gitignore
diff --git a/src/.env.example b/src/.env.example
new file mode 100644
index 0000000..4287ca8
--- /dev/null
+++ b/src/.env.example
@@ -0,0 +1 @@
+#
\ No newline at end of file
diff --git a/src/__init__.py b/src/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/tutorial/tutorial/items.py b/src/items/BookItem.py
similarity index 82%
rename from src/tutorial/tutorial/items.py
rename to src/items/BookItem.py
index 42f495d..205efbc 100644
--- a/src/tutorial/tutorial/items.py
+++ b/src/items/BookItem.py
@@ -8,12 +8,6 @@
import scrapy
-class TutorialItem(scrapy.Item):
- # define the fields for your item here like:
- # name = scrapy.Field()
- pass
-
-
class BookItem(scrapy.Item):
url = scrapy.Field()
diff --git a/src/tutorial/tutorial/spiders/__init__.py b/src/items/__init__.py
similarity index 100%
rename from src/tutorial/tutorial/spiders/__init__.py
rename to src/items/__init__.py
diff --git a/src/middlewares/__init__.py b/src/middlewares/__init__.py
new file mode 100644
index 0000000..ebd689a
--- /dev/null
+++ b/src/middlewares/__init__.py
@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.
diff --git a/src/tutorial/tutorial/middlewares.py b/src/middlewares/middlewares.py
similarity index 100%
rename from src/tutorial/tutorial/middlewares.py
rename to src/middlewares/middlewares.py
diff --git a/src/pipelines/__init__.py b/src/pipelines/__init__.py
new file mode 100644
index 0000000..ebd689a
--- /dev/null
+++ b/src/pipelines/__init__.py
@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.
diff --git a/src/tutorial/tutorial/pipelines.py b/src/pipelines/pipelines.py
similarity index 100%
rename from src/tutorial/tutorial/pipelines.py
rename to src/pipelines/pipelines.py
diff --git a/src/tutorial/scrapy.cfg b/src/scrapy.cfg
similarity index 89%
rename from src/tutorial/scrapy.cfg
rename to src/scrapy.cfg
index d79f6f1..5bb0b9f 100644
--- a/src/tutorial/scrapy.cfg
+++ b/src/scrapy.cfg
@@ -4,7 +4,7 @@
# https://scrapyd.readthedocs.io/en/latest/deploy.html
[settings]
-default = tutorial.settings
+default = settings
[deploy]
#url = http://localhost:6800/
diff --git a/src/tutorial/tutorial/settings.py b/src/settings.py
similarity index 97%
rename from src/tutorial/tutorial/settings.py
rename to src/settings.py
index 66cd185..1f3f738 100644
--- a/src/tutorial/tutorial/settings.py
+++ b/src/settings.py
@@ -11,8 +11,8 @@
BOT_NAME = 'tutorial'
-SPIDER_MODULES = ['tutorial.spiders']
-NEWSPIDER_MODULE = 'tutorial.spiders'
+SPIDER_MODULES = ['spiders']
+NEWSPIDER_MODULE = 'spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
diff --git a/src/spiders/__init__.py b/src/spiders/__init__.py
new file mode 100644
index 0000000..ebd689a
--- /dev/null
+++ b/src/spiders/__init__.py
@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.
diff --git a/src/tutorial/tutorial/spiders/books_spider.py b/src/spiders/books_spider.py
similarity index 98%
rename from src/tutorial/tutorial/spiders/books_spider.py
rename to src/spiders/books_spider.py
index 9c30a11..5fa6448 100644
--- a/src/tutorial/tutorial/spiders/books_spider.py
+++ b/src/spiders/books_spider.py
@@ -1,6 +1,7 @@
import scrapy
import re
-from ..items import BookItem
+from items.BookItem import BookItem
+
class BooksSpider(scrapy.Spider):
name = 'books'
diff --git a/src/tutorial/tutorial/spiders/quotes_spider.py b/src/spiders/quotes_spider.py
similarity index 100%
rename from src/tutorial/tutorial/spiders/quotes_spider.py
rename to src/spiders/quotes_spider.py
diff --git a/src/tutorial/alembic.ini b/src/tutorial/alembic.ini
deleted file mode 100644
index cc41841..0000000
--- a/src/tutorial/alembic.ini
+++ /dev/null
@@ -1,74 +0,0 @@
-# A generic, single database configuration.
-
-[alembic]
-# path to migration scripts
-script_location = alembic
-
-# template used to generate migration files
-# file_template = %%(rev)s_%%(slug)s
-
-# timezone to use when rendering the date
-# within the migration file as well as the filename.
-# string value is passed to dateutil.tz.gettz()
-# leave blank for localtime
-# timezone =
-
-# max length of characters to apply to the
-# "slug" field
-# truncate_slug_length = 40
-
-# set to 'true' to run the environment during
-# the 'revision' command, regardless of autogenerate
-# revision_environment = false
-
-# set to 'true' to allow .pyc and .pyo files without
-# a source .py file to be detected as revisions in the
-# versions/ directory
-# sourceless = false
-
-# version location specification; this defaults
-# to alembic/versions. When using multiple version
-# directories, initial revisions must be specified with --version-path
-# version_locations = %(here)s/bar %(here)s/bat alembic/versions
-
-# the output encoding used when revision files
-# are written from script.py.mako
-# output_encoding = utf-8
-
-sqlalchemy.url = driver://user:pass@localhost/dbname
-
-
-# Logging configuration
-[loggers]
-keys = root,sqlalchemy,alembic
-
-[handlers]
-keys = console
-
-[formatters]
-keys = generic
-
-[logger_root]
-level = WARN
-handlers = console
-qualname =
-
-[logger_sqlalchemy]
-level = WARN
-handlers =
-qualname = sqlalchemy.engine
-
-[logger_alembic]
-level = INFO
-handlers =
-qualname = alembic
-
-[handler_console]
-class = StreamHandler
-args = (sys.stderr,)
-level = NOTSET
-formatter = generic
-
-[formatter_generic]
-format = %(levelname)-5.5s [%(name)s] %(message)s
-datefmt = %H:%M:%S
diff --git a/src/tutorial/alembic/README b/src/tutorial/alembic/README
deleted file mode 100644
index 22a269b..0000000
--- a/src/tutorial/alembic/README
+++ /dev/null
@@ -1 +0,0 @@
-Generic single-database configuration.alembic list_templates
\ No newline at end of file
diff --git a/src/tutorial/alembic/env.py b/src/tutorial/alembic/env.py
deleted file mode 100644
index 15cb472..0000000
--- a/src/tutorial/alembic/env.py
+++ /dev/null
@@ -1,75 +0,0 @@
-
-from logging.config import fileConfig
-
-from sqlalchemy import engine_from_config
-from sqlalchemy import pool
-
-from alembic import context
-
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
-config = context.config
-
-# Interpret the config file for Python logging.
-# This line sets up loggers basically.
-fileConfig(config.config_file_name)
-
-# add your model's MetaData object here
-# for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
-target_metadata = None
-
-# other values from the config, defined by the needs of env.py,
-# can be acquired:
-# my_important_option = config.get_main_option("my_important_option")
-# ... etc.
-
-
-def run_migrations_offline():
- """Run migrations in 'offline' mode.
-
- This configures the context with just a URL
- and not an Engine, though an Engine is acceptable
- here as well. By skipping the Engine creation
- we don't even need a DBAPI to be available.
-
- Calls to context.execute() here emit the given string to the
- script output.
-
- """
- url = config.get_main_option("sqlalchemy.url")
- context.configure(
- url=url, target_metadata=target_metadata, literal_binds=True
- )
-
- with context.begin_transaction():
- context.run_migrations()
-
-
-def run_migrations_online():
- """Run migrations in 'online' mode.
-
- In this scenario we need to create an Engine
- and associate a connection with the context.
-
- """
- connectable = engine_from_config(
- config.get_section(config.config_ini_section),
- prefix="sqlalchemy.",
- poolclass=pool.NullPool,
- )
-
- with connectable.connect() as connection:
- context.configure(
- connection=connection, target_metadata=target_metadata
- )
-
- with context.begin_transaction():
- context.run_migrations()
-
-
-if context.is_offline_mode():
- run_migrations_offline()
-else:
- run_migrations_online()
diff --git a/src/tutorial/alembic/script.py.mako b/src/tutorial/alembic/script.py.mako
deleted file mode 100644
index 2c01563..0000000
--- a/src/tutorial/alembic/script.py.mako
+++ /dev/null
@@ -1,24 +0,0 @@
-"""${message}
-
-Revision ID: ${up_revision}
-Revises: ${down_revision | comma,n}
-Create Date: ${create_date}
-
-"""
-from alembic import op
-import sqlalchemy as sa
-${imports if imports else ""}
-
-# revision identifiers, used by Alembic.
-revision = ${repr(up_revision)}
-down_revision = ${repr(down_revision)}
-branch_labels = ${repr(branch_labels)}
-depends_on = ${repr(depends_on)}
-
-
-def upgrade():
- ${upgrades if upgrades else "pass"}
-
-
-def downgrade():
- ${downgrades if downgrades else "pass"}
--
GitLab