scrapy-ingest 1.0.0__tar.gz

scrapy_ingest-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Fawad Ali
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scrapy_ingest-1.0.0/PKG-INFO

@@ -0,0 +1,163 @@
+Metadata-Version: 2.4
+Name: scrapy-ingest
+Version: 1.0.0
+Summary: Scrapy extension for database ingestion with job/spider tracking
+Home-page: https://github.com/fawadss1/scrapy_item_ingest
+Author: Fawad Ali
+Author-email: fawadstar6@gmail.com
+Project-URL: Documentation, https://scrapy-ingest.readthedocs.io/
+Project-URL: Source, https://github.com/fawadss1/scrapy_item_ingest
+Project-URL: Tracker, https://github.com/fawadss1/scrapy_item_ingest/issues
+Keywords: scrapy,database,postgresql,web-scraping,data-pipeline
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Framework :: Scrapy
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Database
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: scrapy>=2.13.3
+Requires-Dist: psycopg2-binary>=2.9.10
+Requires-Dist: itemadapter>=0.11.0
+Requires-Dist: SQLAlchemy>=2.0.41
+Requires-Dist: pytz>=2025.2
+Requires-Dist: w3lib>=1.22.0
+Provides-Extra: docs
+Requires-Dist: sphinx>=5.0.0; extra == "docs"
+Requires-Dist: sphinx_rtd_theme>=1.2.0; extra == "docs"
+Requires-Dist: myst-parser>=0.18.0; extra == "docs"
+Requires-Dist: sphinx-autodoc-typehints>=1.19.0; extra == "docs"
+Requires-Dist: sphinx-copybutton>=0.5.0; extra == "docs"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: black>=22.0.0; extra == "dev"
+Requires-Dist: flake8>=5.0.0; extra == "dev"
+Requires-Dist: mypy>=0.991; extra == "dev"
+Requires-Dist: pre-commit>=2.20.0; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == "test"
+Requires-Dist: pytest-cov>=4.0.0; extra == "test"
+Requires-Dist: pytest-mock>=3.8.0; extra == "test"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Scrapy Ingest
+
+A Scrapy addon that saves **items, requests, logs, and stats** to PostgreSQL — with parent_url tracking, failed-request errors, and full job log capture (including `print()`).
+
+## Install
+
+```bash
+pip install scrapy-ingest
+```
+
+## Minimal setup (settings.py)
+
+Only the item pipeline is required — requests, logs, stats, parent_url, and error logging are enabled automatically:
+
+```python
+ITEM_PIPELINES = {
+    "scrapy_ingest.pipelines.DbInsertPipeline": 300,
+}
+
+# Pick ONE of the two database config styles:
+DB_URL = "postgresql://user:password@localhost:5432/database"
+# Or use discrete fields (avoids URL encoding):
+# DB_TYPE = "postgres"
+# DB_HOST = "localhost"
+# DB_PORT = 5432
+# DB_USER = "user"
+# DB_PASSWORD = "password"
+# DB_NAME = "database"
+
+# Optional
+CREATE_TABLES = True     # auto-create tables on first run (default True)
+JOB_ID = 1               # or omit; a unique id is generated per crawl
+INGEST_BATCH_SIZE = 50   # flush when this many rows are buffered
+```
+
+Run your spider:
+
+```bash
+scrapy crawl your_spider
+```
+
+Log level follows Scrapy `LOG_LEVEL`.
+
+## What is stored
+
+| Table          | Contents                                                                                                |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| `jobs`         | One row per crawl: `id`, unique `job_id` string, spider, status, start/finish, counts, items/min, stats |
+| `job_items`    | JSON items (`crawled_at` added). `job_id` = `jobs.id` (CASCADE)                                         |
+| `job_requests` | url, parent_url, parent_id, status, response_time, error, success. `job_id` = `jobs.id` (CASCADE)       |
+| `job_logs`     | time, logger, level, message, exception. `job_id` = `jobs.id` (CASCADE)                                 |
+
+Request `parent_url` is the page that scheduled the request (e.g. sitemap → product). Start URLs are `null`.
+
+Data flushes on batch size, every 10s, and on engine/process stop.
+
+## Troubleshooting
+
+- Password has special characters like `@` or `$`?
+  - In a URL, encode them: `@` -> `%40`, `$` -> `%24`.
+  - Example: `postgresql://user:PAK%40swat1%24@localhost:5432/db`
+  - Or use the discrete fields (no encoding needed).
+- **Yield items** from callbacks (not only `return` inside a generator).
+
+## Useful settings (optional)
+
+- `DB_TYPE` (default: `postgres`) — used when building a URL from `DB_HOST` / `DB_*` fields
+- `INGEST_BATCH_SIZE` (default: `50`) — flush when this many items+requests+logs are buffered
+- `INGEST_FLUSH_INTERVAL` (default: `10`) — periodic flush in seconds
+- `CREATE_TABLES` (default: `True`) — create tables on startup
+- `ITEMS_TABLE`, `REQUESTS_TABLE`, `LOGS_TABLE`, `JOBS_TABLE` — override table names
+- `TIMEZONE` (default: `Asia/Karachi`) — timezone for `created_at`
+- `JOB_ID` — omit to auto-generate a unique id (`spider-YYYYMMDDHHMMSS-xxxxxxxx`)
+
+## Standalone components
+
+If you only want part of the collection:
+
+```python
+# Items only
+ITEM_PIPELINES = {"scrapy_ingest.pipelines.ItemsPipeline": 300}
+
+# Requests only (parent_url + errors)
+ITEM_PIPELINES = {"scrapy_ingest.pipelines.RequestsPipeline": 300}
+
+# Logs only
+EXTENSIONS = {"scrapy_ingest.extensions.LoggingExtension": 500}
+```
+
+## Links
+
+- Docs: https://scrapy-ingest.readthedocs.io/
+- Changelog: docs/development/changelog.rst
+- Issues: https://github.com/fawadss1/scrapy_item_ingest/issues
+
+## License
+
+MIT License. See [LICENSE](LICENSE).

scrapy_ingest-1.0.0/README.md

@@ -0,0 +1,98 @@
+# Scrapy Ingest
+
+A Scrapy addon that saves **items, requests, logs, and stats** to PostgreSQL — with parent_url tracking, failed-request errors, and full job log capture (including `print()`).
+
+## Install
+
+```bash
+pip install scrapy-ingest
+```
+
+## Minimal setup (settings.py)
+
+Only the item pipeline is required — requests, logs, stats, parent_url, and error logging are enabled automatically:
+
+```python
+ITEM_PIPELINES = {
+    "scrapy_ingest.pipelines.DbInsertPipeline": 300,
+}
+
+# Pick ONE of the two database config styles:
+DB_URL = "postgresql://user:password@localhost:5432/database"
+# Or use discrete fields (avoids URL encoding):
+# DB_TYPE = "postgres"
+# DB_HOST = "localhost"
+# DB_PORT = 5432
+# DB_USER = "user"
+# DB_PASSWORD = "password"
+# DB_NAME = "database"
+
+# Optional
+CREATE_TABLES = True     # auto-create tables on first run (default True)
+JOB_ID = 1               # or omit; a unique id is generated per crawl
+INGEST_BATCH_SIZE = 50   # flush when this many rows are buffered
+```
+
+Run your spider:
+
+```bash
+scrapy crawl your_spider
+```
+
+Log level follows Scrapy `LOG_LEVEL`.
+
+## What is stored
+
+| Table          | Contents                                                                                                |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| `jobs`         | One row per crawl: `id`, unique `job_id` string, spider, status, start/finish, counts, items/min, stats |
+| `job_items`    | JSON items (`crawled_at` added). `job_id` = `jobs.id` (CASCADE)                                         |
+| `job_requests` | url, parent_url, parent_id, status, response_time, error, success. `job_id` = `jobs.id` (CASCADE)       |
+| `job_logs`     | time, logger, level, message, exception. `job_id` = `jobs.id` (CASCADE)                                 |
+
+Request `parent_url` is the page that scheduled the request (e.g. sitemap → product). Start URLs are `null`.
+
+Data flushes on batch size, every 10s, and on engine/process stop.
+
+## Troubleshooting
+
+- Password has special characters like `@` or `$`?
+  - In a URL, encode them: `@` -> `%40`, `$` -> `%24`.
+  - Example: `postgresql://user:PAK%40swat1%24@localhost:5432/db`
+  - Or use the discrete fields (no encoding needed).
+- **Yield items** from callbacks (not only `return` inside a generator).
+
+## Useful settings (optional)
+
+- `DB_TYPE` (default: `postgres`) — used when building a URL from `DB_HOST` / `DB_*` fields
+- `INGEST_BATCH_SIZE` (default: `50`) — flush when this many items+requests+logs are buffered
+- `INGEST_FLUSH_INTERVAL` (default: `10`) — periodic flush in seconds
+- `CREATE_TABLES` (default: `True`) — create tables on startup
+- `ITEMS_TABLE`, `REQUESTS_TABLE`, `LOGS_TABLE`, `JOBS_TABLE` — override table names
+- `TIMEZONE` (default: `Asia/Karachi`) — timezone for `created_at`
+- `JOB_ID` — omit to auto-generate a unique id (`spider-YYYYMMDDHHMMSS-xxxxxxxx`)
+
+## Standalone components
+
+If you only want part of the collection:
+
+```python
+# Items only
+ITEM_PIPELINES = {"scrapy_ingest.pipelines.ItemsPipeline": 300}
+
+# Requests only (parent_url + errors)
+ITEM_PIPELINES = {"scrapy_ingest.pipelines.RequestsPipeline": 300}
+
+# Logs only
+EXTENSIONS = {"scrapy_ingest.extensions.LoggingExtension": 500}
+```
+
+## Links
+
+- Docs: https://scrapy-ingest.readthedocs.io/
+- Changelog: docs/development/changelog.rst
+- Issues: https://github.com/fawadss1/scrapy_item_ingest/issues
+
+## License
+
+MIT License. See [LICENSE](LICENSE).

scrapy_ingest-1.0.0/scrapy_ingest/__init__.py

@@ -0,0 +1,36 @@
+"""
+scrapy_ingest - A Scrapy extension for ingesting items, requests, logs, and stats into PostgreSQL.
+
+Enabling DbInsertPipeline auto-enables request logging (with parent_url),
+error logging, full job logs (including print()), and crawl stats.
+"""
+
+from .extensions.log_handler import install_early
+
+install_early()
+
+__version__ = "1.0.0"
+__author__ = "Fawad Ali"
+__description__ = "Scrapy extension for database ingestion with job/spider tracking"
+
+from .pipelines.main import DbInsertPipeline
+from .extensions.logging import LoggingExtension
+from .extensions.stats import StatsExtension
+from .pipelines.items import ItemsPipeline
+from .pipelines.requests import RequestsPipeline
+from .extensions.request_logger import RequestLogger
+from .config.settings import Settings, validate_settings
+
+__all__ = [
+    "DbInsertPipeline",
+    "LoggingExtension",
+    "StatsExtension",
+    "ItemsPipeline",
+    "RequestsPipeline",
+    "RequestLogger",
+    "Settings",
+    "validate_settings",
+    "__version__",
+    "__author__",
+    "__description__",
+]

scrapy_ingest-1.0.0/scrapy_ingest/bootstrap/__init__.py

@@ -0,0 +1,5 @@
+"""Bootstrap helpers that auto-enable ingest components."""
+
+from .bootstrap import attach_runtime_hooks, enable_ingest
+
+__all__ = ["attach_runtime_hooks", "enable_ingest"]

scrapy_ingest-1.0.0/scrapy_ingest/bootstrap/bootstrap.py

@@ -0,0 +1,85 @@
+"""Enable all ingest components from a single entry point (the item pipeline)."""
+
+import inspect
+
+from scrapy.core.engine import ExecutionEngine
+from scrapy.core.scraper import Scraper
+
+from ..collector import ensure_collector
+from ..extensions.logging import LoggingExtension
+from ..extensions.stats import StatsExtension
+from ..extensions.request_logger import RequestLogger
+from ..middleware import ErrorMiddleware, _inject_into_scraper
+
+
+def _inject_error_middleware(crawler, engine=None):
+    if getattr(crawler, "_ingest_error_mw_hooked", False):
+        return
+    try:
+        engine = engine or crawler.engine
+        mwman = engine.downloader.middleware
+    except (AttributeError, RuntimeError):
+        return
+
+    mw = ErrorMiddleware()
+    mw.crawler = crawler
+    mw.collector = crawler.ingest_collector
+    mwman._add_middleware(mw)
+    crawler._ingest_error_mw_hooked = True
+
+
+def _find_building(cls):
+    """Find an in-progress constructor instance of *cls* on the call stack."""
+    for frame_info in inspect.stack()[1:]:
+        obj = frame_info.frame.f_locals.get("self")
+        if isinstance(obj, cls):
+            return obj
+    return None
+
+
+def attach_runtime_hooks(crawler):
+    """
+    Attach parent_url + error hooks.
+
+    ITEM_PIPELINES load inside Scraper.__init__ (before crawler.engine is set),
+    so we locate the in-progress Scraper/Engine on the stack and patch those.
+    """
+    if not hasattr(crawler, "ingest_parent_by_fp"):
+        crawler.ingest_parent_by_fp = {}
+
+    scraper = _find_building(Scraper)
+    if scraper is not None:
+        _inject_into_scraper(crawler, scraper)
+    else:
+        try:
+            _inject_into_scraper(crawler, crawler.engine.scraper)
+        except (AttributeError, RuntimeError):
+            pass
+
+    engine = _find_building(ExecutionEngine)
+    if engine is not None and getattr(engine, "downloader", None) is not None:
+        _inject_error_middleware(crawler, engine)
+    else:
+        try:
+            _inject_error_middleware(crawler, crawler.engine)
+        except (AttributeError, RuntimeError):
+            pass
+
+
+def enable_ingest(crawler):
+    """
+    Idempotently enable request logging, error logging, parent_url tracking,
+    job logs, and stats. Called from DbInsertPipeline so projects only
+    need ITEM_PIPELINES.
+    """
+    if getattr(crawler, "_ingest_enabled", False):
+        return
+    crawler._ingest_enabled = True
+
+    ensure_collector(crawler)
+
+    LoggingExtension.from_crawler(crawler)
+    StatsExtension.from_crawler(crawler)
+    RequestLogger.from_crawler(crawler)
+
+    attach_runtime_hooks(crawler)

scrapy_ingest-1.0.0/scrapy_ingest/collector/__init__.py

@@ -0,0 +1,5 @@
+"""Shared batch collector for items, requests, logs, and stats."""
+
+from .collector import DataCollector, ensure_collector
+
+__all__ = ["DataCollector", "ensure_collector"]

scrapy_ingest-1.0.0/scrapy_ingest/collector/collector.py

@@ -0,0 +1,78 @@
+"""Thread-safe batch buffer for requests, items, logs, and stats."""
+from threading import Lock
+
+
+def ensure_collector(crawler):
+    """Return the crawler's shared DataCollector, creating it if needed."""
+    if not hasattr(crawler, "ingest_collector"):
+        crawler.ingest_collector = DataCollector()
+    return crawler.ingest_collector
+
+
+class DataCollector:
+    """
+    Collects requests, items, logs, and stats in batches.
+
+    Thread-safe. One instance is stored on the crawler and shared by the
+    pipeline, request logger, error middleware, and logging extension.
+    """
+
+    def __init__(self):
+        self.requests = []
+        self.items = []
+        self.logs = []
+        self.stats = None
+        self.lock = Lock()
+
+    def add_request(self, request_log: dict):
+        with self.lock:
+            self.requests.append(request_log)
+
+    def add_item(self, item: dict):
+        with self.lock:
+            self.items.append(item)
+
+    def add_log(self, log_entry: dict):
+        with self.lock:
+            self.logs.append(log_entry)
+
+    def set_stats(self, stats: dict):
+        with self.lock:
+            self.stats = stats
+
+    def get_and_clear(self):
+        """Return collected data and clear buffers. None if empty."""
+        with self.lock:
+            if not (self.requests or self.items or self.logs or self.stats):
+                return None
+
+            data = {
+                "requests": self.requests[:],
+                "items": self.items[:],
+                "logs": self.logs[:],
+            }
+            if self.stats:
+                data["stats"] = self.stats
+
+            self.requests.clear()
+            self.items.clear()
+            self.logs.clear()
+            self.stats = None
+            return data
+
+    def requeue(self, data: dict):
+        """Put data back at the front of the buffers after a failed flush."""
+        with self.lock:
+            self.requests = data.get("requests", []) + self.requests
+            self.items = data.get("items", []) + self.items
+            self.logs = data.get("logs", []) + self.logs
+            if data.get("stats") is not None and self.stats is None:
+                self.stats = data["stats"]
+
+    def has_data(self) -> bool:
+        with self.lock:
+            return bool(self.requests or self.items or self.logs or self.stats)
+
+    def size(self) -> int:
+        with self.lock:
+            return len(self.requests) + len(self.items) + len(self.logs)

scrapy_ingest-1.0.0/scrapy_ingest/config/__init__.py

@@ -0,0 +1,2 @@
+"""Configuration modules for scrapy_ingest."""
+

scrapy_ingest-1.0.0/scrapy_ingest/config/settings.py

@@ -0,0 +1,133 @@
+"""
+Module for managing and validating crawler settings.
+"""
+from urllib.parse import quote_plus
+
+
+class Settings:
+    """
+    Handles settings configuration for crawlers, providing access to default values,
+    database table names, and other operational parameters defined in crawler settings.
+    """
+
+    DEFAULT_ITEMS_TABLE = "job_items"
+    DEFAULT_REQUESTS_TABLE = "job_requests"
+    DEFAULT_LOGS_TABLE = "job_logs"
+    DEFAULT_JOBS_TABLE = "jobs"
+    DEFAULT_DB_TYPE = "postgres"
+    DEFAULT_TIMEZONE = "Asia/Karachi"
+    DEFAULT_BATCH_SIZE = 50
+    DEFAULT_FLUSH_INTERVAL = 10.0
+    _DB_SCHEMES = {
+        "postgres": "postgresql",
+        "postgresql": "postgresql",
+    }
+    _DB_PORTS = {
+        "postgres": 5432,
+        "postgresql": 5432,
+    }
+
+    def __init__(self, crawler_settings):
+        self.crawler_settings = crawler_settings
+
+    @property
+    def db_url(self):
+        """Database URL from DB_URL, or built from discrete DB_* fields."""
+        url = self.crawler_settings.get("DB_URL")
+        if url:
+            return url
+
+        host = self.crawler_settings.get("DB_HOST")
+        if not host:
+            return None
+
+        user = self.crawler_settings.get("DB_USER") or ""
+        password = self.crawler_settings.get("DB_PASSWORD") or ""
+        port = self.crawler_settings.get("DB_PORT", self._default_port())
+        name = self.crawler_settings.get("DB_NAME") or ""
+        scheme = self._DB_SCHEMES.get(self.db_type, self.db_type)
+        return (
+            f"{scheme}://{quote_plus(str(user))}:{quote_plus(str(password))}"
+            f"@{host}:{port}/{name}"
+        )
+
+    @property
+    def db_type(self):
+        """Database engine from ``DB_TYPE`` (default: postgres)."""
+        raw = self.crawler_settings.get("DB_TYPE", self.DEFAULT_DB_TYPE)
+        if raw in (None, ""):
+            return self.DEFAULT_DB_TYPE
+        return str(raw).strip().lower()
+
+    def _default_port(self):
+        return self._DB_PORTS.get(self.db_type, 5432)
+
+    @property
+    def db_items_table(self):
+        return self.crawler_settings.get("ITEMS_TABLE", self.DEFAULT_ITEMS_TABLE)
+
+    @property
+    def db_requests_table(self):
+        return self.crawler_settings.get("REQUESTS_TABLE", self.DEFAULT_REQUESTS_TABLE)
+
+    @property
+    def db_logs_table(self):
+        return self.crawler_settings.get("LOGS_TABLE", self.DEFAULT_LOGS_TABLE)
+
+    @property
+    def db_jobs_table(self):
+        return (
+            self.crawler_settings.get("JOBS_TABLE")
+            or self.crawler_settings.get("DETAILS_TABLE")
+            or self.DEFAULT_JOBS_TABLE
+        )
+
+    @property
+    def create_tables(self):
+        return self.crawler_settings.getbool("CREATE_TABLES", True)
+
+    @property
+    def ingest_batch_size(self):
+        return self.crawler_settings.getint("INGEST_BATCH_SIZE", self.DEFAULT_BATCH_SIZE)
+
+    @property
+    def ingest_flush_interval(self):
+        return self.crawler_settings.getfloat(
+            "INGEST_FLUSH_INTERVAL", self.DEFAULT_FLUSH_INTERVAL
+        )
+
+    def get_tz(self):
+        return self.crawler_settings.get("TIMEZONE", self.DEFAULT_TIMEZONE)
+
+    @staticmethod
+    def get_identifier_column():
+        return "job_id"
+
+    def get_identifier_value(self, spider):
+        """Use JOB_ID if set; otherwise a unique generated id (cached per crawl)."""
+        from ..utils.job_id import cache_job_id, cached_job_id, generate_job_id
+
+        configured = self.crawler_settings.get("JOB_ID", None)
+        if configured not in (None, ""):
+            return cache_job_id(spider, str(configured))
+
+        existing = cached_job_id(spider)
+        if existing:
+            return existing
+
+        return cache_job_id(spider, generate_job_id(getattr(spider, "name", "spider")))
+
+
+def validate_settings(settings):
+    """Validate configuration settings. Accepts DB_URL or discrete DB_* fields."""
+    if settings.db_type not in settings._DB_SCHEMES:
+        supported = ", ".join(sorted(settings._DB_SCHEMES))
+        raise ValueError(
+            f"Unsupported DB_TYPE={settings.db_type!r}. Supported: {supported}"
+        )
+    if not settings.db_url:
+        raise ValueError(
+            "Database connection is required: set DB_URL or "
+            "DB_HOST / DB_USER / DB_PASSWORD / DB_NAME"
+        )
+    return True

scrapy_ingest-1.0.0/scrapy_ingest/database/__init__.py

@@ -0,0 +1,2 @@
+"""Database modules for scrapy_ingest."""
+