scrapy-item-ingest 0.2.4__py3-none-any.whl

scrapy_item_ingest/__init__.py

@@ -0,0 +1,48 @@
+"""
+scrapy_item_ingest - A Scrapy extension for ingesting items and requests into databases.
+
+This package provides pipelines and extensions for storing scraped data, tracking requests,
+and logging spider events to PostgreSQL databases with support for both spider-based and
+job-based identification.
+
+Main Components:
+- DbInsertPipeline: Combined pipeline for items and requests
+- LoggingExtension: Extension for logging spider events
+- ItemsPipeline: Standalone items processing pipeline
+- RequestsPipeline: Standalone requests tracking pipeline
+"""
+
+__version__ = "0.2.4"
+__author__ = "Fawad Ali"
+__description__ = "Scrapy extension for database ingestion with job/spider tracking"
+
+# Import main classes directly from organized modules
+from .pipelines.main import DbInsertPipeline
+from .extensions.logging import LoggingExtension
+
+# Import individual components for advanced users
+from .pipelines.items import ItemsPipeline
+from .pipelines.requests import RequestsPipeline
+
+# Import configuration utilities
+from .config.settings import Settings, validate_settings
+
+# Define what gets imported with "from scrapy_item_ingest import *"
+__all__ = [
+    # Main classes (most commonly used)
+    'DbInsertPipeline',
+    'LoggingExtension',
+
+    # Individual components
+    'ItemsPipeline',
+    'RequestsPipeline',
+
+    # Configuration
+    'Settings',
+    'validate_settings',
+
+    # Package metadata
+    '__version__',
+    '__author__',
+    '__description__',
+]

scrapy_item_ingest/config/__init__.py

@@ -0,0 +1,2 @@
+"""Configuration modules for scrapy_item_ingest."""
+

scrapy_item_ingest/config/settings.py

@@ -0,0 +1,134 @@
+"""
+Module for managing and validating crawler settings.
+
+This module provides utility classes and functions for handling the settings of
+a crawler, including the database configuration, operational parameters, and
+customizable options. The primary class `Settings` provides an interface for
+accessing settings dynamically and offers default fallbacks where values are
+not explicitly defined. Utility function `validate_settings` ensures critical
+configuration is present.
+"""
+
+
+class Settings:
+    """
+    Handles settings configuration for crawlers, providing access to default values,
+    database table names, and other operational parameters defined in crawler settings.
+
+    This class facilitates the standardized management and retrieval of settings that
+    are essential for database operations and crawler configurations. Its purpose
+    is to provide default fallbacks and dynamically adapt to user-specified settings.
+    """
+
+    DEFAULT_ITEMS_TABLE = 'job_items'
+    DEFAULT_REQUESTS_TABLE = 'job_requests'
+    DEFAULT_LOGS_TABLE = 'job_logs'
+    DEFAULT_TIMEZONE = "Asia/Karachi"
+
+    def __init__(self, crawler_settings):
+        self.crawler_settings = crawler_settings
+
+    @property
+    def db_url(self):
+        """
+        Provides access to the database URL from the crawler settings.
+
+        This property is used to retrieve the database URL defined in the
+        crawler's settings. It is helpful when a database configuration
+        needs to be accessed dynamically.
+
+        :return: The database URL as defined in the crawler's configuration
+        :rtype: str or None
+        """
+        return self.crawler_settings.get('DB_URL')
+
+    @property
+    def db_type(self):
+        """
+        Retrieves the database type from the crawler settings.
+
+        This property fetches the value assigned to the key `DB_TYPE` within
+        the `crawler_settings`. Defaults to 'postgres' if the key is not set.
+
+        :return: The database type as a string.
+        :rtype: str
+        """
+        return self.crawler_settings.get('DB_TYPE', 'postgres')
+
+    @property
+    def db_items_table(self):
+        """Return static table name for items"""
+        return self.crawler_settings.get('ITEMS_TABLE', self.DEFAULT_ITEMS_TABLE)
+
+    @property
+    def db_requests_table(self):
+        """
+        This property fetches the name of the database table used to store request
+        information. It retrieves the value from crawler settings if defined;
+        otherwise, it defaults to the value of `DEFAULT_REQUESTS_TABLE`.
+
+        :return: Name of the database table for storing requests.
+        :rtype: str
+        """
+        return self.crawler_settings.get('REQUESTS_TABLE', self.DEFAULT_REQUESTS_TABLE)
+
+    @property
+    def db_logs_table(self):
+        """
+        Retrieve the name of the database logs table.
+
+        This property fetches the value of the database logs table name
+        provided in the crawler settings. If the value is not explicitly
+        defined in the settings, it falls back to the default logs table.
+
+        :return: The name of the database logs table.
+        :rtype: Str
+        """
+        return self.crawler_settings.get('LOGS_TABLE', self.DEFAULT_LOGS_TABLE)
+
+    @property
+    def create_tables(self):
+        """
+        Retrieve the setting for creating database tables from crawler settings.
+
+        This property fetches the value of the 'CREATE_TABLES' option from the crawler
+        settings. If the option is not specified in the settings, it defaults to True.
+
+        :return: Boolean value indicating whether to create tables.
+        :rtype: Bool
+        """
+        return self.crawler_settings.getbool('CREATE_TABLES', True)
+
+    def get_tz(self):
+        """
+        Return the timezone string for the project.
+        This checks for a 'TIMEZONE' setting in the crawler settings and falls back to the default ('Asia/Karachi').
+        Returns:
+            str: The timezone string (e.g., 'Asia/Karachi').
+        """
+        return self.crawler_settings.get('TIMEZONE', self.DEFAULT_TIMEZONE)
+
+    @staticmethod
+    def get_identifier_column():
+        """Get the identifier column name"""
+        return "job_id"
+
+    def get_identifier_value(self, spider):
+        """Get the identifier value with smart fallback"""
+        job_id = self.crawler_settings.get('JOB_ID', None)
+
+        if self.create_tables:
+            # When creating tables, use JOB_ID if provided, else spider name
+            return job_id if job_id else spider.name
+        else:
+            # When using existing tables, use JOB_ID if provided, else spider name
+            return job_id if job_id else spider.name
+
+
+def validate_settings(settings):
+    """Validate configuration settings"""
+    if not settings.db_url:
+        raise ValueError("DB_URL must be set in settings")
+
+    # Job ID is now optional - will use spider name as fallback
+    return True

scrapy_item_ingest/database/__init__.py

@@ -0,0 +1,2 @@
+"""Database modules for scrapy_item_ingest."""
+

scrapy_item_ingest/database/connection.py

@@ -0,0 +1,150 @@
+# scrapy_item_ingest/database/connection.py
+
+import logging
+from typing import Optional, Any, Sequence
+from urllib.parse import urlsplit, urlunsplit, quote, unquote
+
+import psycopg2
+from psycopg2 import OperationalError
+
+
+class DBConnection:
+    """
+    PostgreSQL connection manager (singleton) with a small convenience API used by
+    pipelines and schema utilities. Supports either a DSN/URL or settings-based
+    configuration and exposes `connect/execute/commit/rollback/close` methods.
+    """
+
+    _instance = None  # Singleton instance
+    _connection = None
+    _db_url: Optional[str] = None
+    _logger = logging.getLogger(__name__)
+
+    def __new__(cls, db_url: Optional[str] = None):
+        # Ensure only one instance exists (singleton) and accept optional db_url
+        if cls._instance is None:
+            cls._instance = super(DBConnection, cls).__new__(cls)
+            if db_url:
+                cls._instance._db_url = db_url
+            cls._instance._initialize_connection()
+        else:
+            # If an URL is passed later and we don't have one stored yet, keep it
+            if db_url and cls._instance._db_url is None:
+                cls._instance._db_url = db_url
+                # Do not auto-reconnect here; next use will reconnect if needed
+        return cls._instance
+
+    def _normalize_dsn(self, dsn: str) -> str:
+        """Normalize a PostgreSQL DSN/URL by URL-encoding credentials if needed.
+        Handles passwords that mistakenly include raw '@' or '$' by treating the last
+        '@' as the boundary between credentials and host.
+        """
+        try:
+            if "://" not in dsn:
+                return dsn
+            scheme, rest = dsn.split("://", 1)
+            # Separate netloc and remaining path/query/fragment
+            if "/" in rest:
+                netloc, tail = rest.split("/", 1)
+                tail = "/" + tail
+            else:
+                netloc, tail = rest, ""
+            if "@" in netloc:
+                userinfo, hostport = netloc.rsplit("@", 1)
+                if ":" in userinfo:
+                    user, pwd = userinfo.split(":", 1)
+                    # Encode only if password contains reserved characters
+                    if any(c in pwd for c in "@:$ /\\"):
+                        user_enc = quote(unquote(user), safe="")
+                        pwd_enc = quote(pwd, safe="")
+                        netloc = f"{user_enc}:{pwd_enc}@{hostport}"
+            return f"{scheme}://{netloc}{tail}"
+        except Exception:
+            return dsn
+
+    def _initialize_connection(self):
+        """Initialize the PostgreSQL connection once (or reconnect if closed)."""
+        if self._connection is not None and getattr(self._connection, "closed", 0) == 0:
+            return
+
+        source = "unknown"
+        try:
+            if self._db_url:
+                source = "db_url"
+                dsn = self._normalize_dsn(self._db_url)
+                self._connection = psycopg2.connect(dsn)
+            else:
+                # Lazy import to avoid module-level dependency on Scrapy
+                from scrapy.utils.project import get_project_settings
+                settings = get_project_settings()
+                source = "Scrapy settings"
+                self._connection = psycopg2.connect(
+                    host=settings.get("DB_HOST"),
+                    port=settings.get("DB_PORT"),
+                    user=settings.get("DB_USER"),
+                    password=settings.get("DB_PASSWORD"),
+                    dbname=settings.get("DB_NAME"),
+                )
+            self._connection.autocommit = False  # manual commit per item
+        except OperationalError as e:
+            # Mask password in logs by not printing full URL; provide hint
+            self._logger.error(
+                "Failed to connect to database via %s: %s. "
+                "Verify DB settings or DSN (host, port, user, dbname).",
+                source,
+                str(e),
+            )
+            raise
+
+    # Public API expected by pipelines/schema
+    def connect(self) -> bool:
+        try:
+            self._initialize_connection()
+            return True
+        except Exception:
+            return False
+
+    def cursor(self):
+        if self._connection is None or getattr(self._connection, "closed", 1):
+            self._initialize_connection()
+        return self._connection.cursor()
+
+    def execute(self, sql: str, params: Sequence[Any] = None):
+        """Execute a SQL statement.
+        Returns the first row (tuple) if the statement produces a result set
+        (e.g., SELECT or INSERT ... RETURNING), otherwise returns None.
+        """
+        with self.cursor() as cur:
+            if params is not None:
+                cur.execute(sql, params)
+            else:
+                cur.execute(sql)
+            # If the statement returns rows, fetch one for callers expecting a value
+            if cur.description is not None:
+                row = cur.fetchone()
+                return row
+            return None
+
+    def commit(self):
+        if self._connection:
+            self._connection.commit()
+
+    def rollback(self):
+        if self._connection:
+            self._connection.rollback()
+
+    def get_connection(self):
+        """Return the active connection (always the same one)."""
+        if self._connection is None or getattr(self._connection, "closed", 1):
+            self._initialize_connection()
+        return self._connection
+
+    def close(self):
+        """Close connection gracefully when the spider ends."""
+        if self._connection and not self._connection.closed:
+            self._connection.close()
+
+
+# Backwards compatibility: older code imports `DatabaseConnection`
+# Export an alias so both names work.
+DatabaseConnection = DBConnection

scrapy_item_ingest/database/schema.py

@@ -0,0 +1,79 @@
+"""
+Database schema management utilities for scrapy_item_ingest.
+"""
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class SchemaManager:
+    """Database schema management"""
+
+    def __init__(self, db_connection, settings):
+        self.db = db_connection
+        self.settings = settings
+
+    def create_items_table(self):
+        """Create items table if it doesn't exist"""
+        items_table_sql = f"""
+        CREATE TABLE IF NOT EXISTS {self.settings.db_items_table} (
+            id SERIAL PRIMARY KEY,
+            job_id VARCHAR(255),
+            item JSONB,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        """
+        self.db.execute(items_table_sql)
+        logger.info(f"Items table {self.settings.db_items_table} created/verified with job_id column")
+
+    def create_requests_table(self):
+        """Create requests table if it doesn't exist"""
+        requests_table_sql = f"""
+        CREATE TABLE IF NOT EXISTS {self.settings.db_requests_table} (
+            id SERIAL PRIMARY KEY,
+            job_id VARCHAR(255),
+            url TEXT,
+            method VARCHAR(10),
+            status_code INTEGER,
+            duration FLOAT,
+            response_time FLOAT,
+            fingerprint VARCHAR(64),
+            parent_id INTEGER,
+            parent_url TEXT,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (parent_id) REFERENCES {self.settings.db_requests_table}(id)
+        )
+        """
+        self.db.execute(requests_table_sql)
+        logger.info(f"Requests table {self.settings.db_requests_table} created/verified with job_id column")
+
+    def create_logs_table(self):
+        """Create logs table if it doesn't exist"""
+        logs_table_sql = f"""
+        CREATE TABLE IF NOT EXISTS {self.settings.db_logs_table} (
+            id SERIAL PRIMARY KEY,
+            job_id VARCHAR(255),
+            level VARCHAR(50),
+            message TEXT,
+            timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        """
+        self.db.execute(logs_table_sql)
+        logger.info(f"Logs table {self.settings.db_logs_table} created/verified with job_id column")
+
+    def ensure_tables_exist(self):
+        """Create all tables if they don't exist (only if create_tables is True)"""
+        if not self.settings.create_tables:
+            logger.info("Table creation disabled. Skipping table creation.")
+            return
+
+        try:
+            self.create_items_table()
+            self.create_requests_table()
+            self.create_logs_table()
+            self.db.commit()
+            logger.info("All tables created/verified successfully")
+        except Exception as e:
+            logger.error(f"Failed to create tables: {e}")
+            self.db.rollback()
+            raise

scrapy_item_ingest/extensions/__init__.py

@@ -0,0 +1,2 @@
+"""Extension modules for scrapy_item_ingest."""
+

scrapy_item_ingest/extensions/base.py

@@ -0,0 +1,81 @@
+"""
+Base extension functionality for scrapy_item_ingest.
+"""
+import logging
+
+from scrapy_item_ingest.config.settings import Settings, validate_settings
+from ..utils.time import get_current_datetime
+from ..database.connection import DatabaseConnection
+from ..database.schema import SchemaManager
+
+logger = logging.getLogger(__name__)
+
+
+class BaseExtension:
+    """Base extension with common functionality"""
+
+    def __init__(self, settings):
+        self.settings = settings
+        validate_settings(settings)
+        # Lazy-initialized shared DB connection and schema manager
+        self._db = None
+        self._schema_manager = None
+        # Prevent repeated error spam if DB logging fails
+        self._db_logging_enabled = True
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """Create extension instance from crawler"""
+        settings = Settings(crawler.settings)
+        return cls(settings)
+
+    def get_identifier_info(self, spider):
+        """Get identifier column and value for the spider"""
+        return self.settings.get_identifier_column(), self.settings.get_identifier_value(spider)
+
+    def _ensure_db_initialized(self):
+        """Initialize DB connection and schema manager lazily."""
+        if self._db is None:
+            self._db = DatabaseConnection(self.settings.db_url)
+            if not self._db.connect():
+                raise RuntimeError("Failed to connect to database for logging")
+        if self._schema_manager is None:
+            self._schema_manager = SchemaManager(self._db, self.settings)
+
+    def _ensure_logs_table_exists(self):
+        """Create logs table if it doesn't exist (only if create_tables is True)."""
+        if not self.settings.create_tables:
+            return
+        try:
+            self._schema_manager.create_logs_table()
+            self._db.commit()
+        except Exception as e:
+            self._db.rollback()
+
+    def _log_to_database(self, spider, log_level, message):
+        """Helper method to log messages to database using shared DBConnection."""
+        if not self._db_logging_enabled:
+            return
+        try:
+            self._ensure_db_initialized()
+            self._ensure_logs_table_exists()
+
+            identifier_column, identifier_value = self.get_identifier_info(spider)
+            sql = f"""
+                INSERT INTO {self.settings.db_logs_table}
+                ({identifier_column}, level, message, timestamp)
+                VALUES (%s, %s, %s, %s)
+            """
+            self._db.execute(
+                sql,
+                (
+                    identifier_value,
+                    log_level,
+                    message,
+                    get_current_datetime(self.settings),
+                ),
+            )
+            self._db.commit()
+        except Exception as e:
+            # Disable further DB logging to avoid spamming errors
+            self._db_logging_enabled = False

scrapy_item_ingest/extensions/logging.py

@@ -0,0 +1,129 @@
+"""
+Logging extension for capturing spider errors and logs and saving them to the database.
+"""
+from __future__ import annotations
+
+import logging
+import threading
+from typing import List
+
+from scrapy import signals
+from scrapy.spiders import Spider
+from scrapy.crawler import Crawler
+
+from .base import BaseExtension
+
+logger = logging.getLogger(__name__)
+
+
+class ScrapyAndRootFilter(logging.Filter):
+    """
+    A logging filter that allows records from the 'root' logger and any logger
+    within the 'scrapy' namespace.
+    """
+    def filter(self, record: logging.LogRecord) -> bool:
+        # Allow logs from the spider itself (which might not be in 'scrapy' namespace)
+        if hasattr(record, 'spider_name') and record.name == getattr(record, 'spider_name', None):
+            return True
+        return record.name == 'root' or record.name.startswith('scrapy')
+
+
+class DatabaseLogHandler(logging.Handler):
+    """
+    Custom logging handler to save log records to the database in real-time.
+    """
+    _local = threading.local()
+
+    def __init__(self, extension: 'LoggingExtension', spider: Spider):
+        super().__init__()
+        self.extension = extension
+        self.spider = spider
+
+    def emit(self, record: logging.LogRecord):
+        if getattr(self._local, 'in_emit', False):
+            return  # Prevent recursion
+
+        # Avoid capturing logs generated by this extension's own exceptions
+        if 'extensions/logging.py' in record.pathname:
+            return
+
+        self._local.in_emit = True
+        try:
+            # Add spider name to record for the filter
+            record.spider_name = self.spider.name
+            msg = self.format(record)
+            level = record.levelname
+            # Log directly to the database in real-time
+            self.extension._log_to_database(self.spider, level, msg)
+        except Exception:
+            # Use logger directly to avoid recursion if formatting fails
+            logger.exception("Error in DatabaseLogHandler.emit")
+        finally:
+            self._local.in_emit = False
+
+
+class LoggingExtension(BaseExtension):
+    """
+    Extension for logging spider events to the database.
+    """
+
+    def __init__(self, settings):
+        super().__init__(settings)
+        crawler_settings = self.settings.crawler_settings
+        self.log_level = crawler_settings.get('LOG_LEVEL', 'INFO').upper()
+        self.log_format = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'
+        self.log_dateformat = '%Y-%m-%d %H:%M:%S'
+
+        self._db_log_handler: DatabaseLogHandler | None = None
+        self._root_logger_ref: logging.Logger | None = None
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler):
+        """Create an extension instance from crawler."""
+        ext = super().from_crawler(crawler)
+        crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened)
+        crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
+        crawler.signals.connect(ext.engine_stopped, signal=signals.engine_stopped)
+        return ext
+
+    def spider_opened(self, spider: Spider):
+        """Called when a spider is opened."""
+        handler = DatabaseLogHandler(self, spider)
+        level = getattr(logging, self.log_level, logging.INFO)
+        handler.setLevel(level)
+        formatter = logging.Formatter(fmt=self.log_format, datefmt=self.log_dateformat)
+        handler.setFormatter(formatter)
+        
+        handler.addFilter(ScrapyAndRootFilter())
+        
+        self._db_log_handler = handler
+
+        root_logger = logging.getLogger()
+        
+        if not any(isinstance(h, DatabaseLogHandler) for h in root_logger.handlers):
+            root_logger.addHandler(handler)
+            self._root_logger_ref = root_logger
+
+        identifier_column, identifier_value = self.get_identifier_info(spider)
+        message = f"{identifier_column.title()} {identifier_value} started"
+        spider.logger.info(message)
+
+    def spider_closed(self, spider: Spider, reason: str):
+        """Called when a spider is closed."""
+        identifier_column, identifier_value = self.get_identifier_info(spider)
+        message = f"{identifier_column.title()} {identifier_value} closed with reason: {reason}"
+        spider.logger.info(message)
+        self._cleanup()
+
+    def engine_stopped(self):
+        """Called when the Scrapy engine stops."""
+        self._cleanup()
+
+    def _cleanup(self):
+        """Removes the log handler."""
+        if self._db_log_handler and self._root_logger_ref:
+            self._root_logger_ref.removeHandler(self._db_log_handler)
+            self._db_log_handler.close()
+
+            self._db_log_handler = None
+            self._root_logger_ref = None

scrapy_item_ingest/pipelines/__init__.py

@@ -0,0 +1,2 @@
+"""Pipeline modules for scrapy_item_ingest."""
+

scrapy_item_ingest/pipelines/base.py

@@ -0,0 +1,48 @@
+"""
+Base pipeline functionality for scrapy_item_ingest.
+"""
+import logging
+
+from ..config.settings import Settings, validate_settings
+from ..database.connection import DatabaseConnection
+from ..database.schema import SchemaManager
+
+logger = logging.getLogger(__name__)
+
+
+class BasePipeline:
+    """Base pipeline with common functionality"""
+
+    def __init__(self, settings):
+        self.settings = settings
+        self.db = None
+        self.schema_manager = None
+        validate_settings(settings)
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """Create pipeline instance from crawler"""
+        settings = Settings(crawler.settings)
+        return cls(settings)
+
+    def open_spider(self, spider):
+        """Called when spider is opened"""
+        # Establish database connection
+        self.db = DatabaseConnection(self.settings.db_url)
+        if not self.db.connect():
+            raise Exception("Failed to connect to database")
+
+        # Initialize schema manager
+        self.schema_manager = SchemaManager(self.db, self.settings)
+
+        # Ensure tables exist
+        self.schema_manager.ensure_tables_exist()
+
+    def close_spider(self, spider):
+        """Called when spider is closed"""
+        if self.db:
+            self.db.close()
+
+    def get_identifier_info(self, spider):
+        """Get identifier column and value for the spider"""
+        return self.settings.get_identifier_column(), self.settings.get_identifier_value(spider)

scrapy_item_ingest/pipelines/items.py

@@ -0,0 +1,38 @@
+"""
+Items pipeline for storing scraped items.
+"""
+import logging
+
+from itemadapter import ItemAdapter
+from scrapy.exceptions import DropItem
+
+from .base import BasePipeline
+from ..utils.serialization import serialize_item_data
+from ..utils.time import get_current_datetime
+
+logger = logging.getLogger(__name__)
+
+
+class ItemsPipeline(BasePipeline):
+    """Pipeline for handling scraped items"""
+
+    def process_item(self, item, spider):
+        """Process and store item in database"""
+        job_id = self.settings.get_identifier_value(spider)
+
+        adapter = ItemAdapter(item)
+        item_dict = adapter.asdict()
+        created_at = get_current_datetime(self.settings)
+
+        # Store everything as JSON in the item column
+        try:
+            sql = f"INSERT INTO {self.settings.db_items_table} (job_id, item, created_at) VALUES (%s, %s, %s)"
+            json_data = serialize_item_data(item_dict)
+
+            self.db.execute(sql, (job_id, json_data, created_at))
+            self.db.commit()
+        except Exception as e:
+            self.db.rollback()
+            raise DropItem(f"DB insert error: {e}")
+
+        return item

scrapy_item_ingest/pipelines/main.py

@@ -0,0 +1,42 @@
+"""
+Main pipeline that combines items and requests functionality.
+"""
+import logging
+
+from .items import ItemsPipeline
+from .requests import RequestsPipeline
+
+logger = logging.getLogger(__name__)
+
+
+class DbInsertPipeline(ItemsPipeline, RequestsPipeline):
+    """
+    Main pipeline that combines item processing and request tracking.
+    Inherits from both ItemsPipeline and RequestsPipeline.
+    """
+
+    def __init__(self, settings):
+        # Initialize both parent classes
+        ItemsPipeline.__init__(self, settings)
+        RequestsPipeline.__init__(self, settings)
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """Create pipeline instance from crawler"""
+        # Use RequestsPipeline's from_crawler to get signal connections
+        return RequestsPipeline.from_crawler.__func__(cls, crawler)
+
+    def open_spider(self, spider):
+        """Called when spider is opened"""
+        # Use the base class implementation
+        super().open_spider(spider)
+
+    def close_spider(self, spider):
+        """Called when spider is closed"""
+        # Use the base class implementation
+        super().close_spider(spider)
+
+    def process_item(self, item, spider):
+        """Process and store item in database"""
+        # Use ItemsPipeline's process_item method
+        return ItemsPipeline.process_item(self, item, spider)

scrapy_item_ingest/pipelines/requests.py

@@ -0,0 +1,138 @@
+"""
+Requests pipeline for tracking request information.
+"""
+import logging
+
+from scrapy import signals
+
+from .base import BasePipeline
+from ..utils.fingerprint import get_request_fingerprint
+from ..utils.time import get_current_datetime
+
+logger = logging.getLogger(__name__)
+
+
+class RequestsPipeline(BasePipeline):
+    """Pipeline for handling request tracking"""
+
+    def __init__(self, settings):
+        super().__init__(settings)
+        self.request_id_map = {}  # Track fingerprint to database ID mapping
+        self.url_to_id_map = {}  # Track URL to database ID mapping
+        self.current_response_url = None  # Track current response being processed
+        self.request_start_times = {}  # Track request start times for response_time calculation
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """Create pipeline instance from crawler"""
+        pipeline = super().from_crawler(crawler)
+        # Connect to both signals to track request timing
+        crawler.signals.connect(pipeline.request_scheduled, signal=signals.request_scheduled)
+        crawler.signals.connect(pipeline.response_received, signal=signals.response_received)
+        return pipeline
+
+    def _get_parent_request_info(self, request, spider):
+        """Extract parent request information if available"""
+        parent_id = None
+        parent_url = None
+
+        # Get job_id for the current spider
+        job_id = self.settings.get_identifier_value(spider)
+
+        try:
+            # Method 1: Use current response URL as parent (most reliable)
+            if self.current_response_url and self.current_response_url != request.url:
+                parent_url = self.current_response_url
+                if parent_url in self.url_to_id_map:
+                    parent_id = self.url_to_id_map[parent_url]
+
+            # Method 2: Check request meta for referer
+            if not parent_id and hasattr(request, 'meta') and request.meta:
+                if 'referer' in request.meta:
+                    parent_url = request.meta['referer']
+
+                    # Look up in our URL mapping first (faster)
+                    if parent_url in self.url_to_id_map:
+                        parent_id = self.url_to_id_map[parent_url]
+                    else:
+                        # Look up in database
+                        try:
+                            sql = f"SELECT id FROM {self.settings.db_requests_table} WHERE url = %s AND job_id = %s ORDER BY created_at DESC LIMIT 1"
+                            result = self.db.execute(sql, (parent_url, job_id))
+                            if result:
+                                parent_id = result[0]
+                                # Cache the result
+                                self.url_to_id_map[parent_url] = parent_id
+
+                        except Exception as e:
+                            logger.warning(f"Could not look up parent ID by referer URL: {e}")
+
+        except Exception as e:
+            logger.warning(f"Could not extract parent request info: {e}")
+
+        return parent_id, parent_url
+
+    def log_request(self, request, spider, response=None):
+        """Log request to database with complete information"""
+        job_id = self.settings.get_identifier_value(spider)
+
+        fingerprint = get_request_fingerprint(request)
+        parent_id, parent_url = self._get_parent_request_info(request, spider)
+        created_at = get_current_datetime(self.settings)
+
+        # Get status code and response time if response is available
+        status_code = response.status if response else None
+        response_time = None
+        
+        if response:
+            fingerprint = get_request_fingerprint(request)
+            request_start_time = self.request_start_times.get(fingerprint)
+            if request_start_time:
+                current_time = created_at.timestamp()
+                response_time = current_time - request_start_time
+                # Clean up the start time to free memory
+                self.request_start_times.pop(fingerprint, None)
+
+        sql = f"""
+        INSERT INTO {self.settings.db_requests_table}
+        (job_id, url, method, fingerprint, parent_id, parent_url, status_code, response_time, created_at) 
+        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id
+        """
+        try:
+            result = self.db.execute(sql, (
+                job_id,
+                request.url,
+                request.method,
+                fingerprint,
+                parent_id,
+                parent_url,
+                status_code,
+                response_time,
+                created_at
+            ))
+
+            # Get the inserted record ID and store it for future parent lookups
+            if result:
+                record_id = result[0]
+                self.request_id_map[fingerprint] = record_id
+                self.url_to_id_map[request.url] = record_id  # Store URL to ID mapping
+
+                self.db.commit()
+
+        except Exception as e:
+            logger.error(f"Failed to log request: {e}")
+            self.db.rollback()
+
+    def request_scheduled(self, request, spider):
+        """Called when a request is scheduled - track start time"""
+        fingerprint = get_request_fingerprint(request)
+        current_time = get_current_datetime(self.settings).timestamp()
+        self.request_start_times[fingerprint] = current_time
+
+    def response_received(self, response, request, spider):
+        """Called when a response is received - log request with complete info"""
+
+        self.current_response_url = response.url
+
+        # Log the request with complete response information
+        self.log_request(request, spider, response)

scrapy_item_ingest/utils/__init__.py

@@ -0,0 +1,2 @@
+"""Utility modules for scrapy_item_ingest."""
+

scrapy_item_ingest/utils/fingerprint.py

@@ -0,0 +1,21 @@
+"""
+Request fingerprint utilities for generating unique request identifiers.
+"""
+import logging
+
+from scrapy.utils.request import fingerprint
+
+logger = logging.getLogger(__name__)
+
+
+def get_request_fingerprint(request):
+    """Generate a fingerprint for the request"""
+
+    fp = fingerprint(request)
+
+    if isinstance(fp, bytes):
+        fp = fp.hex()
+
+    fp = fp.replace("\\x", "")
+
+    return fp

scrapy_item_ingest/utils/serialization.py

@@ -0,0 +1,9 @@
+"""
+Serialization utilities for converting data to JSON-serializable format.
+"""
+import json
+
+
+def serialize_item_data(item_dict):
+    """Serialize item data to JSON string"""
+    return json.dumps(item_dict, ensure_ascii=False, default=str)

scrapy_item_ingest/utils/time.py

@@ -0,0 +1,19 @@
+from datetime import datetime
+
+import pytz
+
+
+def get_current_datetime(settings):
+    """
+    Returns the current datetime localized to the timezone defined by settings.get_tz().
+    Raises a TypeError if settings is None or invalid.
+    """
+    if settings is None:
+        raise TypeError("settings must not be None")
+    tzname = settings.get_tz()
+    try:
+        tz = pytz.timezone(tzname)
+    except Exception as e:
+        raise ValueError(f"invalid timezone '{tzname}'") from None
+
+    return tz.localize(datetime.now())

scrapy_item_ingest-0.2.4.dist-info/METADATA

@@ -0,0 +1,131 @@
+Metadata-Version: 2.4
+Name: scrapy_item_ingest
+Version: 0.2.4
+Summary: Scrapy extension for database ingestion with job/spider tracking
+Home-page: https://github.com/fawadss1/scrapy_item_ingest
+Author: Fawad Ali
+Author-email: fawadstar6@gmail.com
+Project-URL: Documentation, https://scrapy-item-ingest.readthedocs.io/
+Project-URL: Source, https://github.com/fawadss1/scrapy_item_ingest
+Project-URL: Tracker, https://github.com/fawadss1/scrapy_item_ingest/issues
+Keywords: scrapy,database,postgresql,web-scraping,data-pipeline
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Framework :: Scrapy
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Database
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: scrapy>=2.13.3
+Requires-Dist: psycopg2-binary>=2.9.10
+Requires-Dist: itemadapter>=0.11.0
+Requires-Dist: SQLAlchemy>=2.0.41
+Requires-Dist: pytz>=2025.2
+Provides-Extra: docs
+Requires-Dist: sphinx>=5.0.0; extra == "docs"
+Requires-Dist: sphinx_rtd_theme>=1.2.0; extra == "docs"
+Requires-Dist: myst-parser>=0.18.0; extra == "docs"
+Requires-Dist: sphinx-autodoc-typehints>=1.19.0; extra == "docs"
+Requires-Dist: sphinx-copybutton>=0.5.0; extra == "docs"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: black>=22.0.0; extra == "dev"
+Requires-Dist: flake8>=5.0.0; extra == "dev"
+Requires-Dist: mypy>=0.991; extra == "dev"
+Requires-Dist: pre-commit>=2.20.0; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == "test"
+Requires-Dist: pytest-cov>=4.0.0; extra == "test"
+Requires-Dist: pytest-mock>=3.8.0; extra == "test"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Scrapy Item Ingest
+
+A tiny, straightforward addon for Scrapy that saves your items, requests, and logs to PostgreSQL. No boilerplate, no ceremony.
+
+## Install
+
+```bash
+pip install scrapy-item-ingest
+```
+
+## Minimal setup (settings.py)
+
+```python
+ITEM_PIPELINES = {
+    'scrapy_item_ingest.DbInsertPipeline': 300,
+}
+
+EXTENSIONS = {
+    'scrapy_item_ingest.LoggingExtension': 500,
+}
+
+# Pick ONE of the two database config styles:
+DB_URL = "postgresql://user:password@localhost:5432/database"
+# Or use discrete fields (avoids URL encoding):
+# DB_HOST = "localhost"
+# DB_PORT = 5432
+# DB_USER = "user"
+# DB_PASSWORD = "password"
+# DB_NAME = "database"
+
+# Optional
+CREATE_TABLES = True     # auto‑create tables on first run (default True)
+JOB_ID = 1               # or omit; spider name will be used
+```
+
+Run your spider:
+
+```bash
+scrapy crawl your_spider
+```
+
+## Troubleshooting
+
+- Password has special characters like `@` or `$`?
+  - In a URL, encode them: `@` -> `%40`, `$` -> `%24`.
+  - Example: `postgresql://user:PAK%40swat1%24@localhost:5432/db`
+  - Or use the discrete fields (no encoding needed).
+
+## Useful settings (optional)
+
+- `LOG_DB_LEVEL` (default: `DEBUG`) — minimum level stored in DB
+- `LOG_DB_CAPTURE_LEVEL` — capture level for Scrapy loggers routed to DB (does not affect console)
+- `LOG_DB_LOGGERS` — allowed logger prefixes (defaults always include `[spider.name, 'scrapy']`)
+- `LOG_DB_EXCLUDE_LOGGERS` (default: `['scrapy.core.scraper']`)
+- `LOG_DB_EXCLUDE_PATTERNS` (default: `['Scraped from <']`)
+- `CREATE_TABLES` (default: `True`) — create `job_items`, `job_requests`, `job_logs` on startup
+- `ITEMS_TABLE`, `REQUESTS_TABLE`, `LOGS_TABLE` — override table names
+
+## Links
+
+- Docs: https://scrapy-item-ingest.readthedocs.io/
+- Changelog: docs/development/changelog.rst
+- Issues: https://github.com/fawadss1/scrapy_item_ingest/issues
+
+## License
+
+MIT License. See [LICENSE](LICENSE).

scrapy_item_ingest-0.2.4.dist-info/RECORD

@@ -0,0 +1,24 @@
+scrapy_item_ingest/__init__.py,sha256=FofylFBUxWl6Xt5n14icxmbxteXOGpUc3PC1cirsnrU,1507
+scrapy_item_ingest/config/__init__.py,sha256=Foyt52_KDRIoDZtSH5ttcWxQXCOUgzebo4IGCPQwriY,55
+scrapy_item_ingest/config/settings.py,sha256=JrbRkF1_ZhrzrhQ0kj_wzoT7ksls5FtpbnaBh9CAKv4,5060
+scrapy_item_ingest/database/__init__.py,sha256=-D9cfI8Hrap74UkIUmcOZ-ikAZ8HKSswZAZMBtjq69A,50
+scrapy_item_ingest/database/connection.py,sha256=B4SGBz3zfh-GzpU-k-EiQeY1x1Rw9inkSDVNXYt6T88,5948
+scrapy_item_ingest/database/schema.py,sha256=2HcBbW3VIWva59YCxyAinwZQDidFuyU5zuOCdCwBZUI,2866
+scrapy_item_ingest/extensions/__init__.py,sha256=G8xe0Bssf4jFvi3D1gNyOpylaDHlo-RKHEX9_tIB2f8,51
+scrapy_item_ingest/extensions/base.py,sha256=OWo44rGYOq3L_c-aZL48We_HJDHMjykQSMyaTFPQXgg,3001
+scrapy_item_ingest/extensions/logging.py,sha256=F3tDA-PAx2E9oXcj1VFNvuC_xoavaPq2T85Rlw4B-vc,4850
+scrapy_item_ingest/pipelines/__init__.py,sha256=NvbUeLCwjFPvVaSTzdnN6LkToJ1ISAM91EmVero9FXo,50
+scrapy_item_ingest/pipelines/base.py,sha256=wTB-VTVOA35TPkPInPeLNMfy-2f7Ab3XM-VCOC964tQ,1521
+scrapy_item_ingest/pipelines/items.py,sha256=-RVZ6PE0Zq5jplImvRsZUyu52ktNIZrqksreLCIxjs0,1180
+scrapy_item_ingest/pipelines/main.py,sha256=tc1y4R8Roc9c3LU49Gfw9LrJ5SaZ2dPBSsMDb0ZlgWQ,1385
+scrapy_item_ingest/pipelines/requests.py,sha256=Sw3b4ZP2DbRxz3l-Cek2chR37EhwC4b02dSZ47UpfjM,5823
+scrapy_item_ingest/utils/__init__.py,sha256=xuzfL-u3NkFElIrBygQISYv0CKMdSVvreuL16JXZMRM,49
+scrapy_item_ingest/utils/fingerprint.py,sha256=jOu2XAxG2WABrk9S6itrTvNqvQwxcjYT1omsVaE0Eyo,421
+scrapy_item_ingest/utils/serialization.py,sha256=GjKEvAQV4oQUXP2hudreCIIBpH6GniQ3MvfFpODxHfk,251
+scrapy_item_ingest/utils/time.py,sha256=YPtfwct4cFxhnhb-o1d9ZB_GI8DimPYsxTVQItdZ_Ao,547
+scrapy_item_ingest-0.2.4.dist-info/licenses/LICENSE,sha256=DhJQ4_j45c_DWghISLKmJshcLvX_Pr7QXaahe2iRMNo,1087
+scrapy_item_ingest-0.2.4.dist-info/METADATA,sha256=orMA19rYt3RtmPcDFntx6TsShzD1uYSx-CO7UPSNgn8,4534
+scrapy_item_ingest-0.2.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+scrapy_item_ingest-0.2.4.dist-info/entry_points.txt,sha256=WKFpo9Dy0qX1S1PT8NvIHqZmSxBCgyAM480LnLR8S1E,172
+scrapy_item_ingest-0.2.4.dist-info/top_level.txt,sha256=bu2ekFWcSH0ANdc8oGDdmZXaSC6kNuhtC-AggLsUQCU,19
+scrapy_item_ingest-0.2.4.dist-info/RECORD,,

scrapy_item_ingest-0.2.4.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

scrapy_item_ingest-0.2.4.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[scrapy.extensions]
+logging_ext = scrapy_item_ingest.extensions.logging:LoggingExtension
+
+[scrapy.pipelines]
+db_ingest = scrapy_item_ingest.pipelines.main:DbInsertPipeline

scrapy_item_ingest-0.2.4.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Fawad Ali
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scrapy_item_ingest-0.2.4.dist-info/top_level.txt

@@ -0,0 +1 @@
+scrapy_item_ingest