scrapy-item-ingest 0.1.0__py3-none-any.whl

scrapy_item_ingest/__init__.py

@@ -0,0 +1,48 @@
+"""
+scrapy_item_ingest - A Scrapy extension for ingesting items and requests into databases.
+
+This package provides pipelines and extensions for storing scraped data, tracking requests,
+and logging spider events to PostgreSQL databases with support for both spider-based and
+job-based identification.
+
+Main Components:
+- DbInsertPipeline: Combined pipeline for items and requests
+- LoggingExtension: Extension for logging spider events
+- ItemsPipeline: Standalone items processing pipeline
+- RequestsPipeline: Standalone requests tracking pipeline
+"""
+
+__version__ = "0.1.0"
+__author__ = "Fawad Ali"
+__description__ = "Scrapy extension for database ingestion with job/spider tracking"
+
+# Import main classes directly from organized modules
+from .pipelines.main import DbInsertPipeline
+from .extensions.logging import LoggingExtension
+
+# Import individual components for advanced users
+from .pipelines.items import ItemsPipeline
+from .pipelines.requests import RequestsPipeline
+
+# Import configuration utilities
+from .config.settings import Settings, validate_settings
+
+# Define what gets imported with "from scrapy_item_ingest import *"
+__all__ = [
+    # Main classes (most commonly used)
+    'DbInsertPipeline',
+    'LoggingExtension',
+
+    # Individual components
+    'ItemsPipeline',
+    'RequestsPipeline',
+
+    # Configuration
+    'Settings',
+    'validate_settings',
+
+    # Package metadata
+    '__version__',
+    '__author__',
+    '__description__',
+]

scrapy_item_ingest/config/__init__.py

@@ -0,0 +1,2 @@
+"""Configuration modules for scrapy_item_ingest."""
+

scrapy_item_ingest/config/settings.py

@@ -0,0 +1,82 @@
+"""
+Configuration settings and utilities for scrapy_item_ingest.
+"""
+
+# Static table names - no longer configurable
+DEFAULT_ITEMS_TABLE = 'job_items'
+DEFAULT_REQUESTS_TABLE = 'job_requests'
+DEFAULT_LOGS_TABLE = 'job_logs'
+
+
+class Settings:
+    """Settings class to handle configuration options"""
+
+    def __init__(self, crawler_settings):
+        self.crawler_settings = crawler_settings
+
+    @property
+    def db_url(self):
+        return self.crawler_settings.get('DB_URL')
+
+    @property
+    def db_type(self):
+        return self.crawler_settings.get('DB_TYPE', 'postgres')
+
+    @property
+    def db_items_table(self):
+        """Return static table name for items"""
+        return DEFAULT_ITEMS_TABLE
+
+    @property
+    def db_requests_table(self):
+        """Return static table name for requests"""
+        return DEFAULT_REQUESTS_TABLE
+
+    @property
+    def db_logs_table(self):
+        """Return static table name for logs"""
+        return DEFAULT_LOGS_TABLE
+
+    @property
+    def create_tables(self):
+        return self.crawler_settings.getbool('CREATE_TABLES', True)
+
+    @property
+    def use_job_id(self):
+        # JOB_ID only works when CREATE_TABLES = False
+        if self.create_tables:
+            return False  # Don't use JOB_ID when creating tables
+        else:
+            return True   # Use JOB_ID when using existing tables
+
+    @property
+    def job_id(self):
+        # Always return JOB_ID or fallback to None (spider name will be used)
+        return self.crawler_settings.get('JOB_ID', None)
+
+    def get_identifier_column(self):
+        """Get the identifier column name based on mode"""
+        if self.create_tables:
+            return "spider"  # Use spider column when creating tables
+        else:
+            return "job_id"  # Use job_id column when using existing tables
+
+    def get_identifier_value(self, spider):
+        """Get the identifier value with smart fallback"""
+        job_id = self.crawler_settings.get('JOB_ID', None)
+
+        if self.create_tables:
+            # When creating tables, use JOB_ID if provided, else spider name
+            return job_id if job_id else spider.name
+        else:
+            # When using existing tables, use JOB_ID if provided, else spider name
+            return job_id if job_id else spider.name
+
+
+def validate_settings(settings):
+    """Validate configuration settings"""
+    if not settings.db_url:
+        raise ValueError("DB_URL must be set in settings")
+
+    # Job ID is now optional - will use spider name as fallback
+    return True

scrapy_item_ingest/database/__init__.py

@@ -0,0 +1,2 @@
+"""Database modules for scrapy_item_ingest."""
+

scrapy_item_ingest/database/connection.py

@@ -0,0 +1,67 @@
+"""
+Database connection utilities for scrapy_item_ingest.
+"""
+import psycopg2
+import logging
+from urllib.parse import urlparse, unquote
+
+logger = logging.getLogger(__name__)
+
+
+class DatabaseConnection:
+    """Database connection manager"""
+
+    def __init__(self, db_url):
+        self.db_url = db_url
+        self.conn = None
+        self.cur = None
+
+    def connect(self):
+        """Establish database connection"""
+        try:
+            result = urlparse(self.db_url)
+            user = result.username
+            password = unquote(result.password) if result.password else None
+            host = result.hostname
+            port = result.port
+            dbname = result.path.lstrip('/')
+
+            self.conn = psycopg2.connect(
+                host=host, port=port, dbname=dbname,
+                user=user, password=password
+            )
+            self.cur = self.conn.cursor()
+            logger.info("Database connection established")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to connect to database: {e}")
+            return False
+
+    def close(self):
+        """Close database connection"""
+        if hasattr(self, 'cur') and self.cur:
+            self.cur.close()
+        if hasattr(self, 'conn') and self.conn:
+            self.conn.close()
+        logger.info("Database connection closed")
+
+    def execute(self, sql, params=None):
+        """Execute SQL query"""
+        try:
+            if params:
+                self.cur.execute(sql, params)
+            else:
+                self.cur.execute(sql)
+            return self.cur.fetchone() if self.cur.description else None
+        except Exception as e:
+            logger.error(f"Failed to execute query: {e}")
+            self.conn.rollback()
+            raise
+
+    def commit(self):
+        """Commit transaction"""
+        self.conn.commit()
+
+    def rollback(self):
+        """Rollback transaction"""
+        self.conn.rollback()

scrapy_item_ingest/database/schema.py

@@ -0,0 +1,79 @@
+"""
+Database schema management utilities for scrapy_item_ingest.
+"""
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class SchemaManager:
+    """Database schema management"""
+
+    def __init__(self, db_connection, settings):
+        self.db = db_connection
+        self.settings = settings
+
+    def create_items_table(self):
+        """Create items table if it doesn't exist"""
+        items_table_sql = f"""
+        CREATE TABLE IF NOT EXISTS {self.settings.db_items_table} (
+            id SERIAL PRIMARY KEY,
+            job_id VARCHAR(255),
+            item JSONB,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        """
+        self.db.execute(items_table_sql)
+        logger.info(f"Items table {self.settings.db_items_table} created/verified with job_id column")
+
+    def create_requests_table(self):
+        """Create requests table if it doesn't exist"""
+        requests_table_sql = f"""
+        CREATE TABLE IF NOT EXISTS {self.settings.db_requests_table} (
+            id SERIAL PRIMARY KEY,
+            job_id VARCHAR(255),
+            url TEXT,
+            method VARCHAR(10),
+            status_code INTEGER,
+            duration FLOAT,
+            response_time FLOAT,
+            fingerprint VARCHAR(64),
+            parent_id INTEGER,
+            parent_url TEXT,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (parent_id) REFERENCES {self.settings.db_requests_table}(id)
+        )
+        """
+        self.db.execute(requests_table_sql)
+        logger.info(f"Requests table {self.settings.db_requests_table} created/verified with job_id column")
+
+    def create_logs_table(self):
+        """Create logs table if it doesn't exist"""
+        logs_table_sql = f"""
+        CREATE TABLE IF NOT EXISTS {self.settings.db_logs_table} (
+            id SERIAL PRIMARY KEY,
+            job_id VARCHAR(255),
+            level VARCHAR(50),
+            message TEXT,
+            timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        """
+        self.db.execute(logs_table_sql)
+        logger.info(f"Logs table {self.settings.db_logs_table} created/verified with job_id column")
+
+    def ensure_tables_exist(self):
+        """Create all tables if they don't exist (only if create_tables is True)"""
+        if not self.settings.create_tables:
+            logger.info("Table creation disabled. Skipping table creation.")
+            return
+
+        try:
+            self.create_items_table()
+            self.create_requests_table()
+            self.create_logs_table()
+            self.db.commit()
+            logger.info("All tables created/verified successfully")
+        except Exception as e:
+            logger.error(f"Failed to create tables: {e}")
+            self.db.rollback()
+            raise

scrapy_item_ingest/extensions/__init__.py

@@ -0,0 +1,2 @@
+"""Extension modules for scrapy_item_ingest."""
+

scrapy_item_ingest/extensions/base.py

@@ -0,0 +1,79 @@
+"""
+Base extension functionality for scrapy_item_ingest.
+"""
+import logging
+from datetime import datetime
+from sqlalchemy import create_engine, text
+from ..config.settings import Settings, validate_settings
+
+logger = logging.getLogger(__name__)
+
+
+class BaseExtension:
+    """Base extension with common functionality"""
+
+    def __init__(self, settings):
+        self.settings = settings
+        validate_settings(settings)
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """Create extension instance from crawler"""
+        settings = Settings(crawler.settings)
+        return cls(settings)
+
+    def get_identifier_info(self, spider):
+        """Get identifier column and value for the spider"""
+        return self.settings.get_identifier_column(), self.settings.get_identifier_value(spider)
+
+    def _ensure_logs_table_exists(self, engine):
+        """Create logs table if it doesn't exist (only if create_tables is True)"""
+        if not self.settings.create_tables:
+            logger.info("Table creation disabled. Skipping logs table creation.")
+            return
+
+        try:
+            with engine.connect() as connection:
+                # Determine the identifier column name
+                identifier_column = self.settings.get_identifier_column()
+
+                # Create logs table with type, message, and timestamp
+                logs_table_sql = f"""
+                CREATE TABLE IF NOT EXISTS {self.settings.db_logs_table} (
+                    id SERIAL PRIMARY KEY,
+                    {identifier_column} VARCHAR(255),
+                    type VARCHAR(50),
+                    message TEXT,
+                    timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+                """
+                connection.execute(text(logs_table_sql))
+                connection.commit()
+                logger.info(f"Logs table {self.settings.db_logs_table} created/verified with {identifier_column} column")
+        except Exception as e:
+            logger.error(f"Failed to create logs table: {e}")
+
+    def _log_to_database(self, spider, log_level, message):
+        """Helper method to log messages to database"""
+        try:
+            identifier_column, identifier_value = self.get_identifier_info(spider)
+
+            engine = create_engine(self.settings.db_url)
+            self._ensure_logs_table_exists(engine)
+
+            stmt = text(f"""
+                INSERT INTO {self.settings.db_logs_table}
+                ({identifier_column}, level, message, timestamp) 
+                VALUES (:identifier, :type, :message, :timestamp)
+            """)
+            with engine.connect() as connection:
+                connection.execute(stmt, {
+                    "identifier": identifier_value,
+                    "type": log_level,
+                    "message": message,
+                    "timestamp": datetime.now()
+                })
+                connection.commit()
+                logger.info(f"Logged {log_level} for {identifier_column} {identifier_value}")
+        except Exception as e:
+            logger.error(f"Failed to log {log_level}: {e}")

scrapy_item_ingest/extensions/logging.py

@@ -0,0 +1,45 @@
+"""
+Logging extension for tracking spider events.
+"""
+import logging
+from scrapy import signals
+from .base import BaseExtension
+
+logger = logging.getLogger(__name__)
+
+
+class LoggingExtension(BaseExtension):
+    """Extension for logging spider events to database"""
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """Create extension instance from crawler"""
+        ext = super().from_crawler(crawler)
+        # Connect to spider signals
+        crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened)
+        crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
+        crawler.signals.connect(ext.spider_error, signal=signals.spider_error)
+        crawler.signals.connect(ext.item_dropped, signal=signals.item_dropped)
+        return ext
+
+    def spider_opened(self, spider):
+        """Called when spider is opened"""
+        identifier_column, identifier_value = self.get_identifier_info(spider)
+        message = f"{identifier_column.title()} {identifier_value} started"
+        self._log_to_database(spider, "SPIDER_OPENED", message)
+
+    def spider_closed(self, spider, reason):
+        """Called when spider is closed"""
+        identifier_column, identifier_value = self.get_identifier_info(spider)
+        message = f"{identifier_column.title()} {identifier_value} closed with reason: {reason}"
+        self._log_to_database(spider, "SPIDER_CLOSED", message)
+
+    def spider_error(self, failure, response, spider):
+        """Called when spider encounters an error"""
+        message = f"Spider error: {str(failure.value)} on {response.url if response else 'unknown URL'}"
+        self._log_to_database(spider, "SPIDER_ERROR", message)
+
+    def item_dropped(self, item, response, spider, exception):
+        """Called when an item is dropped"""
+        message = f"Item dropped: {str(exception)} from {response.url if response else 'unknown URL'}"
+        self._log_to_database(spider, "ITEM_DROPPED", message)

scrapy_item_ingest/pipelines/__init__.py

@@ -0,0 +1,2 @@
+"""Pipeline modules for scrapy_item_ingest."""
+

scrapy_item_ingest/pipelines/base.py

@@ -0,0 +1,50 @@
+"""
+Base pipeline functionality for scrapy_item_ingest.
+"""
+import logging
+from ..config.settings import Settings, validate_settings
+from ..database.connection import DatabaseConnection
+from ..database.schema import SchemaManager
+
+logger = logging.getLogger(__name__)
+
+
+class BasePipeline:
+    """Base pipeline with common functionality"""
+
+    def __init__(self, settings):
+        self.settings = settings
+        self.db = None
+        self.schema_manager = None
+        validate_settings(settings)
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """Create pipeline instance from crawler"""
+        settings = Settings(crawler.settings)
+        return cls(settings)
+
+    def open_spider(self, spider):
+        """Called when spider is opened"""
+        # Establish database connection
+        self.db = DatabaseConnection(self.settings.db_url)
+        if not self.db.connect():
+            raise Exception("Failed to connect to database")
+
+        # Initialize schema manager
+        self.schema_manager = SchemaManager(self.db, self.settings)
+
+        # Ensure tables exist
+        self.schema_manager.ensure_tables_exist()
+
+        logger.info(f"Pipeline opened for {self.settings.get_identifier_column()}: {self.settings.get_identifier_value(spider)}")
+
+    def close_spider(self, spider):
+        """Called when spider is closed"""
+        if self.db:
+            self.db.close()
+        logger.info(f"Pipeline closed for {self.settings.get_identifier_column()}: {self.settings.get_identifier_value(spider)}")
+
+    def get_identifier_info(self, spider):
+        """Get identifier column and value for the spider"""
+        return self.settings.get_identifier_column(), self.settings.get_identifier_value(spider)

scrapy_item_ingest/pipelines/items.py

@@ -0,0 +1,42 @@
+"""
+Items pipeline for storing scraped items.
+"""
+import logging
+from datetime import datetime, timezone
+from itemadapter import ItemAdapter
+from scrapy.exceptions import DropItem
+from .base import BasePipeline
+from ..utils.serialization import serialize_item_data
+
+logger = logging.getLogger(__name__)
+
+
+class ItemsPipeline(BasePipeline):
+    """Pipeline for handling scraped items"""
+
+    def process_item(self, item, spider):
+        """Process and store item in database"""
+        job_id = self.settings.get_identifier_value(spider)
+
+        logger.info(f"Processing item for job_id {job_id}: {item}")
+        adapter = ItemAdapter(item)
+        item_dict = adapter.asdict()
+        created_at = datetime.now(timezone.utc)
+
+        logger.info(f"Item dict prepared: {item_dict}")
+
+        # Store everything as JSON in the item column
+        try:
+            sql = f"INSERT INTO {self.settings.db_items_table} (job_id, item, created_at) VALUES (%s, %s, %s)"
+            json_data = serialize_item_data(item_dict)
+            logger.info(f"Executing SQL: {sql} with JSON data")
+
+            self.db.execute(sql, (job_id, json_data, created_at))
+            self.db.commit()
+            logger.info(f"Successfully inserted item for job_id {job_id}")
+        except Exception as e:
+            logger.error(f"Failed to insert item: {e}")
+            self.db.rollback()
+            raise DropItem(f"DB insert error: {e}")
+
+        return item

scrapy_item_ingest/pipelines/main.py

@@ -0,0 +1,41 @@
+"""
+Main pipeline that combines items and requests functionality.
+"""
+import logging
+from .items import ItemsPipeline
+from .requests import RequestsPipeline
+
+logger = logging.getLogger(__name__)
+
+
+class DbInsertPipeline(ItemsPipeline, RequestsPipeline):
+    """
+    Main pipeline that combines item processing and request tracking.
+    Inherits from both ItemsPipeline and RequestsPipeline.
+    """
+
+    def __init__(self, settings):
+        # Initialize both parent classes
+        ItemsPipeline.__init__(self, settings)
+        RequestsPipeline.__init__(self, settings)
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """Create pipeline instance from crawler"""
+        # Use RequestsPipeline's from_crawler to get signal connections
+        return RequestsPipeline.from_crawler.__func__(cls, crawler)
+
+    def open_spider(self, spider):
+        """Called when spider is opened"""
+        # Use the base class implementation
+        super().open_spider(spider)
+
+    def close_spider(self, spider):
+        """Called when spider is closed"""
+        # Use the base class implementation
+        super().close_spider(spider)
+
+    def process_item(self, item, spider):
+        """Process and store item in database"""
+        # Use ItemsPipeline's process_item method
+        return ItemsPipeline.process_item(self, item, spider)

scrapy_item_ingest/pipelines/requests.py

@@ -0,0 +1,169 @@
+"""
+Requests pipeline for tracking request information.
+"""
+import logging
+import time
+from datetime import datetime, timezone
+from scrapy import signals
+from .base import BasePipeline
+from ..utils.fingerprint import get_request_fingerprint
+
+logger = logging.getLogger(__name__)
+
+
+class RequestsPipeline(BasePipeline):
+    """Pipeline for handling request tracking"""
+
+    def __init__(self, settings):
+        super().__init__(settings)
+        self.request_start_times = {}  # Track request start times
+        self.request_id_map = {}  # Track fingerprint to database ID mapping
+        self.url_to_id_map = {}  # Track URL to database ID mapping
+        self.current_response_url = None  # Track current response being processed
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """Create pipeline instance from crawler"""
+        pipeline = super().from_crawler(crawler)
+        # Connect to request signals to automatically log requests
+        crawler.signals.connect(pipeline.request_scheduled, signal=signals.request_scheduled)
+        crawler.signals.connect(pipeline.response_received, signal=signals.response_received)
+        return pipeline
+
+    def _get_parent_request_info(self, request, spider):
+        """Extract parent request information if available"""
+        parent_id = None
+        parent_url = None
+
+        # Get job_id for the current spider
+        job_id = self.settings.get_identifier_value(spider)
+
+        try:
+            # Method 1: Use current response URL as parent (most reliable)
+            if self.current_response_url and self.current_response_url != request.url:
+                parent_url = self.current_response_url
+                if parent_url in self.url_to_id_map:
+                    parent_id = self.url_to_id_map[parent_url]
+                    logger.info(f"Found parent ID {parent_id} from current response URL: {parent_url}")
+
+            # Method 2: Check request meta for referer
+            if not parent_id and hasattr(request, 'meta') and request.meta:
+                if 'referer' in request.meta:
+                    parent_url = request.meta['referer']
+                    logger.info(f"Found referer in meta: {parent_url}")
+
+                    # Look up in our URL mapping first (faster)
+                    if parent_url in self.url_to_id_map:
+                        parent_id = self.url_to_id_map[parent_url]
+                        logger.info(f"Found parent ID {parent_id} from URL mapping")
+                    else:
+                        # Look up in database
+                        try:
+                            sql = f"SELECT id FROM {self.settings.db_requests_table} WHERE url = %s AND job_id = %s ORDER BY created_at DESC LIMIT 1"
+                            result = self.db.execute(sql, (parent_url, job_id))
+                            if result:
+                                parent_id = result[0]
+                                # Cache the result
+                                self.url_to_id_map[parent_url] = parent_id
+                                logger.info(f"Found parent ID {parent_id} from database lookup")
+                        except Exception as e:
+                            logger.warning(f"Could not look up parent ID by referer URL: {e}")
+
+                # Debug: Log request meta information
+                logger.debug(f"Request URL: {request.url}")
+                logger.debug(f"Request meta keys: {list(request.meta.keys()) if request.meta else 'None'}")
+                if 'depth' in request.meta:
+                    logger.debug(f"Request depth: {request.meta['depth']}")
+
+        except Exception as e:
+            logger.warning(f"Could not extract parent request info: {e}")
+
+        # If we still don't have parent info, log for debugging
+        if not parent_id and not parent_url:
+            logger.debug(f"No parent found for request: {request.url}")
+
+        return parent_id, parent_url
+
+    def log_request(self, request, spider):
+        """Log request to database"""
+        job_id = self.settings.get_identifier_value(spider)
+
+        logger.info(f"Logging request for job_id {job_id}: {request.url}")
+        fingerprint = get_request_fingerprint(request)
+        parent_id, parent_url = self._get_parent_request_info(request, spider)
+        request_time = time.time()
+        created_at = datetime.now(timezone.utc)
+
+        # Store request start time for duration calculation
+        self.request_start_times[fingerprint] = request_time
+
+        sql = f"""
+        INSERT INTO {self.settings.db_requests_table} 
+        (job_id, url, method, fingerprint, parent_id, parent_url, created_at) 
+        VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING id
+        """
+        try:
+            result = self.db.execute(sql, (
+                job_id,
+                request.url,
+                request.method,
+                fingerprint,
+                parent_id,
+                parent_url,
+                created_at
+            ))
+
+            # Get the inserted record ID and store it for future parent lookups
+            if result:
+                record_id = result[0]
+                self.request_id_map[fingerprint] = record_id
+                self.url_to_id_map[request.url] = record_id  # Store URL to ID mapping
+
+                self.db.commit()
+
+                log_msg = f"Successfully logged request for job_id {job_id} with fingerprint {fingerprint} (ID: {record_id})"
+                if parent_id:
+                    log_msg += f" (parent ID: {parent_id}, parent URL: {parent_url})"
+                else:
+                    log_msg += " (no parent found)"
+                logger.info(log_msg)
+        except Exception as e:
+            logger.error(f"Failed to log request: {e}")
+            self.db.rollback()
+
+    def request_scheduled(self, request, spider):
+        """Called when a request is scheduled"""
+        job_id = self.settings.get_identifier_value(spider)
+        logger.info(f"Request scheduled for job_id {job_id}: {request.url}")
+        self.log_request(request, spider)
+
+    def response_received(self, response, request, spider):
+        """Called when a response is received"""
+        job_id = self.settings.get_identifier_value(spider)
+
+        logger.info(f"Response received for job_id {job_id}: {response.url} (status: {response.status})")
+
+        # Set current response URL for parent tracking
+        self.current_response_url = response.url
+
+        fingerprint = get_request_fingerprint(request)
+        response_time = time.time()
+
+        # Update the request log with response info
+        try:
+            sql = f"""
+            UPDATE {self.settings.db_requests_table} 
+            SET status_code = %s, response_time = %s
+            WHERE job_id = %s AND fingerprint = %s AND status_code IS NULL
+            """
+            self.db.execute(sql, (
+                response.status,
+                response_time,
+                job_id,
+                fingerprint
+            ))
+            self.db.commit()
+            logger.info(f"Updated request status {response.status} and response_time for fingerprint {fingerprint}")
+        except Exception as e:
+            logger.error(f"Failed to update request status: {e}")
+            self.db.rollback()

scrapy_item_ingest/utils/__init__.py

@@ -0,0 +1,2 @@
+"""Utility modules for scrapy_item_ingest."""
+

scrapy_item_ingest/utils/fingerprint.py

@@ -0,0 +1,25 @@
+"""
+Request fingerprint utilities for generating unique request identifiers.
+"""
+import hashlib
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def get_request_fingerprint(request):
+    """Generate fingerprint for the request"""
+    try:
+        from scrapy.utils.request import request_fingerprint
+        return request_fingerprint(request)
+    except Exception as e:
+        logger.warning(f"Could not generate fingerprint: {e}")
+        # Fallback fingerprint generation
+        fingerprint_data = f"{request.method}:{request.url}"
+        return hashlib.sha1(fingerprint_data.encode()).hexdigest()
+
+
+def generate_url_fingerprint(method, url):
+    """Generate a simple fingerprint for URL and method combination"""
+    fingerprint_data = f"{method}:{url}"
+    return hashlib.sha1(fingerprint_data.encode()).hexdigest()

scrapy_item_ingest/utils/serialization.py

@@ -0,0 +1,28 @@
+"""
+Serialization utilities for converting data to JSON-serializable format.
+"""
+import json
+from datetime import datetime, date, time
+from decimal import Decimal
+
+
+def serialize_stats(obj):
+    """Recursively convert stats to JSON-serializable format"""
+    if isinstance(obj, dict):
+        return {key: serialize_stats(value) for key, value in obj.items()}
+    elif isinstance(obj, (list, tuple)):
+        return [serialize_stats(item) for item in obj]
+    elif isinstance(obj, (datetime, date, time)):
+        return obj.isoformat()
+    elif isinstance(obj, Decimal):
+        return float(obj)
+    elif isinstance(obj, (int, float, str, bool)) or obj is None:
+        return obj
+    else:
+        # For any other type, convert to string
+        return str(obj)
+
+
+def serialize_item_data(item_dict):
+    """Serialize item data to JSON string"""
+    return json.dumps(item_dict, ensure_ascii=False, default=str)

scrapy_item_ingest-0.1.0.dist-info/METADATA

@@ -0,0 +1,132 @@
+Metadata-Version: 2.4
+Name: scrapy_item_ingest
+Version: 0.1.0
+Summary: Scrapy extension for database ingestion with job/spider tracking
+Home-page: https://github.com/fawadss1/scrapy_item_ingest
+Author: Fawad Ali
+Author-email: fawadstar6@gmail.com
+Project-URL: Documentation, https://scrapy-item-ingest.readthedocs.io/
+Project-URL: Source, https://github.com/fawadss1/scrapy_item_ingest
+Project-URL: Tracker, https://github.com/fawadss1/scrapy_item_ingest/issues
+Keywords: scrapy,database,postgresql,web-scraping,data-pipeline
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Framework :: Scrapy
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Database
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: scrapy>=2.5.0
+Requires-Dist: psycopg2-binary>=2.8.0
+Requires-Dist: itemadapter>=0.6.0
+Requires-Dist: SQLAlchemy>=1.4.0
+Provides-Extra: docs
+Requires-Dist: sphinx>=5.0.0; extra == "docs"
+Requires-Dist: sphinx_rtd_theme>=1.2.0; extra == "docs"
+Requires-Dist: myst-parser>=0.18.0; extra == "docs"
+Requires-Dist: sphinx-autodoc-typehints>=1.19.0; extra == "docs"
+Requires-Dist: sphinx-copybutton>=0.5.0; extra == "docs"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: black>=22.0.0; extra == "dev"
+Requires-Dist: flake8>=5.0.0; extra == "dev"
+Requires-Dist: mypy>=0.991; extra == "dev"
+Requires-Dist: pre-commit>=2.20.0; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == "test"
+Requires-Dist: pytest-cov>=4.0.0; extra == "test"
+Requires-Dist: pytest-mock>=3.8.0; extra == "test"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Scrapy Item Ingest
+
+[![PyPI Version](https://img.shields.io/pypi/v/scrapy-item-ingest.svg)](https://pypi.org/project/scrapy-item-ingest/)
+[![PyPI Downloads](https://img.shields.io/pypi/dm/scrapy-item-ingest.svg)](https://pypi.org/project/scrapy-item-ingest/)
+[![Supported Python Versions](https://img.shields.io/pypi/pyversions/scrapy-item-ingest.svg)](https://pypi.org/project/scrapy-item-ingest/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+[![GitHub Stars](https://img.shields.io/github/stars/fawadss1/scrapy_item_ingest.svg)](https://github.com/fawadss1/scrapy_item_ingest/stargazers)
+[![GitHub Issues](https://img.shields.io/github/issues/fawadss1/scrapy_item_ingest.svg)](https://github.com/fawadss1/scrapy_item_ingest/issues)
+[![GitHub Last Commit](https://img.shields.io/github/last-commit/fawadss1/scrapy_item_ingest.svg)](https://github.com/fawadss1/scrapy_item_ingest/commits)
+
+A comprehensive Scrapy extension for ingesting scraped items, requests, and logs into PostgreSQL databases with advanced tracking capabilities. This library provides a clean, production-ready solution for storing and monitoring your Scrapy crawling operations with real-time data ingestion and comprehensive logging.
+
+## Documentation
+
+Full documentation is available at: [https://scrapy-item-ingest.readthedocs.io/en/latest/](https://scrapy-item-ingest.readthedocs.io/en/latest/)
+
+## Key Features
+
+- 🔄 **Real-time Data Ingestion**: Store items, requests, and logs as they're processed
+- 📊 **Request Tracking**: Track request response times, fingerprints, and parent-child relationships
+- 🔍 **Comprehensive Logging**: Capture spider events, errors, and custom messages
+- 🏗️ **Flexible Schema**: Support for both auto-creation and existing table modes
+- ⚙️ **Modular Design**: Use individual components or the complete pipeline
+- 🛡️ **Production Ready**: Handles both development and production scenarios
+- 📝 **JSONB Storage**: Store complex item data as JSONB for flexible querying
+- 🐳 **Docker Support**: Complete containerization with Docker and Kubernetes
+- 📈 **Performance Optimized**: Connection pooling and batch processing
+- 🔧 **Easy Configuration**: Environment-based configuration with validation
+- 📊 **Monitoring Ready**: Built-in metrics and health checks
+
+## Installation
+
+```bash
+pip install scrapy-item-ingest
+```
+
+## Development
+
+### Setting up for Development
+
+```bash
+git clone https://github.com/fawadss1/scrapy_item_ingest.git
+cd scrapy_item_ingest
+pip install -e ".[dev]"
+```
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## Support
+
+For support and questions:
+
+- **Email**: fawadstar6@gmail.com
+- **Documentation**: [https://scrapy-item-ingest.readthedocs.io/](https://scrapy-item-ingest.readthedocs.io/)
+- **Issues**: Please report bugs and feature requests at [GitHub Issues](https://github.com/fawadss1/scrapy_item_ingest/issues)
+
+## Changelog
+
+### v0.1.0 (Current)
+
+- Initial release
+- Core pipeline functionality for items, requests, and logs
+- PostgreSQL database integration with JSONB storage
+- Comprehensive documentation and examples
+- Production deployment guides
+- Docker and Kubernetes support

scrapy_item_ingest-0.1.0.dist-info/RECORD

@@ -0,0 +1,23 @@
+scrapy_item_ingest/__init__.py,sha256=2Cj2dnPpsGOTGH95ihMAdwASfT-Nhw3i4M5zfxLfgaw,1507
+scrapy_item_ingest/config/__init__.py,sha256=Foyt52_KDRIoDZtSH5ttcWxQXCOUgzebo4IGCPQwriY,55
+scrapy_item_ingest/config/settings.py,sha256=5GFQAqRf-6oc4KaMdMGuzLdtSP6fmn67anzGnKpejTI,2619
+scrapy_item_ingest/database/__init__.py,sha256=-D9cfI8Hrap74UkIUmcOZ-ikAZ8HKSswZAZMBtjq69A,50
+scrapy_item_ingest/database/connection.py,sha256=bvSTCQfgBMcuKu-VzMCwMtSNBORzeeaYKm63eRtwKz8,2027
+scrapy_item_ingest/database/schema.py,sha256=2HcBbW3VIWva59YCxyAinwZQDidFuyU5zuOCdCwBZUI,2866
+scrapy_item_ingest/extensions/__init__.py,sha256=G8xe0Bssf4jFvi3D1gNyOpylaDHlo-RKHEX9_tIB2f8,51
+scrapy_item_ingest/extensions/base.py,sha256=aodB_V47O8ihox2VdDizclAhQ6VonAUZ4lIOitub7kw,3192
+scrapy_item_ingest/extensions/logging.py,sha256=pI-MlQfNLHFmNuOSp60SfFAGRxJwPwurZp1xt88PeWU,2017
+scrapy_item_ingest/pipelines/__init__.py,sha256=NvbUeLCwjFPvVaSTzdnN6LkToJ1ISAM91EmVero9FXo,50
+scrapy_item_ingest/pipelines/base.py,sha256=C6lk37lhNr6oADXofi4aqnN0cZ9u1KrWxEZMp6Oo7oA,1783
+scrapy_item_ingest/pipelines/items.py,sha256=FcgzmuFJ3qPVVQDZv9OjEPjlSbbW5MZykMmhAcdZ4Tc,1487
+scrapy_item_ingest/pipelines/main.py,sha256=udABcEJPsJRdOFWZckP03UUUIJrHUHlfOttookdClMI,1383
+scrapy_item_ingest/pipelines/requests.py,sha256=3Wyzx6kgf7B_gg2DC0jhekCPSIEaJiRJquhdWyLrHBs,7427
+scrapy_item_ingest/utils/__init__.py,sha256=xuzfL-u3NkFElIrBygQISYv0CKMdSVvreuL16JXZMRM,49
+scrapy_item_ingest/utils/fingerprint.py,sha256=Qdby72nLNQp4-sxL51RM85MuxwFJHxmuYDbQv1c7hPc,855
+scrapy_item_ingest/utils/serialization.py,sha256=iKGhWnVwMKLKZ63kek4Hov9ESy9igA13CuOfDRD1W-M,942
+scrapy_item_ingest-0.1.0.dist-info/licenses/LICENSE,sha256=DhJQ4_j45c_DWghISLKmJshcLvX_Pr7QXaahe2iRMNo,1087
+scrapy_item_ingest-0.1.0.dist-info/METADATA,sha256=eiZF00MlFqNBdExA4kW2i4C_p_TvLsd4sLAsFE4uiD4,5868
+scrapy_item_ingest-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+scrapy_item_ingest-0.1.0.dist-info/entry_points.txt,sha256=WKFpo9Dy0qX1S1PT8NvIHqZmSxBCgyAM480LnLR8S1E,172
+scrapy_item_ingest-0.1.0.dist-info/top_level.txt,sha256=bu2ekFWcSH0ANdc8oGDdmZXaSC6kNuhtC-AggLsUQCU,19
+scrapy_item_ingest-0.1.0.dist-info/RECORD,,

scrapy_item_ingest-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

scrapy_item_ingest-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[scrapy.extensions]
+logging_ext = scrapy_item_ingest.extensions.logging:LoggingExtension
+
+[scrapy.pipelines]
+db_ingest = scrapy_item_ingest.pipelines.main:DbInsertPipeline

scrapy_item_ingest-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Fawad Ali
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scrapy_item_ingest-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+scrapy_item_ingest