scrapy-item-ingest 0.1.0__tar.gz → 0.1.2__tar.gz

{scrapy_item_ingest-0.1.0 → scrapy_item_ingest-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapy_item_ingest
-Version: 0.1.0
+Version: 0.1.2
 Summary: Scrapy extension for database ingestion with job/spider tracking
 Home-page: https://github.com/fawadss1/scrapy_item_ingest
 Author: Fawad Ali
@@ -122,7 +122,7 @@ For support and questions:
 
 ## Changelog
 
-### v0.1.0 (Current)
+### v0.1.2 (Current)
 
 - Initial release
 - Core pipeline functionality for items, requests, and logs

{scrapy_item_ingest-0.1.0 → scrapy_item_ingest-0.1.2}/README.md

@@ -59,7 +59,7 @@ For support and questions:
 
 ## Changelog
 
-### v0.1.0 (Current)
+### v0.1.2 (Current)
 
 - Initial release
 - Core pipeline functionality for items, requests, and logs

{scrapy_item_ingest-0.1.0 → scrapy_item_ingest-0.1.2}/scrapy_item_ingest/__init__.py

@@ -12,7 +12,7 @@ Main Components:
 - RequestsPipeline: Standalone requests tracking pipeline
 """
 
-__version__ = "0.1.0"
+__version__ = "0.1.2"
 __author__ = "Fawad Ali"
 __description__ = "Scrapy extension for database ingestion with job/spider tracking"
 

{scrapy_item_ingest-0.1.0 → scrapy_item_ingest-0.1.2}/scrapy_item_ingest/config/settings.py

@@ -54,12 +54,10 @@ class Settings:
         # Always return JOB_ID or fallback to None (spider name will be used)
         return self.crawler_settings.get('JOB_ID', None)
 
-    def get_identifier_column(self):
-        """Get the identifier column name based on mode"""
-        if self.create_tables:
-            return "spider"  # Use spider column when creating tables
-        else:
-            return "job_id"  # Use job_id column when using existing tables
+    @staticmethod
+    def get_identifier_column():
+        """Get the identifier column name"""
+        return "job_id"
 
     def get_identifier_value(self, spider):
         """Get the identifier value with smart fallback"""

scrapy_item_ingest-0.1.2/scrapy_item_ingest/extensions/logging.py

@@ -0,0 +1,102 @@
+"""
+Logging extension for tracking spider events.
+"""
+import logging
+import threading
+from scrapy import signals
+from .base import BaseExtension
+
+logger = logging.getLogger(__name__)
+
+
+class DatabaseLogHandler(logging.Handler):
+    """Custom logging handler to save all log records to the database in batches."""
+    _local = threading.local()
+    BATCH_SIZE = 100
+
+    def __init__(self, extension, spider):
+        super().__init__()
+        self.extension = extension
+        self.spider = spider
+        self._buffer = []
+
+    def emit(self, record):
+        if getattr(self._local, 'in_emit', False):
+            return  # Prevent recursion
+        self._local.in_emit = True
+        try:
+            # Format the log message
+            msg = self.format(record)
+            level = record.levelname
+            self._buffer.append((self.spider, level, msg))
+            if len(self._buffer) >= self.BATCH_SIZE:
+                self.flush()
+        except Exception:
+            # Avoid infinite recursion if logging fails
+            pass
+        finally:
+            self._local.in_emit = False
+
+    def flush(self):
+        if not self._buffer:
+            return
+        try:
+            for spider, level, msg in self._buffer:
+                self.extension._log_to_database(spider, level, msg)
+        except Exception:
+            pass
+        finally:
+            self._buffer.clear()
+
+
+class LoggingExtension(BaseExtension):
+    """Extension for logging spider events to database"""
+
+    def __init__(self, settings):
+        super().__init__(settings)
+        self._db_log_handler = None
+        self._spider = None
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """Create extension instance from crawler"""
+        ext = super().from_crawler(crawler)
+        # Connect to spider signals
+        crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened)
+        crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
+        crawler.signals.connect(ext.spider_error, signal=signals.spider_error)
+        crawler.signals.connect(ext.item_dropped, signal=signals.item_dropped)
+        return ext
+
+    def spider_opened(self, spider):
+        """Called when spider is opened"""
+        identifier_column, identifier_value = self.get_identifier_info(spider)
+        message = f"{identifier_column.title()} {identifier_value} started"
+        self._log_to_database(spider, "INFO", message)
+        # Attach custom DB log handler to root logger
+        self._spider = spider
+        self._db_log_handler = DatabaseLogHandler(self, spider)
+        self._db_log_handler.setLevel(logging.DEBUG)
+        logging.getLogger().addHandler(self._db_log_handler)
+
+    def spider_closed(self, spider, reason):
+        """Called when spider is closed"""
+        identifier_column, identifier_value = self.get_identifier_info(spider)
+        message = f"{identifier_column.title()} {identifier_value} closed with reason: {reason}"
+        self._log_to_database(spider, "INFO", message)
+        # Remove the DB log handler
+        if self._db_log_handler:
+            self._db_log_handler.flush()  # Flush any remaining logs
+            logging.getLogger().removeHandler(self._db_log_handler)
+            self._db_log_handler = None
+            self._spider = None
+
+    def spider_error(self, failure, response, spider):
+        """Called when spider encounters an error"""
+        message = f"Spider error: {str(failure.value)} on {response.url if response else 'unknown URL'}"
+        self._log_to_database(spider, "ERROR", message)
+
+    def item_dropped(self, item, response, spider, exception):
+        """Called when an item is dropped"""
+        message = f"Item dropped: {str(exception)} from {response.url if response else 'unknown URL'}"
+        self._log_to_database(spider, "INFO", message)

{scrapy_item_ingest-0.1.0 → scrapy_item_ingest-0.1.2}/scrapy_item_ingest.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapy_item_ingest
-Version: 0.1.0
+Version: 0.1.2
 Summary: Scrapy extension for database ingestion with job/spider tracking
 Home-page: https://github.com/fawadss1/scrapy_item_ingest
 Author: Fawad Ali
@@ -122,7 +122,7 @@ For support and questions:
 
 ## Changelog
 
-### v0.1.0 (Current)
+### v0.1.2 (Current)
 
 - Initial release
 - Core pipeline functionality for items, requests, and logs

{scrapy_item_ingest-0.1.0 → scrapy_item_ingest-0.1.2}/setup.py

@@ -2,14 +2,14 @@ from setuptools import setup, find_packages
 
 # Read the README file for long description
 try:
-    with open("README.md", "r", encoding="utf-8") as fh:
+    with open("README.md", encoding="utf-8") as fh:
         long_description = fh.read()
 except FileNotFoundError:
     long_description = "A comprehensive Scrapy extension for ingesting scraped items, requests, and logs into PostgreSQL databases."
 
 setup(
     name="scrapy_item_ingest",
-    version="0.1.0",
+    version="0.1.2",
     description="Scrapy extension for database ingestion with job/spider tracking",
     long_description=long_description,
     long_description_content_type="text/markdown",

scrapy_item_ingest-0.1.0/scrapy_item_ingest/extensions/logging.py

@@ -1,45 +0,0 @@
-"""
-Logging extension for tracking spider events.
-"""
-import logging
-from scrapy import signals
-from .base import BaseExtension
-
-logger = logging.getLogger(__name__)
-
-
-class LoggingExtension(BaseExtension):
-    """Extension for logging spider events to database"""
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        """Create extension instance from crawler"""
-        ext = super().from_crawler(crawler)
-        # Connect to spider signals
-        crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened)
-        crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
-        crawler.signals.connect(ext.spider_error, signal=signals.spider_error)
-        crawler.signals.connect(ext.item_dropped, signal=signals.item_dropped)
-        return ext
-
-    def spider_opened(self, spider):
-        """Called when spider is opened"""
-        identifier_column, identifier_value = self.get_identifier_info(spider)
-        message = f"{identifier_column.title()} {identifier_value} started"
-        self._log_to_database(spider, "SPIDER_OPENED", message)
-
-    def spider_closed(self, spider, reason):
-        """Called when spider is closed"""
-        identifier_column, identifier_value = self.get_identifier_info(spider)
-        message = f"{identifier_column.title()} {identifier_value} closed with reason: {reason}"
-        self._log_to_database(spider, "SPIDER_CLOSED", message)
-
-    def spider_error(self, failure, response, spider):
-        """Called when spider encounters an error"""
-        message = f"Spider error: {str(failure.value)} on {response.url if response else 'unknown URL'}"
-        self._log_to_database(spider, "SPIDER_ERROR", message)
-
-    def item_dropped(self, item, response, spider, exception):
-        """Called when an item is dropped"""
-        message = f"Item dropped: {str(exception)} from {response.url if response else 'unknown URL'}"
-        self._log_to_database(spider, "ITEM_DROPPED", message)