scrapy-item-ingest 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

scrapy_item_ingest/__init__.py

@@ -12,7 +12,7 @@ Main Components:
 - RequestsPipeline: Standalone requests tracking pipeline
 """
 
-__version__ = "0.1.0"
+__version__ = "0.1.1"
 __author__ = "Fawad Ali"
 __description__ = "Scrapy extension for database ingestion with job/spider tracking"
 

scrapy_item_ingest/extensions/logging.py

@@ -2,15 +2,61 @@
 Logging extension for tracking spider events.
 """
 import logging
+import threading
 from scrapy import signals
 from .base import BaseExtension
 
 logger = logging.getLogger(__name__)
 
 
+class DatabaseLogHandler(logging.Handler):
+    """Custom logging handler to save all log records to the database in batches."""
+    _local = threading.local()
+    BATCH_SIZE = 100
+
+    def __init__(self, extension, spider):
+        super().__init__()
+        self.extension = extension
+        self.spider = spider
+        self._buffer = []
+
+    def emit(self, record):
+        if getattr(self._local, 'in_emit', False):
+            return  # Prevent recursion
+        self._local.in_emit = True
+        try:
+            # Format the log message
+            msg = self.format(record)
+            level = record.levelname
+            self._buffer.append((self.spider, level, msg))
+            if len(self._buffer) >= self.BATCH_SIZE:
+                self.flush()
+        except Exception:
+            # Avoid infinite recursion if logging fails
+            pass
+        finally:
+            self._local.in_emit = False
+
+    def flush(self):
+        if not self._buffer:
+            return
+        try:
+            for spider, level, msg in self._buffer:
+                self.extension._log_to_database(spider, level, msg)
+        except Exception:
+            pass
+        finally:
+            self._buffer.clear()
+
+
 class LoggingExtension(BaseExtension):
     """Extension for logging spider events to database"""
 
+    def __init__(self, settings):
+        super().__init__(settings)
+        self._db_log_handler = None
+        self._spider = None
+
     @classmethod
     def from_crawler(cls, crawler):
         """Create extension instance from crawler"""
@@ -26,20 +72,31 @@ class LoggingExtension(BaseExtension):
         """Called when spider is opened"""
         identifier_column, identifier_value = self.get_identifier_info(spider)
         message = f"{identifier_column.title()} {identifier_value} started"
-        self._log_to_database(spider, "SPIDER_OPENED", message)
+        self._log_to_database(spider, "INFO", message)
+        # Attach custom DB log handler to root logger
+        self._spider = spider
+        self._db_log_handler = DatabaseLogHandler(self, spider)
+        self._db_log_handler.setLevel(logging.DEBUG)
+        logging.getLogger().addHandler(self._db_log_handler)
 
     def spider_closed(self, spider, reason):
         """Called when spider is closed"""
         identifier_column, identifier_value = self.get_identifier_info(spider)
         message = f"{identifier_column.title()} {identifier_value} closed with reason: {reason}"
-        self._log_to_database(spider, "SPIDER_CLOSED", message)
+        self._log_to_database(spider, "INFO", message)
+        # Remove the DB log handler
+        if self._db_log_handler:
+            self._db_log_handler.flush()  # Flush any remaining logs
+            logging.getLogger().removeHandler(self._db_log_handler)
+            self._db_log_handler = None
+            self._spider = None
 
     def spider_error(self, failure, response, spider):
         """Called when spider encounters an error"""
         message = f"Spider error: {str(failure.value)} on {response.url if response else 'unknown URL'}"
-        self._log_to_database(spider, "SPIDER_ERROR", message)
+        self._log_to_database(spider, "ERROR", message)
 
     def item_dropped(self, item, response, spider, exception):
         """Called when an item is dropped"""
         message = f"Item dropped: {str(exception)} from {response.url if response else 'unknown URL'}"
-        self._log_to_database(spider, "ITEM_DROPPED", message)
+        self._log_to_database(spider, "INFO", message)

{scrapy_item_ingest-0.1.0.dist-info → scrapy_item_ingest-0.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapy_item_ingest
-Version: 0.1.0
+Version: 0.1.1
 Summary: Scrapy extension for database ingestion with job/spider tracking
 Home-page: https://github.com/fawadss1/scrapy_item_ingest
 Author: Fawad Ali
@@ -122,7 +122,7 @@ For support and questions:
 
 ## Changelog
 
-### v0.1.0 (Current)
+### v0.1.1 (Current)
 
 - Initial release
 - Core pipeline functionality for items, requests, and logs

{scrapy_item_ingest-0.1.0.dist-info → scrapy_item_ingest-0.1.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-scrapy_item_ingest/__init__.py,sha256=2Cj2dnPpsGOTGH95ihMAdwASfT-Nhw3i4M5zfxLfgaw,1507
+scrapy_item_ingest/__init__.py,sha256=7kzpn5xDcnAf05mxtrGs0YJVzExkPuSZ9F2rpVKGUK0,1507
 scrapy_item_ingest/config/__init__.py,sha256=Foyt52_KDRIoDZtSH5ttcWxQXCOUgzebo4IGCPQwriY,55
 scrapy_item_ingest/config/settings.py,sha256=5GFQAqRf-6oc4KaMdMGuzLdtSP6fmn67anzGnKpejTI,2619
 scrapy_item_ingest/database/__init__.py,sha256=-D9cfI8Hrap74UkIUmcOZ-ikAZ8HKSswZAZMBtjq69A,50
@@ -6,7 +6,7 @@ scrapy_item_ingest/database/connection.py,sha256=bvSTCQfgBMcuKu-VzMCwMtSNBORzeea
 scrapy_item_ingest/database/schema.py,sha256=2HcBbW3VIWva59YCxyAinwZQDidFuyU5zuOCdCwBZUI,2866
 scrapy_item_ingest/extensions/__init__.py,sha256=G8xe0Bssf4jFvi3D1gNyOpylaDHlo-RKHEX9_tIB2f8,51
 scrapy_item_ingest/extensions/base.py,sha256=aodB_V47O8ihox2VdDizclAhQ6VonAUZ4lIOitub7kw,3192
-scrapy_item_ingest/extensions/logging.py,sha256=pI-MlQfNLHFmNuOSp60SfFAGRxJwPwurZp1xt88PeWU,2017
+scrapy_item_ingest/extensions/logging.py,sha256=gJGL5Ozl5kqivND1jSrw7ywilEt4Nz-rsYJiec4ZMKU,3948
 scrapy_item_ingest/pipelines/__init__.py,sha256=NvbUeLCwjFPvVaSTzdnN6LkToJ1ISAM91EmVero9FXo,50
 scrapy_item_ingest/pipelines/base.py,sha256=C6lk37lhNr6oADXofi4aqnN0cZ9u1KrWxEZMp6Oo7oA,1783
 scrapy_item_ingest/pipelines/items.py,sha256=FcgzmuFJ3qPVVQDZv9OjEPjlSbbW5MZykMmhAcdZ4Tc,1487
@@ -15,9 +15,9 @@ scrapy_item_ingest/pipelines/requests.py,sha256=3Wyzx6kgf7B_gg2DC0jhekCPSIEaJiRJ
 scrapy_item_ingest/utils/__init__.py,sha256=xuzfL-u3NkFElIrBygQISYv0CKMdSVvreuL16JXZMRM,49
 scrapy_item_ingest/utils/fingerprint.py,sha256=Qdby72nLNQp4-sxL51RM85MuxwFJHxmuYDbQv1c7hPc,855
 scrapy_item_ingest/utils/serialization.py,sha256=iKGhWnVwMKLKZ63kek4Hov9ESy9igA13CuOfDRD1W-M,942
-scrapy_item_ingest-0.1.0.dist-info/licenses/LICENSE,sha256=DhJQ4_j45c_DWghISLKmJshcLvX_Pr7QXaahe2iRMNo,1087
-scrapy_item_ingest-0.1.0.dist-info/METADATA,sha256=eiZF00MlFqNBdExA4kW2i4C_p_TvLsd4sLAsFE4uiD4,5868
-scrapy_item_ingest-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-scrapy_item_ingest-0.1.0.dist-info/entry_points.txt,sha256=WKFpo9Dy0qX1S1PT8NvIHqZmSxBCgyAM480LnLR8S1E,172
-scrapy_item_ingest-0.1.0.dist-info/top_level.txt,sha256=bu2ekFWcSH0ANdc8oGDdmZXaSC6kNuhtC-AggLsUQCU,19
-scrapy_item_ingest-0.1.0.dist-info/RECORD,,
+scrapy_item_ingest-0.1.1.dist-info/licenses/LICENSE,sha256=DhJQ4_j45c_DWghISLKmJshcLvX_Pr7QXaahe2iRMNo,1087
+scrapy_item_ingest-0.1.1.dist-info/METADATA,sha256=zwUBfROB8fYjYvgS-m36pSfexc4oE4FOe482jBr-cos,5868
+scrapy_item_ingest-0.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+scrapy_item_ingest-0.1.1.dist-info/entry_points.txt,sha256=WKFpo9Dy0qX1S1PT8NvIHqZmSxBCgyAM480LnLR8S1E,172
+scrapy_item_ingest-0.1.1.dist-info/top_level.txt,sha256=bu2ekFWcSH0ANdc8oGDdmZXaSC6kNuhtC-AggLsUQCU,19
+scrapy_item_ingest-0.1.1.dist-info/RECORD,,