zenx 0.6.9__tar.gz → 0.7.1__tar.gz

{zenx-0.6.9 → zenx-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zenx
-Version: 0.6.9
+Version: 0.7.1
 Summary: mini-framework
 Requires-Python: >=3.12
 Requires-Dist: curl-cffi>=0.12.0

{zenx-0.6.9 → zenx-0.7.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "zenx"
-version = "0.6.9"
+version = "0.7.1"
 description = "mini-framework"
 requires-python = ">=3.12"
 dependencies = [

zenx-0.7.1/zenx/pipelines/manager.py

@@ -0,0 +1,44 @@
+import asyncio
+from typing import Dict, List
+from structlog import BoundLogger
+
+from zenx.exceptions import DropItem
+from zenx.pipelines.base import Pipeline
+from zenx.clients.database import DBClient
+from zenx.settings import Settings
+
+
+class PipelineManager:
+
+
+    def __init__(self, pipeline_names: List[str], logger: BoundLogger, db: DBClient, settings: Settings) -> None:
+        self.logger = logger
+        self.pipelines = {name:Pipeline.get_pipeline(name)(logger, db, settings) for name in pipeline_names}
+        self.settings = settings
+        self._fire_and_forget_pipelines = [p for p in self.pipelines.values() if p.name != "preprocess"]
+    
+    
+    async def start_pipelines(self) -> None:
+        """ connect and monitor """
+        for pipeline in self.pipelines.values():
+            await pipeline.start()
+            
+
+    async def process_item(self, item: Dict, spider: str) -> None:
+        preprocess_pipeline = self.pipelines.get("preprocess")
+        if preprocess_pipeline:
+            try:
+                item = await preprocess_pipeline.process_item(item, spider)
+            except DropItem:
+                self.logger.debug("dropped", id=item.get("_id"), pipeline=preprocess_pipeline.name)
+                return
+            except Exception:
+                self.logger.exception("process_item", item=item, pipeline=preprocess_pipeline.name)
+                raise
+        for pipeline in self._fire_and_forget_pipelines:
+            asyncio.create_task(pipeline.process_item(item, spider))
+
+    
+    async def close_pipelines(self) -> None:
+        for pipeline in self.pipelines.values():
+            await pipeline.close()

{zenx-0.6.9 → zenx-0.7.1}/zenx/pipelines/preprocess.py

@@ -18,9 +18,9 @@ class PreprocessPipeline(Pipeline):
         self.drop_if_scraped_too_late(item)
         _id = item.get("_id")
         if _id:
-            if await self.db.exists(_id, spider):
+            inserted = await self.db.insert(_id, spider)
+            if not inserted:
                 raise DropItem
-            await self.db.insert(_id, spider)
 
         scraped_time = item['scraped_at'] - item['responded_at']
         self.logger.info("scraped", id=item.get("_id"), time_ms=scraped_time, pipeline=self.name)

{zenx-0.6.9 → zenx-0.7.1}/zenx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zenx
-Version: 0.6.9
+Version: 0.7.1
 Summary: mini-framework
 Requires-Python: >=3.12
 Requires-Dist: curl-cffi>=0.12.0

zenx-0.6.9/zenx/pipelines/manager.py

@@ -1,40 +0,0 @@
-from typing import Dict, List
-from structlog import BoundLogger
-
-from zenx.exceptions import DropItem
-from zenx.pipelines.base import Pipeline
-from zenx.clients.database import DBClient
-from zenx.settings import Settings
-
-
-class PipelineManager:
-
-
-    def __init__(self, pipeline_names: List[str], logger: BoundLogger, db: DBClient, settings: Settings) -> None:
-        self.logger = logger
-        self.pipelines = [Pipeline.get_pipeline(name)(logger, db, settings) for name in pipeline_names]
-        self.settings = settings
-    
-    
-    async def start_pipelines(self) -> None:
-        """ connect and monitor """
-        for pipeline in self.pipelines:
-            await pipeline.start()
-            
-
-    async def process_item(self, item: Dict, spider: str) -> Dict:
-        for pipeline in self.pipelines:
-            try:
-                item = await pipeline.process_item(item, spider)
-            except DropItem:
-                self.logger.debug("dropped", id=item.get("_id"))
-                break
-            except Exception:
-                self.logger.exception("process_item", pipeline=pipeline.name, item=item)
-                raise
-        return item
-
-    
-    async def close_pipelines(self) -> None:
-        for pipeline in self.pipelines:
-            await pipeline.close()