zenx 0.6.9__py3-none-any.whl → 0.7.0__py3-none-any.whl

zenx/pipelines/manager.py

@@ -1,3 +1,4 @@
+import asyncio
 from typing import Dict, List
 from structlog import BoundLogger
 
@@ -12,29 +13,32 @@ class PipelineManager:
 
     def __init__(self, pipeline_names: List[str], logger: BoundLogger, db: DBClient, settings: Settings) -> None:
         self.logger = logger
-        self.pipelines = [Pipeline.get_pipeline(name)(logger, db, settings) for name in pipeline_names]
+        self.pipelines = {name:Pipeline.get_pipeline(name)(logger, db, settings) for name in pipeline_names}
         self.settings = settings
+        self._fire_and_forget_pipelines = [p for p in self.pipelines.values() if p.name != "preprocess"]
     
     
     async def start_pipelines(self) -> None:
         """ connect and monitor """
-        for pipeline in self.pipelines:
+        for pipeline in self.pipelines.values():
             await pipeline.start()
             
 
-    async def process_item(self, item: Dict, spider: str) -> Dict:
-        for pipeline in self.pipelines:
+    async def process_item(self, item: Dict, spider: str) -> None:
+        preprocess_pipeline = self.pipelines.get("preprocess")
+        if preprocess_pipeline:
             try:
-                item = await pipeline.process_item(item, spider)
+                item = await preprocess_pipeline.process_item(item, spider)
             except DropItem:
-                self.logger.debug("dropped", id=item.get("_id"))
-                break
+                self.logger.debug("dropped", id=item.get("_id"), pipeline=preprocess_pipeline.name)
+                return
             except Exception:
-                self.logger.exception("process_item", pipeline=pipeline.name, item=item)
+                self.logger.exception("process_item", item=item, pipeline=preprocess_pipeline.name)
                 raise
-        return item
+        for pipeline in self._fire_and_forget_pipelines:
+            asyncio.create_task(pipeline.process_item(item, spider))
 
     
     async def close_pipelines(self) -> None:
-        for pipeline in self.pipelines:
+        for pipeline in self.pipelines.values():
             await pipeline.close()

{zenx-0.6.9.dist-info → zenx-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zenx
-Version: 0.6.9
+Version: 0.7.0
 Summary: mini-framework
 Requires-Python: >=3.12
 Requires-Dist: curl-cffi>=0.12.0

{zenx-0.6.9.dist-info → zenx-0.7.0.dist-info}/RECORD

@@ -12,7 +12,7 @@ zenx/clients/http.py,sha256=fb6COYot6vidNFRBWgoU6CYEfnYWJP0JuVkydvxsHb4,5700
 zenx/pipelines/__init__.py,sha256=IxkZ0UpEJdYjLdd-PMcC9PzzzArTBNNcpgKc7NiOe5Y,131
 zenx/pipelines/base.py,sha256=N_388z5DFMeaU6wMwcClZAbQFWKh4kpAF7eUJhpQevs,1863
 zenx/pipelines/google_rpc.py,sha256=F7p5ml9W1UliZbrDrF9MFNVKlCP5pG1WpO6rdmBgKp8,4707
-zenx/pipelines/manager.py,sha256=LRejhnSiXospa4cMSX9Srp0jfscGnHDggXyl1aUNhtg,1300
+zenx/pipelines/manager.py,sha256=bP2WIMblbWYHOJPU44l_IF3xLD0eHBYV4CCTFZc2FHI,1662
 zenx/pipelines/preprocess.py,sha256=k1Ake8H3nm_S_0ZnruyXAl7cFlBhLn-wpQZHuovIH4o,774
 zenx/pipelines/websocket.py,sha256=74XIV7_YbBngCgqKP3ZiBpDfQte-VB2G4kjNsZyycIc,4456
 zenx/resources/proto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -20,8 +20,8 @@ zenx/resources/proto/feed_pb2.py,sha256=ZyICOLnyuXekkvV4bAHZ1nE1-wwzcYYRRrmRJCMr
 zenx/resources/proto/feed_pb2_grpc.py,sha256=Mim6FfBgIMj0PmTqHk036nVUMJH3A6I3ts6r1j3bQF8,7441
 zenx/spiders/__init__.py,sha256=rs5LuqdM2MQlUYiTGJrzkYhzN8_SSLTrR7wGjSRrrSo,25
 zenx/spiders/base.py,sha256=YB-KqsAzfIUTzDMy5_ElgW1mul-I4Ltft6JAJxpy4hg,1672
-zenx-0.6.9.dist-info/METADATA,sha256=WqdAikf-ESTs5RbByLL7hQQuvuDeFwdobnAog5CiNFM,1273
-zenx-0.6.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-zenx-0.6.9.dist-info/entry_points.txt,sha256=8JXob2f1VtvzGFris-e9Usqywg7oca-cChDlH9moOZU,38
-zenx-0.6.9.dist-info/top_level.txt,sha256=JeXwvK86d7sB-2x-avugFnZIZa33zaHWKI8RHWJR6KY,5
-zenx-0.6.9.dist-info/RECORD,,
+zenx-0.7.0.dist-info/METADATA,sha256=aMtFlE8YY8qgo-DPOg0xz4igPppYrBBro8yXYxcnI5I,1273
+zenx-0.7.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+zenx-0.7.0.dist-info/entry_points.txt,sha256=8JXob2f1VtvzGFris-e9Usqywg7oca-cChDlH9moOZU,38
+zenx-0.7.0.dist-info/top_level.txt,sha256=JeXwvK86d7sB-2x-avugFnZIZa33zaHWKI8RHWJR6KY,5
+zenx-0.7.0.dist-info/RECORD,,