zenx 0.8.0__py3-none-any.whl → 0.8.2__py3-none-any.whl

zenx/engine.py

@@ -61,7 +61,7 @@ class Engine:
             if self.shutdown_event.is_set():
                 logger.info("shutdown", spider=spider_name)
             if spider.background_tasks:
-                logger.debug("waiting", background_tasks=len(spider.background_tasks))
+                logger.debug("waiting", background_tasks=len(spider.background_tasks), belong_to="spider")
                 await asyncio.gather(*spider.background_tasks)
             await client.close()
             await db.close()

zenx/pipelines/google_rpc.py

@@ -83,8 +83,6 @@ try:
                 await self._stub.SubmitFeedMessage(feed_message)
             except grpc.RpcError as e:
                 self.logger.error("processing", exception=str(e), id=item['_id'], feed=self._feed_id, pipeline=self.name)
-            else:
-                self.logger.info("processed", id=item['_id'], feed=self._feed_id, pipeline=self.name)
 
 
         async def close(self) -> None:

zenx/pipelines/manager.py

@@ -16,7 +16,8 @@ class PipelineManager:
         self.pipelines = {name:Pipeline.get_pipeline(name)(logger, db, settings) for name in pipeline_names}
         self.settings = settings
         self._fire_and_forget_pipelines = [p for p in self.pipelines.values() if p.name != "preprocess"]
-    
+        self._background_tasks = set()
+
     
     async def start_pipelines(self) -> None:
         """ connect and monitor """
@@ -36,9 +37,14 @@ class PipelineManager:
                 self.logger.exception("process_item", item=item, pipeline=preprocess_pipeline.name)
                 raise
         for pipeline in self._fire_and_forget_pipelines:
-            asyncio.create_task(pipeline.process_item(item, spider))
+            t = asyncio.create_task(pipeline.process_item(item, spider))
+            self._background_tasks.add(t)
+            t.add_done_callback(self._background_tasks.discard)
 
     
     async def close_pipelines(self) -> None:
+        if self._background_tasks:
+            self.logger.debug("waiting", background_tasks=len(self._background_tasks), belong_to="pipeline_manager")
+            await asyncio.gather(*self._background_tasks)
         for pipeline in self.pipelines.values():
             await pipeline.close()

zenx/pipelines/websocket.py

@@ -24,6 +24,7 @@ try:
             self._connected = asyncio.Event()
             self._ws_client: Optional[websockets.ClientConnection] = None
             self._monitor_state_task: Optional[asyncio.Task] = None
+            self._listening_task: Optional[asyncio.Task] = None
 
 
         async def start(self) -> None:
@@ -37,6 +38,7 @@ try:
                 raise
             else:
                 self._monitor_state_task = asyncio.create_task(self._monitor_state())
+                self._listening_task = asyncio.create_task(self._listen())
 
         
         async def _monitor_state(self) -> None:
@@ -48,6 +50,15 @@ try:
                     await asyncio.sleep(0.5)
                 
 
+        async def _listen(self) -> None:
+            try:
+                await self._connected.wait()
+                async for msg in self._ws_client:
+                    self.logger.debug("received", msg=msg.decode(), pipeline=self.name)
+            except Exception:
+                await asyncio.sleep(1)
+
+
         async def _connect(self) -> None:
             self._connected.clear()
             self.logger.debug("connecting", pipeline=self.name)
@@ -84,12 +95,13 @@ try:
             
 
         async def close(self) -> None:
-            if self._monitor_state_task and not self._monitor_state_task.done():
-                self._monitor_state_task.cancel()
-                try:
-                    await self._monitor_state_task
-                except asyncio.CancelledError:
-                    pass
+            for t in [self._monitor_state_task, self._listening_task]:
+                if t and not t.done():
+                    t.cancel()
+                    try:
+                        await t
+                    except asyncio.CancelledError:
+                        pass
             await self._ws_client.close()
             self.logger.debug("closed", pipeline=self.name)
 

{zenx-0.8.0.dist-info → zenx-0.8.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zenx
-Version: 0.8.0
+Version: 0.8.2
 Summary: mini-framework
 Requires-Python: >=3.12
 Requires-Dist: curl-cffi>=0.12.0

{zenx-0.8.0.dist-info → zenx-0.8.2.dist-info}/RECORD

@@ -1,7 +1,7 @@
 zenx/cli.py,sha256=pHKhOTdqI6NQQoYK91waRIMpxCXLYtXEryzVaTbmvqc,2810
 zenx/debug_runner.py,sha256=B2Jd9A4_EHDa-ohLcwnFIxOV73FagTWXX2fl3qgwlpY,809
 zenx/discovery.py,sha256=YANVGzy2IG1fYruUud-11Y-ynyO6iEp3EjlHnhIQJQI,1014
-zenx/engine.py,sha256=fZ080nkZmCKp0Ra0sGEm9xBQ9dCzU6vT2CWF4ma-QA0,2531
+zenx/engine.py,sha256=CYwvmPSaI1lC-y_DfuOzX_1Ii14fxU8IdfMM7AIrUUU,2551
 zenx/exceptions.py,sha256=BJXxzwwX2CU6inhppfblx8c8Z6Mhvsk7MAhQ1LAnhBg,37
 zenx/logger.py,sha256=UmEk0vV1mSCozV7z_DDgCCXdAManDr5wgkrhKiRQtyU,1651
 zenx/settings.py,sha256=0RLeKcqHJdD1vuYIQtGeXkp2YIvv3tNyz9O8jw620VQ,1008
@@ -11,17 +11,17 @@ zenx/clients/database.py,sha256=AF-L7iYrWRNzUZKn7taveiihpu--mXXC6eWOrMNlqzQ,4806
 zenx/clients/http.py,sha256=fb6COYot6vidNFRBWgoU6CYEfnYWJP0JuVkydvxsHb4,5700
 zenx/pipelines/__init__.py,sha256=IxkZ0UpEJdYjLdd-PMcC9PzzzArTBNNcpgKc7NiOe5Y,131
 zenx/pipelines/base.py,sha256=N_388z5DFMeaU6wMwcClZAbQFWKh4kpAF7eUJhpQevs,1863
-zenx/pipelines/google_rpc.py,sha256=8DGcUlE6Th39XgD7WMMxqBMhUkH0kr3l9-cN_1x4CY4,4610
-zenx/pipelines/manager.py,sha256=bP2WIMblbWYHOJPU44l_IF3xLD0eHBYV4CCTFZc2FHI,1662
+zenx/pipelines/google_rpc.py,sha256=nR7hNCqAcg1YGWYm3xoj8G8vOTs74lAeYc0dYk842n8,4490
+zenx/pipelines/manager.py,sha256=lm2A_4h8NNdurFfwrrLwx5Z1tqJ9yZp2qgYvwGWd1lc,2017
 zenx/pipelines/preprocess.py,sha256=zXuhaLdH5KfBi_Bf-K5BgzFEujdmPWWpwYKkpOifT5U,816
-zenx/pipelines/websocket.py,sha256=jkRp5a6ro3q2JPAm8qtZWnf0EzFFUFOQgzgSxURXTvk,4359
+zenx/pipelines/websocket.py,sha256=27eda1wpdl2KI8fz78kyiehLFAwzuOhOdYHEgygdnEo,4813
 zenx/resources/proto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 zenx/resources/proto/feed_pb2.py,sha256=ZyICOLnyuXekkvV4bAHZ1nE1-wwzcYYRRrmRJCMrSoo,2810
 zenx/resources/proto/feed_pb2_grpc.py,sha256=Mim6FfBgIMj0PmTqHk036nVUMJH3A6I3ts6r1j3bQF8,7441
 zenx/spiders/__init__.py,sha256=rs5LuqdM2MQlUYiTGJrzkYhzN8_SSLTrR7wGjSRrrSo,25
 zenx/spiders/base.py,sha256=BGxOvNnjaz6gmZm7NIgYc0RddWDmpIp53ukxprUUd-Y,1932
-zenx-0.8.0.dist-info/METADATA,sha256=7-IqNel1NhxaqGWQ4YY38IUVM4mglWRyDgugcT0D6mc,1273
-zenx-0.8.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-zenx-0.8.0.dist-info/entry_points.txt,sha256=8JXob2f1VtvzGFris-e9Usqywg7oca-cChDlH9moOZU,38
-zenx-0.8.0.dist-info/top_level.txt,sha256=JeXwvK86d7sB-2x-avugFnZIZa33zaHWKI8RHWJR6KY,5
-zenx-0.8.0.dist-info/RECORD,,
+zenx-0.8.2.dist-info/METADATA,sha256=iQA1IwASxOkMUH_WPu22DKBLv9gFNkN6zNObL80mqok,1273
+zenx-0.8.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+zenx-0.8.2.dist-info/entry_points.txt,sha256=8JXob2f1VtvzGFris-e9Usqywg7oca-cChDlH9moOZU,38
+zenx-0.8.2.dist-info/top_level.txt,sha256=JeXwvK86d7sB-2x-avugFnZIZa33zaHWKI8RHWJR6KY,5
+zenx-0.8.2.dist-info/RECORD,,