zenx 0.10.2__py3-none-any.whl → 0.10.4__py3-none-any.whl

zenx/engine.py

@@ -1,6 +1,6 @@
 import asyncio
 import signal
-from typing import List, Optional
+from typing import List
 from dotenv import load_dotenv
 import pebble
 import uvloop
@@ -26,7 +26,7 @@ class Engine:
         self.shutdown_event.set()
 
 
-    async def _execute_spider(self, spider_name: str) -> None:
+    async def _execute(self, spider_name: str) -> None:
         loop = asyncio.get_running_loop()
         loop.add_signal_handler(signal.SIGINT, self._shutdown_handler)
         loop.add_signal_handler(signal.SIGTERM, self._shutdown_handler)
@@ -47,41 +47,33 @@ class Engine:
         await pm.start_pipelines()
 
         spider = spider_cls(client=client, pm=pm, logger=logger, settings=settings)
-        crawl_task: Optional[asyncio.Task] = None
         try:
             if self.forever:
                 while not self.shutdown_event.is_set():
-                    crawl_task = asyncio.create_task(spider.crawl())
+                    tasks = [asyncio.create_task(spider.crawl()) for _ in range(settings.CONCURRENCY)]
                     try:
-                        await crawl_task
+                        results = await asyncio.gather(*tasks, return_exceptions=True)
+                        for result in results:
+                            if isinstance(result, Exception):
+                                raise result
                     except Exception:
                         logger.exception("crawl")
                         await asyncio.sleep(0.01)
             else:
-                crawl_task = asyncio.create_task(spider.crawl())
-                await crawl_task
+                await spider.crawl()
         finally:
             if self.shutdown_event.is_set():
                 logger.info("shutdown", spider=spider_name)
-
-            if crawl_task and not crawl_task.done():
-                crawl_task.cancel()
-                logger.debug("cancelled", task="crawl")
-                try:
-                    await crawl_task
-                except asyncio.CancelledError:
-                    pass
             if spider.background_tasks:
                 logger.debug("waiting", background_tasks=len(spider.background_tasks), belong_to="spider")
                 await asyncio.gather(*spider.background_tasks)
-
             await client.close()
             await db.close()
             await pm.close_pipelines()
     
 
     def run_spider(self, spider: str) -> None:
-        uvloop.run(self._execute_spider(spider))
+        uvloop.run(self._execute(spider))
 
 
     def run_spiders(self, spiders: List[str]) -> None:

zenx/settings.py

@@ -10,6 +10,7 @@ class Settings(BaseSettings):
     MAX_SCRAPE_DELAY: int = 10  # 10 seconds
     DQ_MAX_SIZE: int = 100  # max size of the deque for memory database
     REDIS_RECORD_EXPIRY_SECONDS: int = 604800 # 7 days (7*24*60*60)
+    CONCURRENCY: int = 1
 
     DB_TYPE: Literal["memory", "redis"] = "memory"
     DB_NAME: str | None = None

{zenx-0.10.2.dist-info → zenx-0.10.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zenx
-Version: 0.10.2
+Version: 0.10.4
 Summary: mini-framework
 Requires-Python: >=3.12
 Requires-Dist: curl-cffi>=0.12.0

{zenx-0.10.2.dist-info → zenx-0.10.4.dist-info}/RECORD

@@ -1,10 +1,10 @@
 zenx/cli.py,sha256=pHKhOTdqI6NQQoYK91waRIMpxCXLYtXEryzVaTbmvqc,2810
 zenx/debug_runner.py,sha256=B2Jd9A4_EHDa-ohLcwnFIxOV73FagTWXX2fl3qgwlpY,809
 zenx/discovery.py,sha256=YANVGzy2IG1fYruUud-11Y-ynyO6iEp3EjlHnhIQJQI,1014
-zenx/engine.py,sha256=rcrVrXdKORLST648XxGmC8sI54i1IVD32HccxGB0bxU,3028
+zenx/engine.py,sha256=G3ryzQ9HjI8D47e4-tlTNMwtFYbhluEMe4jrU_-FhUQ,2850
 zenx/exceptions.py,sha256=BJXxzwwX2CU6inhppfblx8c8Z6Mhvsk7MAhQ1LAnhBg,37
 zenx/logger.py,sha256=lr45XGbV769NQcwn8-lAcPfFbR4yBN8LNh7o-i4Aa9M,1652
-zenx/settings.py,sha256=anTT_jSQxwIabBHCM7egYFUOnALxp4fF_26_2ZPtTt4,1044
+zenx/settings.py,sha256=Ob4gvov-WVzA4GgpqmIgNcKaX7jWCAyYgpM9DXbbeHU,1069
 zenx/utils.py,sha256=ouvXUd3-HaxLS3dDKbshEf-CXRf44OcovziEVIRsWNs,725
 zenx/clients/__init__.py,sha256=CaAAuNa8DPyMdejR0KNSDDg_UzC3WxaTol5_QvwwwG8,132
 zenx/clients/database.py,sha256=AF-L7iYrWRNzUZKn7taveiihpu--mXXC6eWOrMNlqzQ,4806
@@ -21,8 +21,8 @@ zenx/resources/proto/feed_pb2.py,sha256=ZyICOLnyuXekkvV4bAHZ1nE1-wwzcYYRRrmRJCMr
 zenx/resources/proto/feed_pb2_grpc.py,sha256=Mim6FfBgIMj0PmTqHk036nVUMJH3A6I3ts6r1j3bQF8,7441
 zenx/spiders/__init__.py,sha256=rs5LuqdM2MQlUYiTGJrzkYhzN8_SSLTrR7wGjSRrrSo,25
 zenx/spiders/base.py,sha256=MeZ3wZOPOyOX4V2ufFXtYGCDtXHZO_mNfnXdKMkisuQ,1951
-zenx-0.10.2.dist-info/METADATA,sha256=6qMTNCG09Dd7MaVcaMSsnMPP_0NKSYuwo1VOKv2q_LA,1422
-zenx-0.10.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-zenx-0.10.2.dist-info/entry_points.txt,sha256=8JXob2f1VtvzGFris-e9Usqywg7oca-cChDlH9moOZU,38
-zenx-0.10.2.dist-info/top_level.txt,sha256=JeXwvK86d7sB-2x-avugFnZIZa33zaHWKI8RHWJR6KY,5
-zenx-0.10.2.dist-info/RECORD,,
+zenx-0.10.4.dist-info/METADATA,sha256=vcpC3WR_UsidsEfDdmDw0Qvlc7Sh7LYySn5wbxJKIUI,1422
+zenx-0.10.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+zenx-0.10.4.dist-info/entry_points.txt,sha256=8JXob2f1VtvzGFris-e9Usqywg7oca-cChDlH9moOZU,38
+zenx-0.10.4.dist-info/top_level.txt,sha256=JeXwvK86d7sB-2x-avugFnZIZa33zaHWKI8RHWJR6KY,5
+zenx-0.10.4.dist-info/RECORD,,