zenx 0.6.2__tar.gz → 0.6.4__tar.gz

{zenx-0.6.2 → zenx-0.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zenx
-Version: 0.6.2
+Version: 0.6.4
 Summary: mini-framework
 Requires-Python: >=3.12
 Requires-Dist: curl-cffi>=0.12.0

{zenx-0.6.2 → zenx-0.6.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "zenx"
-version = "0.6.2"
+version = "0.6.4"
 description = "mini-framework"
 requires-python = ">=3.12"
 dependencies = [

{zenx-0.6.2 → zenx-0.6.4}/zenx/engine.py

@@ -26,6 +26,16 @@ class Engine:
         self.shutdown_event.set()
 
 
+    async def _log_stats(self, spider: Spider) -> None:
+        while True:
+            await asyncio.sleep(60)
+            try:
+                rpm = spider.get_stats().get("rpm")
+                spider.logger.info("stats", rpm=rpm)
+            except asyncio.CancelledError:
+                break
+
+
     async def _execute(self, spider_name: str) -> None:
         loop = asyncio.get_running_loop()
         loop.add_signal_handler(signal.SIGINT, self._shutdown_handler)
@@ -48,6 +58,7 @@ class Engine:
 
         spider = spider_cls(client=client, pm=pm, logger=logger, settings=settings)
         
+        stats_task = asyncio.create_task(self._log_stats(spider))
         try:
             if self.forever:
                 while not self.shutdown_event.is_set():
@@ -61,6 +72,15 @@ class Engine:
         finally:
             if self.shutdown_event.is_set():
                 logger.info("shutdown", spider=spider_name)
+
+            stats_task.cancel()
+            try:
+                await stats_task
+            except asyncio.CancelledError:
+                pass
+            stats = spider.get_stats()
+            logger.info("summary", **stats)
+
             await client.close()
             await db.close()
             await pm.close_pipelines()

{zenx-0.6.2 → zenx-0.6.4}/zenx/spiders/base.py

@@ -1,5 +1,7 @@
 from __future__ import annotations 
 from abc import ABC, abstractmethod
+from collections import deque
+import time
 from typing import ClassVar, Dict, List, Literal, Type
 from structlog import BoundLogger
 
@@ -13,7 +15,7 @@ class Spider(ABC):
     # central registry
     name: ClassVar[str]
     _registry: ClassVar[Dict[str, Type[Spider]]] = {}
-    pipelines: ClassVar[List[Literal["synoptic_websocket","synoptic_grpc"]]]
+    pipelines: ClassVar[List[Literal["preprocess","synoptic_websocket","synoptic_grpc"]]]
     client_name: ClassVar[Literal["curl_cffi"]] = "curl_cffi"
 
 
@@ -44,6 +46,22 @@ class Spider(ABC):
         self.pm = pm
         self.logger = logger
         self.settings = settings
+        # stats
+        self.requests_timestamps = deque()
+        self.total_requests = 0
+
+
+    def get_stats(self) -> Dict:
+        # calculate RPM based on rolling window of 1 min
+        now = time.time()
+        # remove requests older than 60 sec
+        while self.requests_timestamps and self.requests_timestamps[0] < (now - 60):
+            self.requests_timestamps.popleft()
+        rpm = len(self.requests_timestamps)
+        return {
+            "rpm": rpm,
+            "total_requests": self.total_requests,
+        }
 
 
     @abstractmethod

{zenx-0.6.2 → zenx-0.6.4}/zenx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zenx
-Version: 0.6.2
+Version: 0.6.4
 Summary: mini-framework
 Requires-Python: >=3.12
 Requires-Dist: curl-cffi>=0.12.0