zenx 0.6.1__py3-none-any.whl → 0.6.3__py3-none-any.whl

zenx/clients/http.py

@@ -124,7 +124,7 @@ class CurlCffi(HttpClient):
                     )
                     recv_at = get_time()
                     latency = recv_at - req_at
-                    self.logger.debug("response", status=response.status_code, url=url, impersonate=session.impersonate, client=self.name, requested_at=req_at, responded_at=recv_at, latency_ms=latency)
+                    self.logger.debug("response", status=response.status_code, url=url, impersonate=impersonate, client=self.name, requested_at=req_at, responded_at=recv_at, latency_ms=latency)
                 except Exception:
                     self.logger.exception("request", url=url, client=self.name)
                     raise

zenx/engine.py

@@ -26,6 +26,16 @@ class Engine:
         self.shutdown_event.set()
 
 
+    async def _log_stats(self, spider: Spider) -> None:
+        while True:
+            await asyncio.sleep(60)
+            try:
+                rpm = spider.get_stats().get("rpm")
+                spider.logger.info("stats", rpm=rpm)
+            except asyncio.CancelledError:
+                break
+
+
     async def _execute(self, spider_name: str) -> None:
         loop = asyncio.get_running_loop()
         loop.add_signal_handler(signal.SIGINT, self._shutdown_handler)
@@ -48,6 +58,7 @@ class Engine:
 
         spider = spider_cls(client=client, pm=pm, logger=logger, settings=settings)
         
+        stats_task = asyncio.create_task(self._log_stats(spider))
         try:
             if self.forever:
                 while not self.shutdown_event.is_set():
@@ -61,6 +72,15 @@ class Engine:
         finally:
             if self.shutdown_event.is_set():
                 logger.info("shutdown", spider=spider_name)
+
+            stats_task.cancel()
+            try:
+                await stats_task
+            except asyncio.CancelledError:
+                pass
+            stats = spider.get_stats()
+            logger.info("summary", **stats)
+
             await client.close()
             await db.close()
             await pm.close_pipelines()

zenx/spiders/base.py

@@ -1,5 +1,7 @@
 from __future__ import annotations 
 from abc import ABC, abstractmethod
+from collections import deque
+import time
 from typing import ClassVar, Dict, List, Literal, Type
 from structlog import BoundLogger
 
@@ -44,6 +46,22 @@ class Spider(ABC):
         self.pm = pm
         self.logger = logger
         self.settings = settings
+        # stats
+        self.requests_timestamps = deque()
+        self.total_requests = 0
+
+
+    def get_stats(self) -> Dict:
+        # calculate RPM based on rolling window of 1 min
+        now = time.time()
+        # remove requests older than 60 sec
+        while self.requests_timestamps and self.requests_timestamps[0] < (now - 60):
+            self.requests_timestamps.popleft()
+        rpm = len(self.requests_timestamps)
+        return {
+            "rpm": rpm,
+            "total_requests": self.total_requests,
+        }
 
 
     @abstractmethod

{zenx-0.6.1.dist-info → zenx-0.6.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zenx
-Version: 0.6.1
+Version: 0.6.3
 Summary: mini-framework
 Requires-Python: >=3.12
 Requires-Dist: curl-cffi>=0.12.0

{zenx-0.6.1.dist-info → zenx-0.6.3.dist-info}/RECORD

@@ -1,14 +1,14 @@
 zenx/cli.py,sha256=pHKhOTdqI6NQQoYK91waRIMpxCXLYtXEryzVaTbmvqc,2810
 zenx/debug_runner.py,sha256=B2Jd9A4_EHDa-ohLcwnFIxOV73FagTWXX2fl3qgwlpY,809
 zenx/discovery.py,sha256=YANVGzy2IG1fYruUud-11Y-ynyO6iEp3EjlHnhIQJQI,1014
-zenx/engine.py,sha256=VX2sDmEqvgfvkN1lDFNTgQIKuX1U1aobt-5WhoCgV_g,2350
+zenx/engine.py,sha256=1rIX44wMjgGrvarVVe6uD3MQO9HJ31cBHSTPqo-qNE8,2948
 zenx/exceptions.py,sha256=BJXxzwwX2CU6inhppfblx8c8Z6Mhvsk7MAhQ1LAnhBg,37
 zenx/logger.py,sha256=UmEk0vV1mSCozV7z_DDgCCXdAManDr5wgkrhKiRQtyU,1651
 zenx/settings.py,sha256=Q0z3oGyVkhucRyqsraZQR1lAPSq1sjiGKmAkDCmQsA0,940
 zenx/utils.py,sha256=ozf3dpcH1KyDXh9coDe3NqzL8OS-1uA6SaQ8lfHfI40,587
 zenx/clients/__init__.py,sha256=CaAAuNa8DPyMdejR0KNSDDg_UzC3WxaTol5_QvwwwG8,132
 zenx/clients/database.py,sha256=GaIafQ2OkYJQ_sz2f6u7sZKn77VOReWun09hogk6dO0,6019
-zenx/clients/http.py,sha256=tkUN-1Lij1GFhxqTVry_mTWhB5XRNt_5t38mRoRdF84,5700
+zenx/clients/http.py,sha256=MXXC0dYqd3mfuywsh4UMPXm0-9d0qoXHUky-ZC1DTE4,5692
 zenx/pipelines/__init__.py,sha256=IxkZ0UpEJdYjLdd-PMcC9PzzzArTBNNcpgKc7NiOe5Y,131
 zenx/pipelines/base.py,sha256=N_388z5DFMeaU6wMwcClZAbQFWKh4kpAF7eUJhpQevs,1863
 zenx/pipelines/google_rpc.py,sha256=F7p5ml9W1UliZbrDrF9MFNVKlCP5pG1WpO6rdmBgKp8,4707
@@ -19,9 +19,9 @@ zenx/resources/proto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 zenx/resources/proto/feed_pb2.py,sha256=ZyICOLnyuXekkvV4bAHZ1nE1-wwzcYYRRrmRJCMrSoo,2810
 zenx/resources/proto/feed_pb2_grpc.py,sha256=Mim6FfBgIMj0PmTqHk036nVUMJH3A6I3ts6r1j3bQF8,7441
 zenx/spiders/__init__.py,sha256=rs5LuqdM2MQlUYiTGJrzkYhzN8_SSLTrR7wGjSRrrSo,25
-zenx/spiders/base.py,sha256=KoMsT7ezwne1clTtP66dfvQGZPzdg2e7ygkM40h2QbA,1659
-zenx-0.6.1.dist-info/METADATA,sha256=OcOwmfvef4XORRAqIE1RrBbOP0dSiPcf5-_eyFWYnJM,1273
-zenx-0.6.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-zenx-0.6.1.dist-info/entry_points.txt,sha256=8JXob2f1VtvzGFris-e9Usqywg7oca-cChDlH9moOZU,38
-zenx-0.6.1.dist-info/top_level.txt,sha256=JeXwvK86d7sB-2x-avugFnZIZa33zaHWKI8RHWJR6KY,5
-zenx-0.6.1.dist-info/RECORD,,
+zenx/spiders/base.py,sha256=LaX29ArNIAiv8E8hUUpJxahBpRnGfskDlxx4hU0gsTg,2232
+zenx-0.6.3.dist-info/METADATA,sha256=eoDHZHHXDTZwAkOx8yovY5VIJnjHV2XJcSHoft8stUg,1273
+zenx-0.6.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+zenx-0.6.3.dist-info/entry_points.txt,sha256=8JXob2f1VtvzGFris-e9Usqywg7oca-cChDlH9moOZU,38
+zenx-0.6.3.dist-info/top_level.txt,sha256=JeXwvK86d7sB-2x-avugFnZIZa33zaHWKI8RHWJR6KY,5
+zenx-0.6.3.dist-info/RECORD,,