zenx 0.6.4__py3-none-any.whl → 0.6.5__py3-none-any.whl

zenx/clients/http.py

@@ -1,4 +1,6 @@
 from __future__ import annotations
+import time
+from collections import deque
 import parsel
 import random
 from curl_cffi.requests.impersonate import BrowserTypeLiteral
@@ -11,7 +13,7 @@ import json
 from structlog import BoundLogger
 
 from zenx.settings import Settings
-from zenx.utils import get_time
+from zenx.utils import get_time, record_request
 
 
 @dataclass
@@ -56,8 +58,24 @@ class HttpClient(ABC):
         self.logger = logger
         self.settings = settings
         self._session_pool: asyncio.Queue
+        # stats
+        self._requests_timestamps = deque()
+        self._total_requests = 0
+    
 
+    def get_stats(self) -> Dict:
+        # calculate RPM based on rolling window of 1 min
+        now = time.time()
+        # remove requests older than 60 sec
+        while self._requests_timestamps and self._requests_timestamps[0] < (now - 60):
+            self._requests_timestamps.popleft()
+        rpm = len(self._requests_timestamps)
+        return {
+            "rpm": rpm,
+            "total_requests": self._total_requests,
+        }
 
+    
     @abstractmethod
     async def request(
         self,
@@ -96,6 +114,7 @@ class CurlCffi(HttpClient):
         return chosen_fingerprint
 
     
+    @record_request
     async def request(
         self,
         url: str,
@@ -124,7 +143,7 @@ class CurlCffi(HttpClient):
                     )
                     recv_at = get_time()
                     latency = recv_at - req_at
-                    self.logger.debug("response", status=response.status_code, url=url, impersonate=impersonate, client=self.name, requested_at=req_at, responded_at=recv_at, latency_ms=latency)
+                    self.logger.debug("response", status=response.status_code, url=url, impersonate=session.impersonate, client=self.name, requested_at=req_at, responded_at=recv_at, latency_ms=latency)
                 except Exception:
                     self.logger.exception("request", url=url, client=self.name)
                     raise

zenx/spiders/base.py

@@ -1,7 +1,5 @@
 from __future__ import annotations 
 from abc import ABC, abstractmethod
-from collections import deque
-import time
 from typing import ClassVar, Dict, List, Literal, Type
 from structlog import BoundLogger
 
@@ -46,22 +44,6 @@ class Spider(ABC):
         self.pm = pm
         self.logger = logger
         self.settings = settings
-        # stats
-        self.requests_timestamps = deque()
-        self.total_requests = 0
-
-
-    def get_stats(self) -> Dict:
-        # calculate RPM based on rolling window of 1 min
-        now = time.time()
-        # remove requests older than 60 sec
-        while self.requests_timestamps and self.requests_timestamps[0] < (now - 60):
-            self.requests_timestamps.popleft()
-        rpm = len(self.requests_timestamps)
-        return {
-            "rpm": rpm,
-            "total_requests": self.total_requests,
-        }
 
 
     @abstractmethod

zenx/utils.py

@@ -2,6 +2,8 @@ import time
 from typing import Dict
 import functools
 
+from zenx.clients.http import HttpClient, Response
+
 
 def get_time() -> int:
     """ current unix time in milliseconds """
@@ -17,3 +19,13 @@ def log_processing_time(func):
         self.logger.info("processed", id=item['_id'], time_ms=processed_time, pipeline=self.name)
         return result
     return wrapper
+
+
+def record_request(func):
+    @functools.wraps(func)
+    async def wrapper(self: HttpClient, *args, **kwargs) -> Response:
+        result = await func(self, *args, **kwargs)
+        self._requests_timestamps.append(time.time())
+        self._total_requests +=1
+        return result
+    return wrapper

{zenx-0.6.4.dist-info → zenx-0.6.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zenx
-Version: 0.6.4
+Version: 0.6.5
 Summary: mini-framework
 Requires-Python: >=3.12
 Requires-Dist: curl-cffi>=0.12.0

{zenx-0.6.4.dist-info → zenx-0.6.5.dist-info}/RECORD

@@ -5,10 +5,10 @@ zenx/engine.py,sha256=1rIX44wMjgGrvarVVe6uD3MQO9HJ31cBHSTPqo-qNE8,2948
 zenx/exceptions.py,sha256=BJXxzwwX2CU6inhppfblx8c8Z6Mhvsk7MAhQ1LAnhBg,37
 zenx/logger.py,sha256=UmEk0vV1mSCozV7z_DDgCCXdAManDr5wgkrhKiRQtyU,1651
 zenx/settings.py,sha256=Q0z3oGyVkhucRyqsraZQR1lAPSq1sjiGKmAkDCmQsA0,940
-zenx/utils.py,sha256=ozf3dpcH1KyDXh9coDe3NqzL8OS-1uA6SaQ8lfHfI40,587
+zenx/utils.py,sha256=oMKKssN8VNSY7JJ7G04liOKuroPz4chLPJEHfpDeUF0,943
 zenx/clients/__init__.py,sha256=CaAAuNa8DPyMdejR0KNSDDg_UzC3WxaTol5_QvwwwG8,132
 zenx/clients/database.py,sha256=GaIafQ2OkYJQ_sz2f6u7sZKn77VOReWun09hogk6dO0,6019
-zenx/clients/http.py,sha256=MXXC0dYqd3mfuywsh4UMPXm0-9d0qoXHUky-ZC1DTE4,5692
+zenx/clients/http.py,sha256=uhwEbrrOGtNclPYKQik4xtQuA61FvwOzWYSkt3VhQiA,6324
 zenx/pipelines/__init__.py,sha256=IxkZ0UpEJdYjLdd-PMcC9PzzzArTBNNcpgKc7NiOe5Y,131
 zenx/pipelines/base.py,sha256=N_388z5DFMeaU6wMwcClZAbQFWKh4kpAF7eUJhpQevs,1863
 zenx/pipelines/google_rpc.py,sha256=F7p5ml9W1UliZbrDrF9MFNVKlCP5pG1WpO6rdmBgKp8,4707
@@ -19,9 +19,9 @@ zenx/resources/proto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 zenx/resources/proto/feed_pb2.py,sha256=ZyICOLnyuXekkvV4bAHZ1nE1-wwzcYYRRrmRJCMrSoo,2810
 zenx/resources/proto/feed_pb2_grpc.py,sha256=Mim6FfBgIMj0PmTqHk036nVUMJH3A6I3ts6r1j3bQF8,7441
 zenx/spiders/__init__.py,sha256=rs5LuqdM2MQlUYiTGJrzkYhzN8_SSLTrR7wGjSRrrSo,25
-zenx/spiders/base.py,sha256=V_fOG_4XGXXbjTsi1T7_yXkQetHhceQ2vzoTAyaeDyI,2245
-zenx-0.6.4.dist-info/METADATA,sha256=87T6NmnGEsrajcvT6_XE-x8lOFbjb0OrtmRUVPZwLQs,1273
-zenx-0.6.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-zenx-0.6.4.dist-info/entry_points.txt,sha256=8JXob2f1VtvzGFris-e9Usqywg7oca-cChDlH9moOZU,38
-zenx-0.6.4.dist-info/top_level.txt,sha256=JeXwvK86d7sB-2x-avugFnZIZa33zaHWKI8RHWJR6KY,5
-zenx-0.6.4.dist-info/RECORD,,
+zenx/spiders/base.py,sha256=YB-KqsAzfIUTzDMy5_ElgW1mul-I4Ltft6JAJxpy4hg,1672
+zenx-0.6.5.dist-info/METADATA,sha256=5Gbx1XogHN6sq_fFCcZeE8rK1W_IqGX-9Ema5jg1pDw,1273
+zenx-0.6.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+zenx-0.6.5.dist-info/entry_points.txt,sha256=8JXob2f1VtvzGFris-e9Usqywg7oca-cChDlH9moOZU,38
+zenx-0.6.5.dist-info/top_level.txt,sha256=JeXwvK86d7sB-2x-avugFnZIZa33zaHWKI8RHWJR6KY,5
+zenx-0.6.5.dist-info/RECORD,,