zenx 0.6.3__tar.gz → 0.6.5__tar.gz

{zenx-0.6.3 → zenx-0.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zenx
-Version: 0.6.3
+Version: 0.6.5
 Summary: mini-framework
 Requires-Python: >=3.12
 Requires-Dist: curl-cffi>=0.12.0

{zenx-0.6.3 → zenx-0.6.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "zenx"
-version = "0.6.3"
+version = "0.6.5"
 description = "mini-framework"
 requires-python = ">=3.12"
 dependencies = [

{zenx-0.6.3 → zenx-0.6.5}/zenx/clients/http.py

@@ -1,4 +1,6 @@
 from __future__ import annotations
+import time
+from collections import deque
 import parsel
 import random
 from curl_cffi.requests.impersonate import BrowserTypeLiteral
@@ -11,7 +13,7 @@ import json
 from structlog import BoundLogger
 
 from zenx.settings import Settings
-from zenx.utils import get_time
+from zenx.utils import get_time, record_request
 
 
 @dataclass
@@ -56,8 +58,24 @@ class HttpClient(ABC):
         self.logger = logger
         self.settings = settings
         self._session_pool: asyncio.Queue
+        # stats
+        self._requests_timestamps = deque()
+        self._total_requests = 0
+    
 
+    def get_stats(self) -> Dict:
+        # calculate RPM based on rolling window of 1 min
+        now = time.time()
+        # remove requests older than 60 sec
+        while self._requests_timestamps and self._requests_timestamps[0] < (now - 60):
+            self._requests_timestamps.popleft()
+        rpm = len(self._requests_timestamps)
+        return {
+            "rpm": rpm,
+            "total_requests": self._total_requests,
+        }
 
+    
     @abstractmethod
     async def request(
         self,
@@ -96,6 +114,7 @@ class CurlCffi(HttpClient):
         return chosen_fingerprint
 
     
+    @record_request
     async def request(
         self,
         url: str,
@@ -124,7 +143,7 @@ class CurlCffi(HttpClient):
                     )
                     recv_at = get_time()
                     latency = recv_at - req_at
-                    self.logger.debug("response", status=response.status_code, url=url, impersonate=impersonate, client=self.name, requested_at=req_at, responded_at=recv_at, latency_ms=latency)
+                    self.logger.debug("response", status=response.status_code, url=url, impersonate=session.impersonate, client=self.name, requested_at=req_at, responded_at=recv_at, latency_ms=latency)
                 except Exception:
                     self.logger.exception("request", url=url, client=self.name)
                     raise

{zenx-0.6.3 → zenx-0.6.5}/zenx/spiders/base.py

@@ -1,7 +1,5 @@
 from __future__ import annotations 
 from abc import ABC, abstractmethod
-from collections import deque
-import time
 from typing import ClassVar, Dict, List, Literal, Type
 from structlog import BoundLogger
 
@@ -15,7 +13,7 @@ class Spider(ABC):
     # central registry
     name: ClassVar[str]
     _registry: ClassVar[Dict[str, Type[Spider]]] = {}
-    pipelines: ClassVar[List[Literal["synoptic_websocket","synoptic_grpc"]]]
+    pipelines: ClassVar[List[Literal["preprocess","synoptic_websocket","synoptic_grpc"]]]
     client_name: ClassVar[Literal["curl_cffi"]] = "curl_cffi"
 
 
@@ -46,22 +44,6 @@ class Spider(ABC):
         self.pm = pm
         self.logger = logger
         self.settings = settings
-        # stats
-        self.requests_timestamps = deque()
-        self.total_requests = 0
-
-
-    def get_stats(self) -> Dict:
-        # calculate RPM based on rolling window of 1 min
-        now = time.time()
-        # remove requests older than 60 sec
-        while self.requests_timestamps and self.requests_timestamps[0] < (now - 60):
-            self.requests_timestamps.popleft()
-        rpm = len(self.requests_timestamps)
-        return {
-            "rpm": rpm,
-            "total_requests": self.total_requests,
-        }
 
 
     @abstractmethod

{zenx-0.6.3 → zenx-0.6.5}/zenx/utils.py

@@ -2,6 +2,8 @@ import time
 from typing import Dict
 import functools
 
+from zenx.clients.http import HttpClient, Response
+
 
 def get_time() -> int:
     """ current unix time in milliseconds """
@@ -17,3 +19,13 @@ def log_processing_time(func):
         self.logger.info("processed", id=item['_id'], time_ms=processed_time, pipeline=self.name)
         return result
     return wrapper
+
+
+def record_request(func):
+    @functools.wraps(func)
+    async def wrapper(self: HttpClient, *args, **kwargs) -> Response:
+        result = await func(self, *args, **kwargs)
+        self._requests_timestamps.append(time.time())
+        self._total_requests +=1
+        return result
+    return wrapper

{zenx-0.6.3 → zenx-0.6.5}/zenx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zenx
-Version: 0.6.3
+Version: 0.6.5
 Summary: mini-framework
 Requires-Python: >=3.12
 Requires-Dist: curl-cffi>=0.12.0