zenx 0.5.1__tar.gz → 0.5.3__tar.gz

{zenx-0.5.1 → zenx-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zenx
-Version: 0.5.1
+Version: 0.5.3
 Summary: mini-framework
 Requires-Python: >=3.12
 Requires-Dist: curl-cffi>=0.12.0

{zenx-0.5.1 → zenx-0.5.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "zenx"
-version = "0.5.1"
+version = "0.5.3"
 description = "mini-framework"
 requires-python = ">=3.12"
 dependencies = [

{zenx-0.5.1 → zenx-0.5.3}/zenx/clients/http.py

@@ -20,7 +20,7 @@ class Response:
     status: int
     text: str
     headers: Dict
-    received_at: int
+    responded_at: int
     requested_at: int
     latency_ms: int
 
@@ -124,7 +124,7 @@ class CurlCffi(HttpClient):
                     )
                     recv_at = get_time()
                     latency = recv_at - req_at
-                    self.logger.debug("response", status=response.status_code, url=url, impersonate=impersonate, client=self.name, requested_at=req_at, received_at=recv_at, latency_ms=latency)
+                    self.logger.debug("response", status=response.status_code, url=url, impersonate=impersonate, client=self.name, requested_at=req_at, responded_at=recv_at, latency_ms=latency)
                 except Exception:
                     self.logger.exception("request", url=url, client=self.name)
                     raise
@@ -144,7 +144,7 @@ class CurlCffi(HttpClient):
                 )
                 recv_at = get_time()
                 latency = recv_at - req_at
-                self.logger.debug("response", status=response.status_code, url=url, impersonate=impersonate, client=self.name, requested_at=req_at, received_at=recv_at, latency_ms=latency)
+                self.logger.debug("response", status=response.status_code, url=url, impersonate=impersonate, client=self.name, requested_at=req_at, responded_at=recv_at, latency_ms=latency)
             except Exception:
                 self.logger.exception("request", url=url, client=self.name)
                 raise
@@ -157,7 +157,7 @@ class CurlCffi(HttpClient):
             text=response.text,
             headers=dict(response.headers),
             requested_at=req_at,
-            received_at=recv_at,
+            responded_at=recv_at,
             latency_ms=latency,
         )
     

{zenx-0.5.1 → zenx-0.5.3}/zenx/pipelines/google_rpc.py

@@ -7,6 +7,7 @@ import structlog
 from zenx.pipelines.base import Pipeline
 from zenx.clients.database import DBClient
 from zenx.settings import Settings
+from zenx.utils import log_processing_time
 
 
 try:
@@ -63,7 +64,8 @@ try:
             self.logger.debug("connected", pipeline=self.name)
             self._connected.set()
 
-
+        
+        @log_processing_time
         async def process_item(self, item: Dict, spider: str) -> Dict:
             await self._process(item)
             return item

{zenx-0.5.1 → zenx-0.5.3}/zenx/pipelines/preprocess.py

@@ -21,8 +21,9 @@ class PreprocessPipeline(Pipeline):
             if await self.db.exists(_id, spider):
                 raise DropItem
             await self.db.insert(_id, spider)
-        scraped_time = item['scraped_at'] - item['received_at']
-        self.logger.info("scraped", id=item.get("_id"), pipeline=self.name, time_ms=scraped_time)
+
+        scraped_time = item['scraped_at'] - item['responded_at']
+        self.logger.info("scraped", id=item.get("_id"), time_ms=scraped_time, pipeline=self.name)
         return item
     
 

{zenx-0.5.1 → zenx-0.5.3}/zenx/pipelines/websocket.py

@@ -7,6 +7,7 @@ from websockets import ConnectionClosed
 from zenx.clients.database import DBClient
 from zenx.settings import Settings
 from zenx.pipelines.base import Pipeline
+from zenx.utils import get_time, log_processing_time
 
 
 try:
@@ -58,7 +59,8 @@ try:
             self.logger.debug("connected", pipeline=self.name, msg=msg)
             self._connected.set()
 
-
+            
+        @log_processing_time
         async def process_item(self, item: Dict, spider: str) -> Dict:
             await self._process(item, spider)
             return item
@@ -80,8 +82,6 @@ try:
                 await self._ws_client.send(json.dumps(_item))
             except ConnectionClosed as e:
                 self.logger.error("processing", exception=str(e), id=item.get("_id"), pipeline=self.name)
-            else:
-                self.logger.debug("processed", id=item['_id'], pipeline=self.name)
             
 
         async def close(self) -> None:

zenx-0.5.3/zenx/utils.py

@@ -0,0 +1,19 @@
+import time
+from typing import Dict
+import functools
+
+
+def get_time() -> int:
+    """ current unix time in milliseconds """
+    return int(time.time() * 1000)
+
+
+def log_processing_time(func):
+    @functools.wraps(func)
+    async def wrapper(self, item: Dict, *args, **kwargs) -> Dict:
+        start_time = get_time()
+        result = await func(item, *args, **kwargs)
+        processed_time = get_time() - start_time    
+        self.logger.debug("processed", id=item['_id'], time_ms=processed_time, pipeline=self.name)
+        return result
+    return wrapper

{zenx-0.5.1 → zenx-0.5.3}/zenx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zenx
-Version: 0.5.1
+Version: 0.5.3
 Summary: mini-framework
 Requires-Python: >=3.12
 Requires-Dist: curl-cffi>=0.12.0

zenx-0.5.1/zenx/utils.py

@@ -1,6 +0,0 @@
-import time
-
-
-def get_time() -> int:
-    """ current unix time in milliseconds """
-    return int(time.time() * 1000)