zenx 0.6.8__py3-none-any.whl → 0.7.0__py3-none-any.whl

zenx/clients/database.py

@@ -45,22 +45,7 @@ class DBClient(ABC):
 
 
     @abstractmethod
-    async def insert(self, id: str, spider_name: str) -> None:
-        ...
-
-
-    @abstractmethod
-    async def exists(self, id: str, spider_name: str) -> bool:
-        ...
-
-
-    @abstractmethod
-    async def delete(self, id: str, spider_name: str) -> None:
-        ...
-
-
-    @abstractmethod
-    async def cleanup(self, days: int) -> None:
+    async def insert(self, id: str, spider_name: str) -> bool:
         ...
 
 
@@ -87,25 +72,14 @@ class MemoryDB(DBClient):
         pass
 
 
-    async def insert(self, id: str, spider_name: str) -> None:
+    async def insert(self, id: str, spider_name: str) -> bool:
         unique_id = f"{spider_name}_{id}"
+        if unique_id in self.dq:
+            self.logger.debug("exists", id=unique_id, db=self.name)
+            return False
         self.dq.append(unique_id)
         self.logger.debug("inserted", id=unique_id, db=self.name)
-
-
-    async def exists(self, id: str, spider_name: str) -> bool:
-        unique_id = f"{spider_name}_{id}"
-        return unique_id in self.dq
-
-
-    async def delete(self, id: str, spider_name: str) -> None:
-        unique_id = f"{spider_name}_{id}"
-        self.dq.remove(unique_id)
-        self.logger.debug("deleted", id=unique_id, db=self.name)
-
-
-    async def cleanup(self, days: int) -> None:
-        pass
+        return True
 
 
     async def close(self) -> None:
@@ -123,7 +97,7 @@ try:
         def __init__(self, logger: BoundLogger, settings: Settings) -> None:
             super().__init__(logger, settings)
             self.r: Optional[redis.Redis] = None
-            self._processed_ids_zset: str = "processed:ids:zset"
+            self._record_expiry_sec = self.settings.REDIS_RECORD_EXPIRY_SECONDS
 
     
         async def start(self) -> None:
@@ -151,28 +125,15 @@ try:
             self.logger.info("connected", db=self.name)
 
 
-        async def insert(self, id: str, spider_name: str) -> None:
-            unique_id = f"{spider_name}_{id}"
-            timestamp = int(time.time())
-            await self.r.zadd(self._processed_ids_zset, {unique_id: timestamp})
-            self.logger.debug("inserted", id=unique_id, db=self.name)
-
-
-        async def exists(self, id: str, spider_name: str) -> bool:
+        async def insert(self, id: str, spider_name: str) -> bool:
             unique_id = f"{spider_name}_{id}"
-            score = await self.r.zscore(self._processed_ids_zset, unique_id)
-            return score is not None
-
-
-        async def delete(self, id: str, spider_name: str) -> None:
-            unique_id = f"{spider_name}_{id}"
-            await self.r.delete(unique_id)
-            self.logger.debug("deleted", id=unique_id, db=self.name)
-
-
-        async def cleanup(self, days: int) -> None:
-            threshold_timestamp = int(time.time()) - (days * 24 * 60 * 60)
-            await self.r.zremrangebyscore(self._processed_ids_zset, 0, threshold_timestamp)
+            result = await self.r.set(unique_id, 1, ex=self._record_expiry_sec, nx=True)
+            if result:
+                self.logger.debug("inserted", id=unique_id, db=self.name)
+                return True
+            else:
+                self.logger.debug("exists", id=unique_id, db=self.name)
+                return False
 
 
         async def close(self) -> None:
@@ -196,8 +157,5 @@ except ModuleNotFoundError:
         
         async def start(self) -> None: pass
         async def _connect(self) -> None: pass
-        async def insert(self, id: str, spider_name: str) -> None: pass
-        async def exists(self, id: str, spider_name: str) -> bool: return False
-        async def delete(self, id: str, spider_name: str) -> None: pass
-        async def cleanup(self, days: int) -> None: pass
+        async def insert(self, id: str, spider_name: str) -> bool: return False
         async def close(self) -> None: pass

zenx/clients/http.py

@@ -1,7 +1,4 @@
 from __future__ import annotations
-import time
-import functools
-from collections import deque
 import parsel
 import random
 from curl_cffi.requests.impersonate import BrowserTypeLiteral
@@ -35,16 +32,6 @@ class Response:
         return sel
 
 
-def record_request(func):
-    @functools.wraps(func)
-    async def wrapper(self: HttpClient, *args, **kwargs) -> Response:
-        result = await func(self, *args, **kwargs)
-        self._requests_timestamps.append(time.time())
-        self._total_requests +=1
-        return result
-    return wrapper
-
-
 class HttpClient(ABC):
     # central registry
     name: ClassVar[str]
@@ -69,23 +56,7 @@ class HttpClient(ABC):
         self.logger = logger
         self.settings = settings
         self._session_pool: asyncio.Queue
-        # stats
-        self._requests_timestamps = deque()
-        self._total_requests = 0
     
-
-    def get_stats(self) -> Dict:
-        # calculate RPM based on rolling window of 1 min
-        now = time.time()
-        # remove requests older than 60 sec
-        while self._requests_timestamps and self._requests_timestamps[0] < (now - 60):
-            self._requests_timestamps.popleft()
-        rpm = len(self._requests_timestamps)
-        return {
-            "rpm": rpm,
-            "total_requests": self._total_requests,
-        }
-
     
     @abstractmethod
     async def request(
@@ -125,7 +96,6 @@ class CurlCffi(HttpClient):
         return chosen_fingerprint
 
     
-    @record_request
     async def request(
         self,
         url: str,

zenx/engine.py

@@ -26,16 +26,6 @@ class Engine:
         self.shutdown_event.set()
 
 
-    async def _log_stats(self, spider: Spider) -> None:
-        while True:
-            await asyncio.sleep(60)
-            try:
-                rpm = spider.client.get_stats().get("rpm")
-                spider.logger.info("stats", rpm=rpm)
-            except asyncio.CancelledError:
-                break
-
-
     async def _execute(self, spider_name: str) -> None:
         loop = asyncio.get_running_loop()
         loop.add_signal_handler(signal.SIGINT, self._shutdown_handler)
@@ -57,8 +47,6 @@ class Engine:
         await pm.start_pipelines()
 
         spider = spider_cls(client=client, pm=pm, logger=logger, settings=settings)
-        
-        stats_task = asyncio.create_task(self._log_stats(spider))
         try:
             if self.forever:
                 while not self.shutdown_event.is_set():
@@ -72,15 +60,6 @@ class Engine:
         finally:
             if self.shutdown_event.is_set():
                 logger.info("shutdown", spider=spider_name)
-
-            stats_task.cancel()
-            try:
-                await stats_task
-            except asyncio.CancelledError:
-                pass
-            stats = spider.client.get_stats()
-            logger.info("summary", **stats)
-
             await client.close()
             await db.close()
             await pm.close_pipelines()

zenx/pipelines/manager.py

@@ -1,3 +1,4 @@
+import asyncio
 from typing import Dict, List
 from structlog import BoundLogger
 
@@ -12,29 +13,32 @@ class PipelineManager:
 
     def __init__(self, pipeline_names: List[str], logger: BoundLogger, db: DBClient, settings: Settings) -> None:
         self.logger = logger
-        self.pipelines = [Pipeline.get_pipeline(name)(logger, db, settings) for name in pipeline_names]
+        self.pipelines = {name:Pipeline.get_pipeline(name)(logger, db, settings) for name in pipeline_names}
         self.settings = settings
+        self._fire_and_forget_pipelines = [p for p in self.pipelines.values() if p.name != "preprocess"]
     
     
     async def start_pipelines(self) -> None:
         """ connect and monitor """
-        for pipeline in self.pipelines:
+        for pipeline in self.pipelines.values():
             await pipeline.start()
             
 
-    async def process_item(self, item: Dict, spider: str) -> Dict:
-        for pipeline in self.pipelines:
+    async def process_item(self, item: Dict, spider: str) -> None:
+        preprocess_pipeline = self.pipelines.get("preprocess")
+        if preprocess_pipeline:
             try:
-                item = await pipeline.process_item(item, spider)
+                item = await preprocess_pipeline.process_item(item, spider)
             except DropItem:
-                self.logger.debug("dropped", id=item.get("_id"))
-                break
+                self.logger.debug("dropped", id=item.get("_id"), pipeline=preprocess_pipeline.name)
+                return
             except Exception:
-                self.logger.exception("process_item", pipeline=pipeline.name, item=item)
+                self.logger.exception("process_item", item=item, pipeline=preprocess_pipeline.name)
                 raise
-        return item
+        for pipeline in self._fire_and_forget_pipelines:
+            asyncio.create_task(pipeline.process_item(item, spider))
 
     
     async def close_pipelines(self) -> None:
-        for pipeline in self.pipelines:
+        for pipeline in self.pipelines.values():
             await pipeline.close()

zenx/settings.py

@@ -9,6 +9,7 @@ class Settings(BaseSettings):
     SESSION_POOL_SIZE: int = 1
     MAX_SCRAPE_DELAY: int = 10  # 10 seconds
     DQ_MAX_SIZE: int = 100  # max size of the deque for memory database
+    REDIS_RECORD_EXPIRY_SECONDS: int = 604800 # 7 days (7*24*60*60)
 
     DB_TYPE: Literal["memory", "redis"] = "memory"
     DB_NAME: str | None = None

{zenx-0.6.8.dist-info → zenx-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zenx
-Version: 0.6.8
+Version: 0.7.0
 Summary: mini-framework
 Requires-Python: >=3.12
 Requires-Dist: curl-cffi>=0.12.0

{zenx-0.6.8.dist-info → zenx-0.7.0.dist-info}/RECORD

@@ -1,18 +1,18 @@
 zenx/cli.py,sha256=pHKhOTdqI6NQQoYK91waRIMpxCXLYtXEryzVaTbmvqc,2810
 zenx/debug_runner.py,sha256=B2Jd9A4_EHDa-ohLcwnFIxOV73FagTWXX2fl3qgwlpY,809
 zenx/discovery.py,sha256=YANVGzy2IG1fYruUud-11Y-ynyO6iEp3EjlHnhIQJQI,1014
-zenx/engine.py,sha256=csnBhGf6lfGY3b_7I5AlH_fPGPN9mu1iEiXbJ9ZNxiw,2962
+zenx/engine.py,sha256=Nsz59CnHjUmlYa3dcqdRCBGZh8ZBHDht-rrZTKd1ysU,2341
 zenx/exceptions.py,sha256=BJXxzwwX2CU6inhppfblx8c8Z6Mhvsk7MAhQ1LAnhBg,37
 zenx/logger.py,sha256=UmEk0vV1mSCozV7z_DDgCCXdAManDr5wgkrhKiRQtyU,1651
-zenx/settings.py,sha256=Q0z3oGyVkhucRyqsraZQR1lAPSq1sjiGKmAkDCmQsA0,940
+zenx/settings.py,sha256=0RLeKcqHJdD1vuYIQtGeXkp2YIvv3tNyz9O8jw620VQ,1008
 zenx/utils.py,sha256=a0JOM4CRQHFCD-W1E8P7voIWmdEbSYXN3yV4db8ni3U,588
 zenx/clients/__init__.py,sha256=CaAAuNa8DPyMdejR0KNSDDg_UzC3WxaTol5_QvwwwG8,132
-zenx/clients/database.py,sha256=GaIafQ2OkYJQ_sz2f6u7sZKn77VOReWun09hogk6dO0,6019
-zenx/clients/http.py,sha256=4n2y1Vsij7ELmD54DoMs1i53HIEEbVtSo2l1Dnz7P_M,6621
+zenx/clients/database.py,sha256=9QW2H-4vscoBKZy_iG_q8WvSExoD0U7Rih4FPsYngbI,4756
+zenx/clients/http.py,sha256=fb6COYot6vidNFRBWgoU6CYEfnYWJP0JuVkydvxsHb4,5700
 zenx/pipelines/__init__.py,sha256=IxkZ0UpEJdYjLdd-PMcC9PzzzArTBNNcpgKc7NiOe5Y,131
 zenx/pipelines/base.py,sha256=N_388z5DFMeaU6wMwcClZAbQFWKh4kpAF7eUJhpQevs,1863
 zenx/pipelines/google_rpc.py,sha256=F7p5ml9W1UliZbrDrF9MFNVKlCP5pG1WpO6rdmBgKp8,4707
-zenx/pipelines/manager.py,sha256=LRejhnSiXospa4cMSX9Srp0jfscGnHDggXyl1aUNhtg,1300
+zenx/pipelines/manager.py,sha256=bP2WIMblbWYHOJPU44l_IF3xLD0eHBYV4CCTFZc2FHI,1662
 zenx/pipelines/preprocess.py,sha256=k1Ake8H3nm_S_0ZnruyXAl7cFlBhLn-wpQZHuovIH4o,774
 zenx/pipelines/websocket.py,sha256=74XIV7_YbBngCgqKP3ZiBpDfQte-VB2G4kjNsZyycIc,4456
 zenx/resources/proto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -20,8 +20,8 @@ zenx/resources/proto/feed_pb2.py,sha256=ZyICOLnyuXekkvV4bAHZ1nE1-wwzcYYRRrmRJCMr
 zenx/resources/proto/feed_pb2_grpc.py,sha256=Mim6FfBgIMj0PmTqHk036nVUMJH3A6I3ts6r1j3bQF8,7441
 zenx/spiders/__init__.py,sha256=rs5LuqdM2MQlUYiTGJrzkYhzN8_SSLTrR7wGjSRrrSo,25
 zenx/spiders/base.py,sha256=YB-KqsAzfIUTzDMy5_ElgW1mul-I4Ltft6JAJxpy4hg,1672
-zenx-0.6.8.dist-info/METADATA,sha256=6dvQwWjNmQWLfwTsyMiXAL4qHD6yvCoJc4gkVMfku5o,1273
-zenx-0.6.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-zenx-0.6.8.dist-info/entry_points.txt,sha256=8JXob2f1VtvzGFris-e9Usqywg7oca-cChDlH9moOZU,38
-zenx-0.6.8.dist-info/top_level.txt,sha256=JeXwvK86d7sB-2x-avugFnZIZa33zaHWKI8RHWJR6KY,5
-zenx-0.6.8.dist-info/RECORD,,
+zenx-0.7.0.dist-info/METADATA,sha256=aMtFlE8YY8qgo-DPOg0xz4igPppYrBBro8yXYxcnI5I,1273
+zenx-0.7.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+zenx-0.7.0.dist-info/entry_points.txt,sha256=8JXob2f1VtvzGFris-e9Usqywg7oca-cChDlH9moOZU,38
+zenx-0.7.0.dist-info/top_level.txt,sha256=JeXwvK86d7sB-2x-avugFnZIZa33zaHWKI8RHWJR6KY,5
+zenx-0.7.0.dist-info/RECORD,,