zenx-lite-python 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
zenx_lite/sessions.py ADDED
@@ -0,0 +1,102 @@
1
+ """Worker-retained sessions with fixed creation age and explicit retirement."""
2
+
3
+ import asyncio
4
+ import time
5
+ from collections.abc import Mapping, Sequence
6
+ from dataclasses import dataclass, field
7
+ from typing import Any, Protocol
8
+
9
+ from curl_cffi import AsyncSession
10
+
11
+ from .errors import ConfigurationError, SessionUnavailable
12
+
13
+
14
+ @dataclass(slots=True)
15
+ class Session:
16
+ transport: AsyncSession
17
+ created_at: float = field(default_factory=time.monotonic)
18
+ retired: bool = False
19
+
20
+ def require_ready(self) -> None:
21
+ if self.retired:
22
+ raise SessionUnavailable("The worker's session has retired; replacement occurs next crawl")
23
+
24
+
25
+ class SessionManager(Protocol):
26
+ async def open(self, *, concurrency: int, proxy: str | None) -> None: ...
27
+
28
+ async def acquire(self, worker_id: int) -> Session | None: ...
29
+
30
+ async def release(self, worker_id: int, session: Session) -> None: ...
31
+
32
+ async def close(self) -> None: ...
33
+
34
+
35
+ class Sessions:
36
+ """Locally provision ordinary curl sessions; no solver or warmup dependency."""
37
+
38
+ def __init__(
39
+ self, *, options: Mapping[str, Any] | None = None,
40
+ proxies: Sequence[str] = (),
41
+ ) -> None:
42
+ self._options = dict(options or {})
43
+ # A transport-level retry would violate the framework's request contract.
44
+ if self._options.get("retry", 0) != 0:
45
+ raise ConfigurationError("Managed sessions do not retry requests")
46
+ self._options["retry"] = 0
47
+ self._options.setdefault("impersonate", "chrome")
48
+ if isinstance(proxies, str) or any(not isinstance(p, str) or not p for p in proxies):
49
+ raise ConfigurationError("proxies must be a sequence of proxy URLs")
50
+ self._proxies = tuple(proxies)
51
+ self._next_proxy = 0
52
+ self._proxy: str | None = None
53
+ self._workers: dict[int, Session] = {}
54
+
55
+ async def open(self, *, concurrency: int, proxy: str | None) -> None:
56
+ self._proxy = proxy
57
+ self._next_proxy = 0
58
+ for worker_id in range(concurrency):
59
+ self._workers[worker_id] = self._create()
60
+
61
+ def _create(self) -> Session:
62
+ options = self._options.copy()
63
+ proxy = self._proxy
64
+ if proxy is None and self._proxies:
65
+ proxy = self._proxies[self._next_proxy % len(self._proxies)]
66
+ self._next_proxy += 1
67
+ if proxy is not None:
68
+ options.pop("proxies", None)
69
+ options["proxy"] = proxy
70
+ return Session(AsyncSession(**options))
71
+
72
+ async def acquire(self, worker_id: int) -> Session:
73
+ session = self._workers.get(worker_id)
74
+ if session is not None and (session.retired or time.monotonic() - session.created_at >= 600):
75
+ session.retired = True
76
+ await session.transport.close()
77
+ del self._workers[worker_id]
78
+ session = None
79
+ if session is None:
80
+ try:
81
+ session = self._create()
82
+ except Exception as exc:
83
+ raise SessionUnavailable("Could not create a replacement session") from exc
84
+ self._workers[worker_id] = session
85
+ return session
86
+
87
+ async def release(self, worker_id: int, session: Session) -> None:
88
+ if session.retired:
89
+ await session.transport.close()
90
+ if self._workers.get(worker_id) is session:
91
+ del self._workers[worker_id]
92
+
93
+ async def close(self) -> None:
94
+ sessions = list(self._workers.values())
95
+ for session in sessions:
96
+ session.retired = True
97
+ try:
98
+ async with asyncio.TaskGroup() as tasks:
99
+ for session in sessions:
100
+ tasks.create_task(session.transport.close())
101
+ finally:
102
+ self._workers.clear()
zenx_lite/store.py ADDED
@@ -0,0 +1,45 @@
1
+ """Atomic item reservation; store keys never alter business payloads."""
2
+
3
+ from typing import Protocol
4
+
5
+ from .errors import ConfigurationError
6
+
7
+ # (project_name, dedup_scope, item_id); scope defaults to the selected spider name.
8
+ type Key = tuple[str, str, str]
9
+
10
+
11
+ class Store(Protocol):
12
+ async def open(self) -> None: ...
13
+
14
+ async def add(self, key: Key) -> bool:
15
+ """Atomically reserve a key; return False if it is already remembered.
16
+
17
+ Storage failures must raise, never silently switch to a weaker store.
18
+ A failed or cancelled call may already have reserved the key.
19
+ """
20
+ ...
21
+
22
+ async def close(self) -> None: ...
23
+
24
+
25
+ class MemoryStore:
26
+ def __init__(self, capacity: int = 500) -> None:
27
+ if type(capacity) is not int or capacity <= 0:
28
+ raise ConfigurationError("memory store capacity must be a positive integer")
29
+ self.capacity = capacity
30
+ self._keys: dict[Key, None] = {}
31
+
32
+ async def open(self) -> None:
33
+ self._keys.clear()
34
+
35
+ async def add(self, key: Key) -> bool:
36
+ # No suspension between checking and inserting: atomic on this event loop.
37
+ if key in self._keys:
38
+ return False
39
+ if len(self._keys) == self.capacity:
40
+ del self._keys[next(iter(self._keys))]
41
+ self._keys[key] = None
42
+ return True
43
+
44
+ async def close(self) -> None:
45
+ self._keys.clear()
@@ -0,0 +1 @@
1
+ """Optional persistent stores; import the selected backend explicitly."""
@@ -0,0 +1,54 @@
1
+ """Shared item reservations that survive application restarts."""
2
+
3
+ import orjson
4
+
5
+ try:
6
+ from redis.asyncio import Redis
7
+ from redis.asyncio.retry import Retry
8
+ from redis.backoff import NoBackoff
9
+ except ImportError as exc:
10
+ raise ImportError("RedisStore requires zenx-lite-python's optional 'redis' extra") from exc
11
+
12
+ from ..errors import ConfigurationError
13
+ from ..store import Key
14
+
15
+
16
+ class RedisStore:
17
+ """Reserve IDs with SET NX, optionally expiring from their first admission.
18
+
19
+ ttl_seconds=None retains IDs until Redis removes them. Server persistence
20
+ and eviction policy determine durability; closing this store deletes nothing.
21
+ """
22
+
23
+ def __init__(self, url: str, *, ttl_seconds: int | None) -> None:
24
+ if not isinstance(url, str) or not url.startswith(("redis://", "rediss://", "unix://")):
25
+ raise ConfigurationError("Redis URL must use redis://, rediss://, or unix://")
26
+ if ttl_seconds is not None and (type(ttl_seconds) is not int or ttl_seconds <= 0):
27
+ raise ConfigurationError("ttl_seconds must be a positive integer or None")
28
+ self._url = url
29
+ self._ttl_seconds = ttl_seconds
30
+ self._client: Redis | None = None
31
+
32
+ async def open(self) -> None:
33
+ if self._client is not None:
34
+ raise RuntimeError("RedisStore is already open; close it before reopening")
35
+ # Retain the client before I/O so runtime cleanup covers partial startup.
36
+ self._client = Redis.from_url(
37
+ self._url, socket_connect_timeout=5.0, socket_timeout=5.0,
38
+ )
39
+ # Set after URL parsing so URL options cannot re-enable command retries.
40
+ # A lost SET reply may mean the ID was reserved; never retry that attempt.
41
+ self._client.set_retry(Retry(NoBackoff(), 0))
42
+ await self._client.ping()
43
+
44
+ async def add(self, key: Key) -> bool:
45
+ if self._client is None:
46
+ raise RuntimeError("RedisStore is not open")
47
+ # JSON framing preserves tuple boundaries even with separators in IDs.
48
+ redis_key = b"zenx_lite:dedup:v1:" + orjson.dumps(key)
49
+ return bool(await self._client.set(redis_key, b"1", nx=True, ex=self._ttl_seconds))
50
+
51
+ async def close(self) -> None:
52
+ client, self._client = self._client, None
53
+ if client is not None:
54
+ await client.aclose()
@@ -0,0 +1,14 @@
1
+ Metadata-Version: 2.5
2
+ Name: zenx-lite-python
3
+ Version: 0.1.0
4
+ Summary: An explicitly composed asyncio scraping runtime
5
+ Requires-Python: >=3.15
6
+ Requires-Dist: curl-cffi-patch<0.17,>=0.16.4
7
+ Requires-Dist: orjson>=3.10
8
+ Requires-Dist: parsel>=1.12.1
9
+ Provides-Extra: config
10
+ Requires-Dist: pyyaml<7,>=6; extra == 'config'
11
+ Provides-Extra: redis
12
+ Requires-Dist: redis<9,>=8.1; extra == 'redis'
13
+ Provides-Extra: source
14
+ Requires-Dist: websockets<18,>=15; extra == 'source'
@@ -0,0 +1,24 @@
1
+ zenx_lite/__init__.py,sha256=8wh-jIDOvP9WRgpWdyGxqCe0rfTlsD2YGD6Ru17kcmE,262
2
+ zenx_lite/app.py,sha256=EE80-5rUf7ei1Qi3fteH1jSBn0eTMgFFIxCwKUaoWtE,17220
3
+ zenx_lite/config.py,sha256=6aoNxv9anB_8ufPifQlutDZxpop6Ya2GCYg1MhQWI58,2523
4
+ zenx_lite/configuration.py,sha256=wMfSHptneAkQJc0A9oAtVt8NQu7iP3yLZDcdw46_nKg,17694
5
+ zenx_lite/crawler.py,sha256=4MPQMGxt-2K9rWlvExLBmsnE-8x63MgJzdXd0q3d4OA,2157
6
+ zenx_lite/delivery.py,sha256=1vrQE61W4QsAQTUo7T4UkuXsR8wxt5KfssI1MgzslYs,5204
7
+ zenx_lite/errors.py,sha256=LZPRdvIEdEU32-JJAt0ozFcCwczcCaLTuiq_2arxors,609
8
+ zenx_lite/http.py,sha256=-kr7BXc4O1-UZoWafdqkmBVbCjb_0DIwm4g7_aKCM2c,4735
9
+ zenx_lite/scheduler.py,sha256=Q8tCHRETyLwOfT8Aas3DUl-utEMcrwQIOFigV9UZIZk,2494
10
+ zenx_lite/sessions.py,sha256=LAQz7b0fbl22Qf9bm74IsnjUu3f58Zcru9nnqXmVTAk,3747
11
+ zenx_lite/store.py,sha256=PrIhOa9qNYpgSBXKU_L9aURrZTZsaJlffF7Q4cNOrys,1404
12
+ zenx_lite/destinations/__init__.py,sha256=8QHMWBLDvpCnw8m_8rsI7dNhwAI4bA5QSb9gk6YGmQE,136
13
+ zenx_lite/destinations/base.py,sha256=BxWdIEniN6gxhyC1B9-FISf7hCINpTOdPIv1Bpy_Wcs,1041
14
+ zenx_lite/destinations/jsonl.py,sha256=UaMo7g-_gHBJiOJhvSErHiCdstrXFsG43YAdsEtPbwQ,5130
15
+ zenx_lite/destinations/synoptic/__init__.py,sha256=7db99zN4Zr-NYd0U40r_VgfKVFrxNrAhaEP_FpCN8JE,144
16
+ zenx_lite/destinations/synoptic/source.py,sha256=tVkg7-4dpnYFUV9kI07K1_VFb-r-KM7y7fe2FPtSr8s,5534
17
+ zenx_lite/notifiers/__init__.py,sha256=78Ubo7SPM4OphlEdoKCEZ0Lw-rKIkFC8VDouow_x9Wc,126
18
+ zenx_lite/notifiers/base.py,sha256=PWWTRc1qLUv69h24eGeyd_lo35X72IbgNTM-z2ZN1vM,389
19
+ zenx_lite/notifiers/magent.py,sha256=FRJiOK2n9ChdattIPC6iTBHW8SgNrv8o7zLwSpZ88XQ,3506
20
+ zenx_lite/stores/__init__.py,sha256=vAK2Yt7RuqHq2CpVefVsGB4kQ__2KrGRRQmrhRy0Sqo,74
21
+ zenx_lite/stores/redis.py,sha256=L0RRg3YRn_fb7YKUYf_ZQN-AvyXpaDeq1xnvwQGgX78,2338
22
+ zenx_lite_python-0.1.0.dist-info/METADATA,sha256=vR7CP0RzY9JaWudwaploe7tqh_rory4g-cYuwUkoNmQ,459
23
+ zenx_lite_python-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
24
+ zenx_lite_python-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any