zenx-lite-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenx_lite/__init__.py +8 -0
- zenx_lite/app.py +377 -0
- zenx_lite/config.py +56 -0
- zenx_lite/configuration.py +359 -0
- zenx_lite/crawler.py +69 -0
- zenx_lite/delivery.py +139 -0
- zenx_lite/destinations/__init__.py +5 -0
- zenx_lite/destinations/base.py +32 -0
- zenx_lite/destinations/jsonl.py +132 -0
- zenx_lite/destinations/synoptic/__init__.py +5 -0
- zenx_lite/destinations/synoptic/source.py +136 -0
- zenx_lite/errors.py +21 -0
- zenx_lite/http.py +130 -0
- zenx_lite/notifiers/__init__.py +5 -0
- zenx_lite/notifiers/base.py +15 -0
- zenx_lite/notifiers/magent.py +86 -0
- zenx_lite/scheduler.py +63 -0
- zenx_lite/sessions.py +102 -0
- zenx_lite/store.py +45 -0
- zenx_lite/stores/__init__.py +1 -0
- zenx_lite/stores/redis.py +54 -0
- zenx_lite_python-0.1.0.dist-info/METADATA +14 -0
- zenx_lite_python-0.1.0.dist-info/RECORD +24 -0
- zenx_lite_python-0.1.0.dist-info/WHEEL +4 -0
zenx_lite/sessions.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Worker-retained sessions with fixed creation age and explicit retirement."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import time
|
|
5
|
+
from collections.abc import Mapping, Sequence
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from typing import Any, Protocol
|
|
8
|
+
|
|
9
|
+
from curl_cffi import AsyncSession
|
|
10
|
+
|
|
11
|
+
from .errors import ConfigurationError, SessionUnavailable
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(slots=True)
|
|
15
|
+
class Session:
|
|
16
|
+
transport: AsyncSession
|
|
17
|
+
created_at: float = field(default_factory=time.monotonic)
|
|
18
|
+
retired: bool = False
|
|
19
|
+
|
|
20
|
+
def require_ready(self) -> None:
|
|
21
|
+
if self.retired:
|
|
22
|
+
raise SessionUnavailable("The worker's session has retired; replacement occurs next crawl")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class SessionManager(Protocol):
|
|
26
|
+
async def open(self, *, concurrency: int, proxy: str | None) -> None: ...
|
|
27
|
+
|
|
28
|
+
async def acquire(self, worker_id: int) -> Session | None: ...
|
|
29
|
+
|
|
30
|
+
async def release(self, worker_id: int, session: Session) -> None: ...
|
|
31
|
+
|
|
32
|
+
async def close(self) -> None: ...
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Sessions:
|
|
36
|
+
"""Locally provision ordinary curl sessions; no solver or warmup dependency."""
|
|
37
|
+
|
|
38
|
+
def __init__(
|
|
39
|
+
self, *, options: Mapping[str, Any] | None = None,
|
|
40
|
+
proxies: Sequence[str] = (),
|
|
41
|
+
) -> None:
|
|
42
|
+
self._options = dict(options or {})
|
|
43
|
+
# A transport-level retry would violate the framework's request contract.
|
|
44
|
+
if self._options.get("retry", 0) != 0:
|
|
45
|
+
raise ConfigurationError("Managed sessions do not retry requests")
|
|
46
|
+
self._options["retry"] = 0
|
|
47
|
+
self._options.setdefault("impersonate", "chrome")
|
|
48
|
+
if isinstance(proxies, str) or any(not isinstance(p, str) or not p for p in proxies):
|
|
49
|
+
raise ConfigurationError("proxies must be a sequence of proxy URLs")
|
|
50
|
+
self._proxies = tuple(proxies)
|
|
51
|
+
self._next_proxy = 0
|
|
52
|
+
self._proxy: str | None = None
|
|
53
|
+
self._workers: dict[int, Session] = {}
|
|
54
|
+
|
|
55
|
+
async def open(self, *, concurrency: int, proxy: str | None) -> None:
|
|
56
|
+
self._proxy = proxy
|
|
57
|
+
self._next_proxy = 0
|
|
58
|
+
for worker_id in range(concurrency):
|
|
59
|
+
self._workers[worker_id] = self._create()
|
|
60
|
+
|
|
61
|
+
def _create(self) -> Session:
|
|
62
|
+
options = self._options.copy()
|
|
63
|
+
proxy = self._proxy
|
|
64
|
+
if proxy is None and self._proxies:
|
|
65
|
+
proxy = self._proxies[self._next_proxy % len(self._proxies)]
|
|
66
|
+
self._next_proxy += 1
|
|
67
|
+
if proxy is not None:
|
|
68
|
+
options.pop("proxies", None)
|
|
69
|
+
options["proxy"] = proxy
|
|
70
|
+
return Session(AsyncSession(**options))
|
|
71
|
+
|
|
72
|
+
async def acquire(self, worker_id: int) -> Session:
|
|
73
|
+
session = self._workers.get(worker_id)
|
|
74
|
+
if session is not None and (session.retired or time.monotonic() - session.created_at >= 600):
|
|
75
|
+
session.retired = True
|
|
76
|
+
await session.transport.close()
|
|
77
|
+
del self._workers[worker_id]
|
|
78
|
+
session = None
|
|
79
|
+
if session is None:
|
|
80
|
+
try:
|
|
81
|
+
session = self._create()
|
|
82
|
+
except Exception as exc:
|
|
83
|
+
raise SessionUnavailable("Could not create a replacement session") from exc
|
|
84
|
+
self._workers[worker_id] = session
|
|
85
|
+
return session
|
|
86
|
+
|
|
87
|
+
async def release(self, worker_id: int, session: Session) -> None:
|
|
88
|
+
if session.retired:
|
|
89
|
+
await session.transport.close()
|
|
90
|
+
if self._workers.get(worker_id) is session:
|
|
91
|
+
del self._workers[worker_id]
|
|
92
|
+
|
|
93
|
+
async def close(self) -> None:
|
|
94
|
+
sessions = list(self._workers.values())
|
|
95
|
+
for session in sessions:
|
|
96
|
+
session.retired = True
|
|
97
|
+
try:
|
|
98
|
+
async with asyncio.TaskGroup() as tasks:
|
|
99
|
+
for session in sessions:
|
|
100
|
+
tasks.create_task(session.transport.close())
|
|
101
|
+
finally:
|
|
102
|
+
self._workers.clear()
|
zenx_lite/store.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Atomic item reservation; store keys never alter business payloads."""
|
|
2
|
+
|
|
3
|
+
from typing import Protocol
|
|
4
|
+
|
|
5
|
+
from .errors import ConfigurationError
|
|
6
|
+
|
|
7
|
+
# (project_name, dedup_scope, item_id); scope defaults to the selected spider name.
|
|
8
|
+
type Key = tuple[str, str, str]
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Store(Protocol):
|
|
12
|
+
async def open(self) -> None: ...
|
|
13
|
+
|
|
14
|
+
async def add(self, key: Key) -> bool:
|
|
15
|
+
"""Atomically reserve a key; return False if it is already remembered.
|
|
16
|
+
|
|
17
|
+
Storage failures must raise, never silently switch to a weaker store.
|
|
18
|
+
A failed or cancelled call may already have reserved the key.
|
|
19
|
+
"""
|
|
20
|
+
...
|
|
21
|
+
|
|
22
|
+
async def close(self) -> None: ...
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class MemoryStore:
|
|
26
|
+
def __init__(self, capacity: int = 500) -> None:
|
|
27
|
+
if type(capacity) is not int or capacity <= 0:
|
|
28
|
+
raise ConfigurationError("memory store capacity must be a positive integer")
|
|
29
|
+
self.capacity = capacity
|
|
30
|
+
self._keys: dict[Key, None] = {}
|
|
31
|
+
|
|
32
|
+
async def open(self) -> None:
|
|
33
|
+
self._keys.clear()
|
|
34
|
+
|
|
35
|
+
async def add(self, key: Key) -> bool:
|
|
36
|
+
# No suspension between checking and inserting: atomic on this event loop.
|
|
37
|
+
if key in self._keys:
|
|
38
|
+
return False
|
|
39
|
+
if len(self._keys) == self.capacity:
|
|
40
|
+
del self._keys[next(iter(self._keys))]
|
|
41
|
+
self._keys[key] = None
|
|
42
|
+
return True
|
|
43
|
+
|
|
44
|
+
async def close(self) -> None:
|
|
45
|
+
self._keys.clear()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Optional persistent stores; import the selected backend explicitly."""
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Shared item reservations that survive application restarts."""
|
|
2
|
+
|
|
3
|
+
import orjson
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
from redis.asyncio import Redis
|
|
7
|
+
from redis.asyncio.retry import Retry
|
|
8
|
+
from redis.backoff import NoBackoff
|
|
9
|
+
except ImportError as exc:
|
|
10
|
+
raise ImportError("RedisStore requires zenx-lite-python's optional 'redis' extra") from exc
|
|
11
|
+
|
|
12
|
+
from ..errors import ConfigurationError
|
|
13
|
+
from ..store import Key
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class RedisStore:
|
|
17
|
+
"""Reserve IDs with SET NX, optionally expiring from their first admission.
|
|
18
|
+
|
|
19
|
+
ttl_seconds=None retains IDs until Redis removes them. Server persistence
|
|
20
|
+
and eviction policy determine durability; closing this store deletes nothing.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(self, url: str, *, ttl_seconds: int | None) -> None:
|
|
24
|
+
if not isinstance(url, str) or not url.startswith(("redis://", "rediss://", "unix://")):
|
|
25
|
+
raise ConfigurationError("Redis URL must use redis://, rediss://, or unix://")
|
|
26
|
+
if ttl_seconds is not None and (type(ttl_seconds) is not int or ttl_seconds <= 0):
|
|
27
|
+
raise ConfigurationError("ttl_seconds must be a positive integer or None")
|
|
28
|
+
self._url = url
|
|
29
|
+
self._ttl_seconds = ttl_seconds
|
|
30
|
+
self._client: Redis | None = None
|
|
31
|
+
|
|
32
|
+
async def open(self) -> None:
|
|
33
|
+
if self._client is not None:
|
|
34
|
+
raise RuntimeError("RedisStore is already open; close it before reopening")
|
|
35
|
+
# Retain the client before I/O so runtime cleanup covers partial startup.
|
|
36
|
+
self._client = Redis.from_url(
|
|
37
|
+
self._url, socket_connect_timeout=5.0, socket_timeout=5.0,
|
|
38
|
+
)
|
|
39
|
+
# Set after URL parsing so URL options cannot re-enable command retries.
|
|
40
|
+
# A lost SET reply may mean the ID was reserved; never retry that attempt.
|
|
41
|
+
self._client.set_retry(Retry(NoBackoff(), 0))
|
|
42
|
+
await self._client.ping()
|
|
43
|
+
|
|
44
|
+
async def add(self, key: Key) -> bool:
|
|
45
|
+
if self._client is None:
|
|
46
|
+
raise RuntimeError("RedisStore is not open")
|
|
47
|
+
# JSON framing preserves tuple boundaries even with separators in IDs.
|
|
48
|
+
redis_key = b"zenx_lite:dedup:v1:" + orjson.dumps(key)
|
|
49
|
+
return bool(await self._client.set(redis_key, b"1", nx=True, ex=self._ttl_seconds))
|
|
50
|
+
|
|
51
|
+
async def close(self) -> None:
|
|
52
|
+
client, self._client = self._client, None
|
|
53
|
+
if client is not None:
|
|
54
|
+
await client.aclose()
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: zenx-lite-python
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An explicitly composed asyncio scraping runtime
|
|
5
|
+
Requires-Python: >=3.15
|
|
6
|
+
Requires-Dist: curl-cffi-patch<0.17,>=0.16.4
|
|
7
|
+
Requires-Dist: orjson>=3.10
|
|
8
|
+
Requires-Dist: parsel>=1.12.1
|
|
9
|
+
Provides-Extra: config
|
|
10
|
+
Requires-Dist: pyyaml<7,>=6; extra == 'config'
|
|
11
|
+
Provides-Extra: redis
|
|
12
|
+
Requires-Dist: redis<9,>=8.1; extra == 'redis'
|
|
13
|
+
Provides-Extra: source
|
|
14
|
+
Requires-Dist: websockets<18,>=15; extra == 'source'
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
zenx_lite/__init__.py,sha256=8wh-jIDOvP9WRgpWdyGxqCe0rfTlsD2YGD6Ru17kcmE,262
|
|
2
|
+
zenx_lite/app.py,sha256=EE80-5rUf7ei1Qi3fteH1jSBn0eTMgFFIxCwKUaoWtE,17220
|
|
3
|
+
zenx_lite/config.py,sha256=6aoNxv9anB_8ufPifQlutDZxpop6Ya2GCYg1MhQWI58,2523
|
|
4
|
+
zenx_lite/configuration.py,sha256=wMfSHptneAkQJc0A9oAtVt8NQu7iP3yLZDcdw46_nKg,17694
|
|
5
|
+
zenx_lite/crawler.py,sha256=4MPQMGxt-2K9rWlvExLBmsnE-8x63MgJzdXd0q3d4OA,2157
|
|
6
|
+
zenx_lite/delivery.py,sha256=1vrQE61W4QsAQTUo7T4UkuXsR8wxt5KfssI1MgzslYs,5204
|
|
7
|
+
zenx_lite/errors.py,sha256=LZPRdvIEdEU32-JJAt0ozFcCwczcCaLTuiq_2arxors,609
|
|
8
|
+
zenx_lite/http.py,sha256=-kr7BXc4O1-UZoWafdqkmBVbCjb_0DIwm4g7_aKCM2c,4735
|
|
9
|
+
zenx_lite/scheduler.py,sha256=Q8tCHRETyLwOfT8Aas3DUl-utEMcrwQIOFigV9UZIZk,2494
|
|
10
|
+
zenx_lite/sessions.py,sha256=LAQz7b0fbl22Qf9bm74IsnjUu3f58Zcru9nnqXmVTAk,3747
|
|
11
|
+
zenx_lite/store.py,sha256=PrIhOa9qNYpgSBXKU_L9aURrZTZsaJlffF7Q4cNOrys,1404
|
|
12
|
+
zenx_lite/destinations/__init__.py,sha256=8QHMWBLDvpCnw8m_8rsI7dNhwAI4bA5QSb9gk6YGmQE,136
|
|
13
|
+
zenx_lite/destinations/base.py,sha256=BxWdIEniN6gxhyC1B9-FISf7hCINpTOdPIv1Bpy_Wcs,1041
|
|
14
|
+
zenx_lite/destinations/jsonl.py,sha256=UaMo7g-_gHBJiOJhvSErHiCdstrXFsG43YAdsEtPbwQ,5130
|
|
15
|
+
zenx_lite/destinations/synoptic/__init__.py,sha256=7db99zN4Zr-NYd0U40r_VgfKVFrxNrAhaEP_FpCN8JE,144
|
|
16
|
+
zenx_lite/destinations/synoptic/source.py,sha256=tVkg7-4dpnYFUV9kI07K1_VFb-r-KM7y7fe2FPtSr8s,5534
|
|
17
|
+
zenx_lite/notifiers/__init__.py,sha256=78Ubo7SPM4OphlEdoKCEZ0Lw-rKIkFC8VDouow_x9Wc,126
|
|
18
|
+
zenx_lite/notifiers/base.py,sha256=PWWTRc1qLUv69h24eGeyd_lo35X72IbgNTM-z2ZN1vM,389
|
|
19
|
+
zenx_lite/notifiers/magent.py,sha256=FRJiOK2n9ChdattIPC6iTBHW8SgNrv8o7zLwSpZ88XQ,3506
|
|
20
|
+
zenx_lite/stores/__init__.py,sha256=vAK2Yt7RuqHq2CpVefVsGB4kQ__2KrGRRQmrhRy0Sqo,74
|
|
21
|
+
zenx_lite/stores/redis.py,sha256=L0RRg3YRn_fb7YKUYf_ZQN-AvyXpaDeq1xnvwQGgX78,2338
|
|
22
|
+
zenx_lite_python-0.1.0.dist-info/METADATA,sha256=vR7CP0RzY9JaWudwaploe7tqh_rory4g-cYuwUkoNmQ,459
|
|
23
|
+
zenx_lite_python-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
24
|
+
zenx_lite_python-0.1.0.dist-info/RECORD,,
|