zenx-lite-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenx_lite/__init__.py +8 -0
- zenx_lite/app.py +377 -0
- zenx_lite/config.py +56 -0
- zenx_lite/configuration.py +359 -0
- zenx_lite/crawler.py +69 -0
- zenx_lite/delivery.py +139 -0
- zenx_lite/destinations/__init__.py +5 -0
- zenx_lite/destinations/base.py +32 -0
- zenx_lite/destinations/jsonl.py +132 -0
- zenx_lite/destinations/synoptic/__init__.py +5 -0
- zenx_lite/destinations/synoptic/source.py +136 -0
- zenx_lite/errors.py +21 -0
- zenx_lite/http.py +130 -0
- zenx_lite/notifiers/__init__.py +5 -0
- zenx_lite/notifiers/base.py +15 -0
- zenx_lite/notifiers/magent.py +86 -0
- zenx_lite/scheduler.py +63 -0
- zenx_lite/sessions.py +102 -0
- zenx_lite/store.py +45 -0
- zenx_lite/stores/__init__.py +1 -0
- zenx_lite/stores/redis.py +54 -0
- zenx_lite_python-0.1.0.dist-info/METADATA +14 -0
- zenx_lite_python-0.1.0.dist-info/RECORD +24 -0
- zenx_lite_python-0.1.0.dist-info/WHEEL +4 -0
zenx_lite/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""One selected spider, explicitly composed dependencies, best-effort delivery."""
|
|
2
|
+
|
|
3
|
+
from .app import App
|
|
4
|
+
from .config import Settings
|
|
5
|
+
from .crawler import Crawler, Spider
|
|
6
|
+
from .http import Response
|
|
7
|
+
|
|
8
|
+
__all__ = ["App", "Crawler", "Response", "Settings", "Spider"]
|
zenx_lite/app.py
ADDED
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
"""Explicit application composition and a single runtime-owned lifecycle."""
|
|
2
|
+
|
|
3
|
+
lazy import argparse
|
|
4
|
+
import asyncio
|
|
5
|
+
import logging
|
|
6
|
+
import signal
|
|
7
|
+
from collections.abc import Awaitable, Callable, Collection, Mapping, Sequence
|
|
8
|
+
from dataclasses import replace
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from .config import Settings, cli_parser
|
|
12
|
+
from .crawler import Crawler, Spider
|
|
13
|
+
from .delivery import Delivery, DeliveryStats, Sender
|
|
14
|
+
from .errors import ConfigurationError, FatalRuntimeError, SessionUnavailable, WorkerOverlap
|
|
15
|
+
from .notifiers.base import Notifier
|
|
16
|
+
from .scheduler import Scheduler, Slots
|
|
17
|
+
from .sessions import Session, SessionManager, Sessions
|
|
18
|
+
from .destinations.base import Destination
|
|
19
|
+
from .store import MemoryStore, Store
|
|
20
|
+
|
|
21
|
+
ONE_SHOT_GRACE = 60.0
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _consume(task: asyncio.Task[Any]) -> None:
|
|
25
|
+
if not task.cancelled():
|
|
26
|
+
task.exception()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class _Runtime:
|
|
30
|
+
def __init__(
|
|
31
|
+
self, *, project_name: str, spider: Spider, destinations: Mapping[str, Destination],
|
|
32
|
+
settings: Settings, scheduler: Scheduler, store: Store, sessions: SessionManager,
|
|
33
|
+
once: bool, dedup_scope: str, notifier: Notifier | None,
|
|
34
|
+
) -> None:
|
|
35
|
+
self.spider = spider
|
|
36
|
+
self.settings = settings
|
|
37
|
+
self.scheduler = scheduler
|
|
38
|
+
self.store = store
|
|
39
|
+
self.sessions = sessions
|
|
40
|
+
self.once = once
|
|
41
|
+
self.notifier = notifier
|
|
42
|
+
self._notifier_ready = False
|
|
43
|
+
self.stop = asyncio.Event()
|
|
44
|
+
logging.getLogger("zenx_lite").setLevel(settings.log_level.upper())
|
|
45
|
+
self.logger = logging.getLogger(f"zenx_lite.{project_name}.{spider.name}")
|
|
46
|
+
self.logger.setLevel(settings.log_level.upper())
|
|
47
|
+
self.senders = {name: Sender(name, destination, self.logger) for name, destination in destinations.items()}
|
|
48
|
+
self.delivery = Delivery(project_name, dedup_scope, store, self.senders, self.logger)
|
|
49
|
+
self._busy: set[int] = set()
|
|
50
|
+
self._active: set[asyncio.Task[Any]] = set()
|
|
51
|
+
self._closers: list[Callable[[], Awaitable[None]]] = []
|
|
52
|
+
self._closing: list[asyncio.Task[None]] | None = None
|
|
53
|
+
|
|
54
|
+
async def _serve(self) -> None:
|
|
55
|
+
# Register cleanup before opening, including partially opened resources.
|
|
56
|
+
if self.notifier is not None:
|
|
57
|
+
self._closers.append(self.notifier.close)
|
|
58
|
+
try:
|
|
59
|
+
await self.notifier.open()
|
|
60
|
+
except Exception:
|
|
61
|
+
self.logger.exception("Notifier startup failed; continuing without telemetry")
|
|
62
|
+
else:
|
|
63
|
+
self._notifier_ready = True
|
|
64
|
+
self._closers.append(self.store.close)
|
|
65
|
+
await self.store.open()
|
|
66
|
+
self._closers.append(self.sessions.close)
|
|
67
|
+
await self.sessions.open(concurrency=1 if self.once else self.settings.concurrency, proxy=self.settings.proxy)
|
|
68
|
+
for sender in self.senders.values():
|
|
69
|
+
self._closers.append(sender.close)
|
|
70
|
+
await sender.open()
|
|
71
|
+
if self.stop.is_set():
|
|
72
|
+
return
|
|
73
|
+
if self.once:
|
|
74
|
+
await self.crawl(0)
|
|
75
|
+
else:
|
|
76
|
+
await self.scheduler.run(self.crawl, concurrency=self.settings.concurrency, stop=self.stop)
|
|
77
|
+
|
|
78
|
+
def _notify(self) -> None:
|
|
79
|
+
if self.notifier is None or not self._notifier_ready:
|
|
80
|
+
return
|
|
81
|
+
try:
|
|
82
|
+
self.notifier.notify()
|
|
83
|
+
except Exception:
|
|
84
|
+
self.logger.exception("Notifier failed; notification discarded")
|
|
85
|
+
|
|
86
|
+
async def crawl(self, worker_id: int) -> None:
|
|
87
|
+
if type(worker_id) is not int or not 0 <= worker_id < self.settings.concurrency:
|
|
88
|
+
raise FatalRuntimeError("Scheduler supplied an invalid worker ID")
|
|
89
|
+
if worker_id in self._busy:
|
|
90
|
+
raise WorkerOverlap(f"Worker {worker_id} already has an active crawl")
|
|
91
|
+
if self.stop.is_set():
|
|
92
|
+
return
|
|
93
|
+
self._busy.add(worker_id)
|
|
94
|
+
task = asyncio.current_task()
|
|
95
|
+
assert task is not None
|
|
96
|
+
self._active.add(task)
|
|
97
|
+
crawler: Crawler | None = None
|
|
98
|
+
session: Session | None = None
|
|
99
|
+
try:
|
|
100
|
+
session = await self.sessions.acquire(worker_id)
|
|
101
|
+
if session is None:
|
|
102
|
+
raise SessionUnavailable(f"No session is ready for worker {worker_id}")
|
|
103
|
+
session.require_ready()
|
|
104
|
+
async with asyncio.TaskGroup() as tasks:
|
|
105
|
+
crawler = Crawler(
|
|
106
|
+
worker_id=worker_id, settings=self.settings,
|
|
107
|
+
logger=self.logger.getChild(f"worker{worker_id}"),
|
|
108
|
+
session=session, tasks=tasks, delivery=self.delivery,
|
|
109
|
+
)
|
|
110
|
+
await self.spider.crawl(crawler)
|
|
111
|
+
except Exception:
|
|
112
|
+
self.logger.exception("Crawl failed on worker %s", worker_id)
|
|
113
|
+
raise
|
|
114
|
+
finally:
|
|
115
|
+
if crawler is not None:
|
|
116
|
+
crawler._active = False
|
|
117
|
+
try:
|
|
118
|
+
if session is not None:
|
|
119
|
+
await self.sessions.release(worker_id, session)
|
|
120
|
+
finally:
|
|
121
|
+
self._busy.discard(worker_id)
|
|
122
|
+
self._active.discard(task)
|
|
123
|
+
self._notify()
|
|
124
|
+
|
|
125
|
+
async def run(self) -> None:
|
|
126
|
+
work = asyncio.create_task(self._serve(), name="zenx:execution")
|
|
127
|
+
stopped = asyncio.create_task(self.stop.wait(), name="zenx:stop")
|
|
128
|
+
cancelled = False
|
|
129
|
+
try:
|
|
130
|
+
await asyncio.wait((work, stopped), return_when=asyncio.FIRST_COMPLETED)
|
|
131
|
+
except asyncio.CancelledError:
|
|
132
|
+
cancelled = True
|
|
133
|
+
finally:
|
|
134
|
+
self.stop.set()
|
|
135
|
+
stopped.cancel()
|
|
136
|
+
await self._shutdown(work)
|
|
137
|
+
if cancelled:
|
|
138
|
+
raise asyncio.CancelledError
|
|
139
|
+
if work.done() and not work.cancelled():
|
|
140
|
+
work.result()
|
|
141
|
+
|
|
142
|
+
async def _graceful(self, work: asyncio.Task[None]) -> None:
|
|
143
|
+
await asyncio.gather(work, return_exceptions=True)
|
|
144
|
+
if self._active:
|
|
145
|
+
await asyncio.gather(*tuple(self._active), return_exceptions=True)
|
|
146
|
+
self.delivery.accepting = False
|
|
147
|
+
for sender in self.senders.values():
|
|
148
|
+
sender.stop_admission()
|
|
149
|
+
await asyncio.gather(*(sender.drain() for sender in self.senders.values()))
|
|
150
|
+
await asyncio.gather(*self._begin_close())
|
|
151
|
+
|
|
152
|
+
async def _close(self, closer: Callable[[], Awaitable[None]]) -> None:
|
|
153
|
+
try:
|
|
154
|
+
await closer()
|
|
155
|
+
except Exception:
|
|
156
|
+
self.logger.exception("Resource cleanup failed")
|
|
157
|
+
|
|
158
|
+
def _begin_close(self) -> list[asyncio.Task[None]]:
|
|
159
|
+
if self._closing is None:
|
|
160
|
+
self._notifier_ready = False
|
|
161
|
+
self._closing = [
|
|
162
|
+
asyncio.create_task(self._close(closer), name="zenx:cleanup")
|
|
163
|
+
for closer in reversed(self._closers)
|
|
164
|
+
]
|
|
165
|
+
return self._closing
|
|
166
|
+
|
|
167
|
+
async def _shutdown(self, work: asyncio.Task[None]) -> None:
|
|
168
|
+
graceful: asyncio.Task[None] | None = None
|
|
169
|
+
if self.once:
|
|
170
|
+
graceful = asyncio.create_task(self._graceful(work), name="zenx:drain")
|
|
171
|
+
# asyncio.wait does not wait for cancellation acknowledgement past
|
|
172
|
+
# its timeout, unlike wait_for and a timeout context around cleanup.
|
|
173
|
+
done, _ = await asyncio.wait((graceful,), timeout=ONE_SHOT_GRACE)
|
|
174
|
+
if done:
|
|
175
|
+
graceful.result()
|
|
176
|
+
return
|
|
177
|
+
self.logger.warning("One-shot shutdown grace expired; discarding pending work")
|
|
178
|
+
graceful.cancel()
|
|
179
|
+
self.delivery.accepting = False
|
|
180
|
+
work.cancel()
|
|
181
|
+
active = tuple(self._active)
|
|
182
|
+
for task in active:
|
|
183
|
+
task.cancel()
|
|
184
|
+
for sender in self.senders.values():
|
|
185
|
+
sender.cancel()
|
|
186
|
+
closing = self._begin_close()
|
|
187
|
+
# Allow ready cancellation/finally callbacks to release resources. There
|
|
188
|
+
# is no I/O wait or new grace period on this immediate shutdown path.
|
|
189
|
+
for _ in range(8):
|
|
190
|
+
await asyncio.sleep(0)
|
|
191
|
+
pending = [work, *active, *closing]
|
|
192
|
+
if graceful is not None:
|
|
193
|
+
pending.append(graceful)
|
|
194
|
+
for task in pending:
|
|
195
|
+
if not task.done():
|
|
196
|
+
task.cancel()
|
|
197
|
+
task.add_done_callback(_consume)
|
|
198
|
+
|
|
199
|
+
def stats(self) -> dict[str, DeliveryStats]:
|
|
200
|
+
return {name: sender.stats() for name, sender in self.senders.items()}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
class App:
|
|
204
|
+
def __init__(
|
|
205
|
+
self, *, project_name: str, spiders: Sequence[Spider], destinations: Mapping[str, Destination],
|
|
206
|
+
settings: Settings | None = None, scheduler: Scheduler | None = None,
|
|
207
|
+
store: Store | None = None, sessions: SessionManager | None = None,
|
|
208
|
+
dedup_scope: str | None = None,
|
|
209
|
+
routes: Mapping[str, Collection[str]] | None = None,
|
|
210
|
+
notifiers: Mapping[str, Notifier] | None = None,
|
|
211
|
+
) -> None:
|
|
212
|
+
if not isinstance(project_name, str) or not project_name.strip():
|
|
213
|
+
raise ConfigurationError("project_name must be explicitly configured")
|
|
214
|
+
if dedup_scope is not None and (not isinstance(dedup_scope, str) or not dedup_scope.strip()):
|
|
215
|
+
raise ConfigurationError("dedup_scope must be a nonempty string or None")
|
|
216
|
+
self.project_name = project_name
|
|
217
|
+
self.dedup_scope = dedup_scope
|
|
218
|
+
self.spiders: dict[str, Spider] = {}
|
|
219
|
+
self._routes: dict[str, tuple[str, ...]] = {}
|
|
220
|
+
self.destinations = dict(destinations)
|
|
221
|
+
routes = dict(routes) if routes is not None else {}
|
|
222
|
+
for name in self.destinations:
|
|
223
|
+
if not isinstance(name, str) or not name:
|
|
224
|
+
raise ConfigurationError("Destination names must be nonempty strings")
|
|
225
|
+
for spider in spiders:
|
|
226
|
+
if not isinstance(spider.name, str) or not spider.name or spider.name in self.spiders:
|
|
227
|
+
raise ConfigurationError("Spiders must have distinct, nonempty names")
|
|
228
|
+
allowed = spider.destinations
|
|
229
|
+
if not isinstance(allowed, Collection) or isinstance(allowed, str) or any(not isinstance(name, str) or not name for name in allowed):
|
|
230
|
+
raise ConfigurationError(f"Spider {spider.name} must declare destination names")
|
|
231
|
+
active = routes.get(spider.name, allowed)
|
|
232
|
+
if not isinstance(active, Collection) or isinstance(active, str) or any(not isinstance(name, str) or name not in allowed or name not in self.destinations for name in active):
|
|
233
|
+
raise ConfigurationError(f"Spider {spider.name} routes must be allowed and registered destinations")
|
|
234
|
+
self._routes[spider.name] = tuple(dict.fromkeys(active))
|
|
235
|
+
self.spiders[spider.name] = spider
|
|
236
|
+
if not self.spiders:
|
|
237
|
+
raise ConfigurationError("Register at least one spider")
|
|
238
|
+
if any(name not in self.spiders for name in routes):
|
|
239
|
+
raise ConfigurationError("Routes reference an unknown spider")
|
|
240
|
+
self.notifiers = dict(notifiers) if notifiers is not None else {}
|
|
241
|
+
if any(name not in self.spiders for name in self.notifiers):
|
|
242
|
+
raise ConfigurationError("Notifiers reference an unknown spider")
|
|
243
|
+
self.settings = settings if settings is not None else Settings()
|
|
244
|
+
self.scheduler = scheduler if scheduler is not None else Slots()
|
|
245
|
+
self.store = store
|
|
246
|
+
self.sessions = sessions
|
|
247
|
+
self._runtime: _Runtime | None = None
|
|
248
|
+
self._last_stats: dict[str, DeliveryStats] = {}
|
|
249
|
+
|
|
250
|
+
@staticmethod
|
|
251
|
+
def cli_parser(*, description: str | None = None) -> argparse.ArgumentParser:
|
|
252
|
+
return cli_parser(description=description)
|
|
253
|
+
|
|
254
|
+
@property
|
|
255
|
+
def delivery_stats(self) -> dict[str, DeliveryStats]:
|
|
256
|
+
return self._runtime.stats() if self._runtime is not None else self._last_stats.copy()
|
|
257
|
+
|
|
258
|
+
def stop(self) -> None:
|
|
259
|
+
"""Request shutdown from the app's event loop (CLI signals call this)."""
|
|
260
|
+
if self._runtime is not None:
|
|
261
|
+
self._runtime.stop.set()
|
|
262
|
+
|
|
263
|
+
def _select(self, name: str | None) -> Spider:
|
|
264
|
+
if name is None and len(self.spiders) == 1:
|
|
265
|
+
return next(iter(self.spiders.values()))
|
|
266
|
+
if name is None:
|
|
267
|
+
raise ConfigurationError("Select one spider with --spider or run(spider=...)")
|
|
268
|
+
if name not in self.spiders:
|
|
269
|
+
raise ConfigurationError(f"Unknown spider: {name}")
|
|
270
|
+
return self.spiders[name]
|
|
271
|
+
|
|
272
|
+
async def run(self, *, spider: str | None = None, once: bool = False) -> None:
|
|
273
|
+
"""Run explicit Python configuration without a CLI/environment overlay."""
|
|
274
|
+
await self._run(spider=spider, once=once, settings=self.settings, scheduler=self.scheduler, store=self.store)
|
|
275
|
+
|
|
276
|
+
async def _run(
|
|
277
|
+
self, *, spider: str | None, once: bool, settings: Settings,
|
|
278
|
+
scheduler: Scheduler, store: Store | None,
|
|
279
|
+
) -> None:
|
|
280
|
+
if self._runtime is not None:
|
|
281
|
+
raise RuntimeError("This App is already running")
|
|
282
|
+
settings.validate()
|
|
283
|
+
selected = self._select(spider)
|
|
284
|
+
runtime = _Runtime(
|
|
285
|
+
project_name=self.project_name, spider=selected,
|
|
286
|
+
destinations={name: self.destinations[name] for name in self._routes[selected.name]},
|
|
287
|
+
settings=settings, scheduler=scheduler,
|
|
288
|
+
store=store if store is not None else MemoryStore(settings.dedup_size),
|
|
289
|
+
sessions=self.sessions if self.sessions is not None else Sessions(), once=once,
|
|
290
|
+
dedup_scope=selected.name if self.dedup_scope is None else self.dedup_scope,
|
|
291
|
+
notifier=self.notifiers.get(selected.name),
|
|
292
|
+
)
|
|
293
|
+
self._runtime = runtime
|
|
294
|
+
try:
|
|
295
|
+
await runtime.run()
|
|
296
|
+
finally:
|
|
297
|
+
self._last_stats = runtime.stats()
|
|
298
|
+
self._runtime = None
|
|
299
|
+
|
|
300
|
+
def cli(
|
|
301
|
+
self, argv: Sequence[str] | None = None, *, args: argparse.Namespace | None = None,
|
|
302
|
+
parser: argparse.ArgumentParser | None = None,
|
|
303
|
+
) -> None:
|
|
304
|
+
"""Run a script, optionally using a project-extended parser/namespace."""
|
|
305
|
+
if argv is not None and args is not None:
|
|
306
|
+
raise TypeError("Supply argv or an already parsed namespace, not both")
|
|
307
|
+
parser = parser if parser is not None else self.cli_parser()
|
|
308
|
+
args = args if args is not None else parser.parse_args(argv)
|
|
309
|
+
try:
|
|
310
|
+
overrides = {name: getattr(args, name) for name in ("concurrency", "dedup_size", "log_level", "proxy") if hasattr(args, name)}
|
|
311
|
+
settings = replace(self.settings, **overrides)
|
|
312
|
+
settings.validate()
|
|
313
|
+
spider = getattr(args, "spider", None)
|
|
314
|
+
self._select(spider)
|
|
315
|
+
scheduler = self.scheduler
|
|
316
|
+
if hasattr(args, "interval"):
|
|
317
|
+
if type(scheduler) is not Slots:
|
|
318
|
+
raise ConfigurationError("--interval / ZENX_LITE_INTERVAL require the Slots scheduler")
|
|
319
|
+
scheduler = Slots(args.interval)
|
|
320
|
+
store = self.store
|
|
321
|
+
if hasattr(args, "dedup_size") and store is not None:
|
|
322
|
+
if not isinstance(store, MemoryStore):
|
|
323
|
+
raise ConfigurationError("--dedup-size applies only to the memory store")
|
|
324
|
+
store = MemoryStore(settings.dedup_size)
|
|
325
|
+
except ConfigurationError as exc:
|
|
326
|
+
parser.error(str(exc))
|
|
327
|
+
logging.basicConfig(level=settings.log_level.upper(), format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
|
328
|
+
try:
|
|
329
|
+
self._cli_loop(spider, getattr(args, "once", False), settings, scheduler, store)
|
|
330
|
+
except Exception:
|
|
331
|
+
logging.getLogger("zenx_lite").exception("Application failed")
|
|
332
|
+
raise SystemExit(1) from None
|
|
333
|
+
|
|
334
|
+
def _cli_loop(
|
|
335
|
+
self, spider: str | None, once: bool, settings: Settings,
|
|
336
|
+
scheduler: Scheduler, store: Store | None,
|
|
337
|
+
) -> None:
|
|
338
|
+
try:
|
|
339
|
+
asyncio.get_running_loop()
|
|
340
|
+
except RuntimeError:
|
|
341
|
+
pass
|
|
342
|
+
else:
|
|
343
|
+
raise RuntimeError("app.cli() owns its loop; use await app.run() inside an event loop")
|
|
344
|
+
loop = asyncio.new_event_loop()
|
|
345
|
+
previous: dict[signal.Signals, Any] = {}
|
|
346
|
+
received: int | None = None
|
|
347
|
+
|
|
348
|
+
def stop(signum: signal.Signals) -> None:
|
|
349
|
+
nonlocal received
|
|
350
|
+
received = signum
|
|
351
|
+
self.stop()
|
|
352
|
+
|
|
353
|
+
try:
|
|
354
|
+
asyncio.set_event_loop(loop)
|
|
355
|
+
for signum in (signal.SIGINT, signal.SIGTERM):
|
|
356
|
+
try:
|
|
357
|
+
handler = signal.getsignal(signum)
|
|
358
|
+
loop.add_signal_handler(signum, stop, signum)
|
|
359
|
+
previous[signum] = handler
|
|
360
|
+
except (NotImplementedError, RuntimeError):
|
|
361
|
+
pass
|
|
362
|
+
loop.run_until_complete(self._run(spider=spider, once=once, settings=settings, scheduler=scheduler, store=store))
|
|
363
|
+
finally:
|
|
364
|
+
for signum, handler in previous.items():
|
|
365
|
+
loop.remove_signal_handler(signum)
|
|
366
|
+
signal.signal(signum, handler)
|
|
367
|
+
# Do not introduce asyncio.run's unbounded cancellation/executor join
|
|
368
|
+
# after the runtime's shutdown deadline has already expired.
|
|
369
|
+
for task in asyncio.all_tasks(loop):
|
|
370
|
+
task.cancel()
|
|
371
|
+
task.add_done_callback(_consume)
|
|
372
|
+
for _ in range(3):
|
|
373
|
+
loop.run_until_complete(asyncio.sleep(0))
|
|
374
|
+
loop.close()
|
|
375
|
+
asyncio.set_event_loop(None)
|
|
376
|
+
if received is not None:
|
|
377
|
+
raise SystemExit(128 + received)
|
zenx_lite/config.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Runtime settings and the deliberately small CLI/environment overlay."""
|
|
2
|
+
|
|
3
|
+
lazy import argparse
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
|
|
9
|
+
from .errors import ConfigurationError
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True, slots=True)
|
|
13
|
+
class Settings:
|
|
14
|
+
concurrency: int = 1
|
|
15
|
+
dedup_size: int = 500
|
|
16
|
+
log_level: str = "INFO"
|
|
17
|
+
proxy: str | None = None
|
|
18
|
+
|
|
19
|
+
def validate(self) -> None:
|
|
20
|
+
for name in ("concurrency", "dedup_size"):
|
|
21
|
+
value = getattr(self, name)
|
|
22
|
+
if type(value) is not int or value <= 0:
|
|
23
|
+
raise ConfigurationError(f"{name} must be a positive integer")
|
|
24
|
+
if not isinstance(self.log_level, str) or self.log_level.upper() not in logging.getLevelNamesMapping():
|
|
25
|
+
raise ConfigurationError("log_level must be a standard logging level")
|
|
26
|
+
if self.proxy is not None and (not isinstance(self.proxy, str) or not self.proxy.strip()):
|
|
27
|
+
raise ConfigurationError("proxy must be a nonempty proxy URL or None")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def cli_parser(
|
|
31
|
+
*, description: str | None = None, environ: Mapping[str, str] | None = None
|
|
32
|
+
) -> argparse.ArgumentParser:
|
|
33
|
+
"""Return a parser projects can extend before constructing their spiders.
|
|
34
|
+
|
|
35
|
+
Absent options stay absent so Python configuration can supply their values.
|
|
36
|
+
argparse converts environment defaults only when a CLI value doesn't replace
|
|
37
|
+
them. An invalid environment value therefore cannot override a valid flag.
|
|
38
|
+
"""
|
|
39
|
+
env = os.environ if environ is None else environ
|
|
40
|
+
parser = argparse.ArgumentParser(description=description)
|
|
41
|
+
options: tuple[tuple[str, str, type, str], ...] = (
|
|
42
|
+
("spider", "ZENX_LITE_SPIDER", str, "Spider to run"),
|
|
43
|
+
("concurrency", "ZENX_LITE_CONCURRENCY", int, "Number of crawl workers"),
|
|
44
|
+
("interval", "ZENX_LITE_INTERVAL", float, "Slots interval in seconds"),
|
|
45
|
+
("log-level", "ZENX_LITE_LOG_LEVEL", str, "Logging level"),
|
|
46
|
+
("dedup-size", "ZENX_LITE_DEDUP_SIZE", int, "Remembered item ID capacity"),
|
|
47
|
+
("proxy", "ZENX_LITE_PROXY", str, "Proxy URL for managed sessions"),
|
|
48
|
+
)
|
|
49
|
+
for flag, variable, value_type, help_text in options:
|
|
50
|
+
default = env.get(variable, None if flag == "spider" else argparse.SUPPRESS)
|
|
51
|
+
parser.add_argument(
|
|
52
|
+
f"--{flag}", type=value_type, default=default,
|
|
53
|
+
help=f"{help_text} (environment: {variable})",
|
|
54
|
+
)
|
|
55
|
+
parser.add_argument("--once", action="store_true", help="Run one crawl, then shut down")
|
|
56
|
+
return parser
|