zenx-lite-python 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
zenx_lite/__init__.py ADDED
@@ -0,0 +1,8 @@
1
+ """One selected spider, explicitly composed dependencies, best-effort delivery."""
2
+
3
+ from .app import App
4
+ from .config import Settings
5
+ from .crawler import Crawler, Spider
6
+ from .http import Response
7
+
8
+ __all__ = ["App", "Crawler", "Response", "Settings", "Spider"]
zenx_lite/app.py ADDED
@@ -0,0 +1,377 @@
1
+ """Explicit application composition and a single runtime-owned lifecycle."""
2
+
3
+ lazy import argparse
4
+ import asyncio
5
+ import logging
6
+ import signal
7
+ from collections.abc import Awaitable, Callable, Collection, Mapping, Sequence
8
+ from dataclasses import replace
9
+ from typing import Any
10
+
11
+ from .config import Settings, cli_parser
12
+ from .crawler import Crawler, Spider
13
+ from .delivery import Delivery, DeliveryStats, Sender
14
+ from .errors import ConfigurationError, FatalRuntimeError, SessionUnavailable, WorkerOverlap
15
+ from .notifiers.base import Notifier
16
+ from .scheduler import Scheduler, Slots
17
+ from .sessions import Session, SessionManager, Sessions
18
+ from .destinations.base import Destination
19
+ from .store import MemoryStore, Store
20
+
21
+ ONE_SHOT_GRACE = 60.0
22
+
23
+
24
+ def _consume(task: asyncio.Task[Any]) -> None:
25
+ if not task.cancelled():
26
+ task.exception()
27
+
28
+
29
+ class _Runtime:
30
+ def __init__(
31
+ self, *, project_name: str, spider: Spider, destinations: Mapping[str, Destination],
32
+ settings: Settings, scheduler: Scheduler, store: Store, sessions: SessionManager,
33
+ once: bool, dedup_scope: str, notifier: Notifier | None,
34
+ ) -> None:
35
+ self.spider = spider
36
+ self.settings = settings
37
+ self.scheduler = scheduler
38
+ self.store = store
39
+ self.sessions = sessions
40
+ self.once = once
41
+ self.notifier = notifier
42
+ self._notifier_ready = False
43
+ self.stop = asyncio.Event()
44
+ logging.getLogger("zenx_lite").setLevel(settings.log_level.upper())
45
+ self.logger = logging.getLogger(f"zenx_lite.{project_name}.{spider.name}")
46
+ self.logger.setLevel(settings.log_level.upper())
47
+ self.senders = {name: Sender(name, destination, self.logger) for name, destination in destinations.items()}
48
+ self.delivery = Delivery(project_name, dedup_scope, store, self.senders, self.logger)
49
+ self._busy: set[int] = set()
50
+ self._active: set[asyncio.Task[Any]] = set()
51
+ self._closers: list[Callable[[], Awaitable[None]]] = []
52
+ self._closing: list[asyncio.Task[None]] | None = None
53
+
54
+ async def _serve(self) -> None:
55
+ # Register cleanup before opening, including partially opened resources.
56
+ if self.notifier is not None:
57
+ self._closers.append(self.notifier.close)
58
+ try:
59
+ await self.notifier.open()
60
+ except Exception:
61
+ self.logger.exception("Notifier startup failed; continuing without telemetry")
62
+ else:
63
+ self._notifier_ready = True
64
+ self._closers.append(self.store.close)
65
+ await self.store.open()
66
+ self._closers.append(self.sessions.close)
67
+ await self.sessions.open(concurrency=1 if self.once else self.settings.concurrency, proxy=self.settings.proxy)
68
+ for sender in self.senders.values():
69
+ self._closers.append(sender.close)
70
+ await sender.open()
71
+ if self.stop.is_set():
72
+ return
73
+ if self.once:
74
+ await self.crawl(0)
75
+ else:
76
+ await self.scheduler.run(self.crawl, concurrency=self.settings.concurrency, stop=self.stop)
77
+
78
+ def _notify(self) -> None:
79
+ if self.notifier is None or not self._notifier_ready:
80
+ return
81
+ try:
82
+ self.notifier.notify()
83
+ except Exception:
84
+ self.logger.exception("Notifier failed; notification discarded")
85
+
86
+ async def crawl(self, worker_id: int) -> None:
87
+ if type(worker_id) is not int or not 0 <= worker_id < self.settings.concurrency:
88
+ raise FatalRuntimeError("Scheduler supplied an invalid worker ID")
89
+ if worker_id in self._busy:
90
+ raise WorkerOverlap(f"Worker {worker_id} already has an active crawl")
91
+ if self.stop.is_set():
92
+ return
93
+ self._busy.add(worker_id)
94
+ task = asyncio.current_task()
95
+ assert task is not None
96
+ self._active.add(task)
97
+ crawler: Crawler | None = None
98
+ session: Session | None = None
99
+ try:
100
+ session = await self.sessions.acquire(worker_id)
101
+ if session is None:
102
+ raise SessionUnavailable(f"No session is ready for worker {worker_id}")
103
+ session.require_ready()
104
+ async with asyncio.TaskGroup() as tasks:
105
+ crawler = Crawler(
106
+ worker_id=worker_id, settings=self.settings,
107
+ logger=self.logger.getChild(f"worker{worker_id}"),
108
+ session=session, tasks=tasks, delivery=self.delivery,
109
+ )
110
+ await self.spider.crawl(crawler)
111
+ except Exception:
112
+ self.logger.exception("Crawl failed on worker %s", worker_id)
113
+ raise
114
+ finally:
115
+ if crawler is not None:
116
+ crawler._active = False
117
+ try:
118
+ if session is not None:
119
+ await self.sessions.release(worker_id, session)
120
+ finally:
121
+ self._busy.discard(worker_id)
122
+ self._active.discard(task)
123
+ self._notify()
124
+
125
+ async def run(self) -> None:
126
+ work = asyncio.create_task(self._serve(), name="zenx:execution")
127
+ stopped = asyncio.create_task(self.stop.wait(), name="zenx:stop")
128
+ cancelled = False
129
+ try:
130
+ await asyncio.wait((work, stopped), return_when=asyncio.FIRST_COMPLETED)
131
+ except asyncio.CancelledError:
132
+ cancelled = True
133
+ finally:
134
+ self.stop.set()
135
+ stopped.cancel()
136
+ await self._shutdown(work)
137
+ if cancelled:
138
+ raise asyncio.CancelledError
139
+ if work.done() and not work.cancelled():
140
+ work.result()
141
+
142
+ async def _graceful(self, work: asyncio.Task[None]) -> None:
143
+ await asyncio.gather(work, return_exceptions=True)
144
+ if self._active:
145
+ await asyncio.gather(*tuple(self._active), return_exceptions=True)
146
+ self.delivery.accepting = False
147
+ for sender in self.senders.values():
148
+ sender.stop_admission()
149
+ await asyncio.gather(*(sender.drain() for sender in self.senders.values()))
150
+ await asyncio.gather(*self._begin_close())
151
+
152
+ async def _close(self, closer: Callable[[], Awaitable[None]]) -> None:
153
+ try:
154
+ await closer()
155
+ except Exception:
156
+ self.logger.exception("Resource cleanup failed")
157
+
158
+ def _begin_close(self) -> list[asyncio.Task[None]]:
159
+ if self._closing is None:
160
+ self._notifier_ready = False
161
+ self._closing = [
162
+ asyncio.create_task(self._close(closer), name="zenx:cleanup")
163
+ for closer in reversed(self._closers)
164
+ ]
165
+ return self._closing
166
+
167
+ async def _shutdown(self, work: asyncio.Task[None]) -> None:
168
+ graceful: asyncio.Task[None] | None = None
169
+ if self.once:
170
+ graceful = asyncio.create_task(self._graceful(work), name="zenx:drain")
171
+ # asyncio.wait does not wait for cancellation acknowledgement past
172
+ # its timeout, unlike wait_for and a timeout context around cleanup.
173
+ done, _ = await asyncio.wait((graceful,), timeout=ONE_SHOT_GRACE)
174
+ if done:
175
+ graceful.result()
176
+ return
177
+ self.logger.warning("One-shot shutdown grace expired; discarding pending work")
178
+ graceful.cancel()
179
+ self.delivery.accepting = False
180
+ work.cancel()
181
+ active = tuple(self._active)
182
+ for task in active:
183
+ task.cancel()
184
+ for sender in self.senders.values():
185
+ sender.cancel()
186
+ closing = self._begin_close()
187
+ # Allow ready cancellation/finally callbacks to release resources. There
188
+ # is no I/O wait or new grace period on this immediate shutdown path.
189
+ for _ in range(8):
190
+ await asyncio.sleep(0)
191
+ pending = [work, *active, *closing]
192
+ if graceful is not None:
193
+ pending.append(graceful)
194
+ for task in pending:
195
+ if not task.done():
196
+ task.cancel()
197
+ task.add_done_callback(_consume)
198
+
199
+ def stats(self) -> dict[str, DeliveryStats]:
200
+ return {name: sender.stats() for name, sender in self.senders.items()}
201
+
202
+
203
+ class App:
204
+ def __init__(
205
+ self, *, project_name: str, spiders: Sequence[Spider], destinations: Mapping[str, Destination],
206
+ settings: Settings | None = None, scheduler: Scheduler | None = None,
207
+ store: Store | None = None, sessions: SessionManager | None = None,
208
+ dedup_scope: str | None = None,
209
+ routes: Mapping[str, Collection[str]] | None = None,
210
+ notifiers: Mapping[str, Notifier] | None = None,
211
+ ) -> None:
212
+ if not isinstance(project_name, str) or not project_name.strip():
213
+ raise ConfigurationError("project_name must be explicitly configured")
214
+ if dedup_scope is not None and (not isinstance(dedup_scope, str) or not dedup_scope.strip()):
215
+ raise ConfigurationError("dedup_scope must be a nonempty string or None")
216
+ self.project_name = project_name
217
+ self.dedup_scope = dedup_scope
218
+ self.spiders: dict[str, Spider] = {}
219
+ self._routes: dict[str, tuple[str, ...]] = {}
220
+ self.destinations = dict(destinations)
221
+ routes = dict(routes) if routes is not None else {}
222
+ for name in self.destinations:
223
+ if not isinstance(name, str) or not name:
224
+ raise ConfigurationError("Destination names must be nonempty strings")
225
+ for spider in spiders:
226
+ if not isinstance(spider.name, str) or not spider.name or spider.name in self.spiders:
227
+ raise ConfigurationError("Spiders must have distinct, nonempty names")
228
+ allowed = spider.destinations
229
+ if not isinstance(allowed, Collection) or isinstance(allowed, str) or any(not isinstance(name, str) or not name for name in allowed):
230
+ raise ConfigurationError(f"Spider {spider.name} must declare destination names")
231
+ active = routes.get(spider.name, allowed)
232
+ if not isinstance(active, Collection) or isinstance(active, str) or any(not isinstance(name, str) or name not in allowed or name not in self.destinations for name in active):
233
+ raise ConfigurationError(f"Spider {spider.name} routes must be allowed and registered destinations")
234
+ self._routes[spider.name] = tuple(dict.fromkeys(active))
235
+ self.spiders[spider.name] = spider
236
+ if not self.spiders:
237
+ raise ConfigurationError("Register at least one spider")
238
+ if any(name not in self.spiders for name in routes):
239
+ raise ConfigurationError("Routes reference an unknown spider")
240
+ self.notifiers = dict(notifiers) if notifiers is not None else {}
241
+ if any(name not in self.spiders for name in self.notifiers):
242
+ raise ConfigurationError("Notifiers reference an unknown spider")
243
+ self.settings = settings if settings is not None else Settings()
244
+ self.scheduler = scheduler if scheduler is not None else Slots()
245
+ self.store = store
246
+ self.sessions = sessions
247
+ self._runtime: _Runtime | None = None
248
+ self._last_stats: dict[str, DeliveryStats] = {}
249
+
250
+ @staticmethod
251
+ def cli_parser(*, description: str | None = None) -> argparse.ArgumentParser:
252
+ return cli_parser(description=description)
253
+
254
+ @property
255
+ def delivery_stats(self) -> dict[str, DeliveryStats]:
256
+ return self._runtime.stats() if self._runtime is not None else self._last_stats.copy()
257
+
258
+ def stop(self) -> None:
259
+ """Request shutdown from the app's event loop (CLI signals call this)."""
260
+ if self._runtime is not None:
261
+ self._runtime.stop.set()
262
+
263
+ def _select(self, name: str | None) -> Spider:
264
+ if name is None and len(self.spiders) == 1:
265
+ return next(iter(self.spiders.values()))
266
+ if name is None:
267
+ raise ConfigurationError("Select one spider with --spider or run(spider=...)")
268
+ if name not in self.spiders:
269
+ raise ConfigurationError(f"Unknown spider: {name}")
270
+ return self.spiders[name]
271
+
272
+ async def run(self, *, spider: str | None = None, once: bool = False) -> None:
273
+ """Run explicit Python configuration without a CLI/environment overlay."""
274
+ await self._run(spider=spider, once=once, settings=self.settings, scheduler=self.scheduler, store=self.store)
275
+
276
+ async def _run(
277
+ self, *, spider: str | None, once: bool, settings: Settings,
278
+ scheduler: Scheduler, store: Store | None,
279
+ ) -> None:
280
+ if self._runtime is not None:
281
+ raise RuntimeError("This App is already running")
282
+ settings.validate()
283
+ selected = self._select(spider)
284
+ runtime = _Runtime(
285
+ project_name=self.project_name, spider=selected,
286
+ destinations={name: self.destinations[name] for name in self._routes[selected.name]},
287
+ settings=settings, scheduler=scheduler,
288
+ store=store if store is not None else MemoryStore(settings.dedup_size),
289
+ sessions=self.sessions if self.sessions is not None else Sessions(), once=once,
290
+ dedup_scope=selected.name if self.dedup_scope is None else self.dedup_scope,
291
+ notifier=self.notifiers.get(selected.name),
292
+ )
293
+ self._runtime = runtime
294
+ try:
295
+ await runtime.run()
296
+ finally:
297
+ self._last_stats = runtime.stats()
298
+ self._runtime = None
299
+
300
+ def cli(
301
+ self, argv: Sequence[str] | None = None, *, args: argparse.Namespace | None = None,
302
+ parser: argparse.ArgumentParser | None = None,
303
+ ) -> None:
304
+ """Run a script, optionally using a project-extended parser/namespace."""
305
+ if argv is not None and args is not None:
306
+ raise TypeError("Supply argv or an already parsed namespace, not both")
307
+ parser = parser if parser is not None else self.cli_parser()
308
+ args = args if args is not None else parser.parse_args(argv)
309
+ try:
310
+ overrides = {name: getattr(args, name) for name in ("concurrency", "dedup_size", "log_level", "proxy") if hasattr(args, name)}
311
+ settings = replace(self.settings, **overrides)
312
+ settings.validate()
313
+ spider = getattr(args, "spider", None)
314
+ self._select(spider)
315
+ scheduler = self.scheduler
316
+ if hasattr(args, "interval"):
317
+ if type(scheduler) is not Slots:
318
+ raise ConfigurationError("--interval / ZENX_LITE_INTERVAL require the Slots scheduler")
319
+ scheduler = Slots(args.interval)
320
+ store = self.store
321
+ if hasattr(args, "dedup_size") and store is not None:
322
+ if not isinstance(store, MemoryStore):
323
+ raise ConfigurationError("--dedup-size applies only to the memory store")
324
+ store = MemoryStore(settings.dedup_size)
325
+ except ConfigurationError as exc:
326
+ parser.error(str(exc))
327
+ logging.basicConfig(level=settings.log_level.upper(), format="%(asctime)s %(levelname)s %(name)s %(message)s")
328
+ try:
329
+ self._cli_loop(spider, getattr(args, "once", False), settings, scheduler, store)
330
+ except Exception:
331
+ logging.getLogger("zenx_lite").exception("Application failed")
332
+ raise SystemExit(1) from None
333
+
334
+ def _cli_loop(
335
+ self, spider: str | None, once: bool, settings: Settings,
336
+ scheduler: Scheduler, store: Store | None,
337
+ ) -> None:
338
+ try:
339
+ asyncio.get_running_loop()
340
+ except RuntimeError:
341
+ pass
342
+ else:
343
+ raise RuntimeError("app.cli() owns its loop; use await app.run() inside an event loop")
344
+ loop = asyncio.new_event_loop()
345
+ previous: dict[signal.Signals, Any] = {}
346
+ received: int | None = None
347
+
348
+ def stop(signum: signal.Signals) -> None:
349
+ nonlocal received
350
+ received = signum
351
+ self.stop()
352
+
353
+ try:
354
+ asyncio.set_event_loop(loop)
355
+ for signum in (signal.SIGINT, signal.SIGTERM):
356
+ try:
357
+ handler = signal.getsignal(signum)
358
+ loop.add_signal_handler(signum, stop, signum)
359
+ previous[signum] = handler
360
+ except (NotImplementedError, RuntimeError):
361
+ pass
362
+ loop.run_until_complete(self._run(spider=spider, once=once, settings=settings, scheduler=scheduler, store=store))
363
+ finally:
364
+ for signum, handler in previous.items():
365
+ loop.remove_signal_handler(signum)
366
+ signal.signal(signum, handler)
367
+ # Do not introduce asyncio.run's unbounded cancellation/executor join
368
+ # after the runtime's shutdown deadline has already expired.
369
+ for task in asyncio.all_tasks(loop):
370
+ task.cancel()
371
+ task.add_done_callback(_consume)
372
+ for _ in range(3):
373
+ loop.run_until_complete(asyncio.sleep(0))
374
+ loop.close()
375
+ asyncio.set_event_loop(None)
376
+ if received is not None:
377
+ raise SystemExit(128 + received)
zenx_lite/config.py ADDED
@@ -0,0 +1,56 @@
1
+ """Runtime settings and the deliberately small CLI/environment overlay."""
2
+
3
+ lazy import argparse
4
+ import logging
5
+ import os
6
+ from collections.abc import Mapping
7
+ from dataclasses import dataclass
8
+
9
+ from .errors import ConfigurationError
10
+
11
+
12
+ @dataclass(frozen=True, slots=True)
13
+ class Settings:
14
+ concurrency: int = 1
15
+ dedup_size: int = 500
16
+ log_level: str = "INFO"
17
+ proxy: str | None = None
18
+
19
+ def validate(self) -> None:
20
+ for name in ("concurrency", "dedup_size"):
21
+ value = getattr(self, name)
22
+ if type(value) is not int or value <= 0:
23
+ raise ConfigurationError(f"{name} must be a positive integer")
24
+ if not isinstance(self.log_level, str) or self.log_level.upper() not in logging.getLevelNamesMapping():
25
+ raise ConfigurationError("log_level must be a standard logging level")
26
+ if self.proxy is not None and (not isinstance(self.proxy, str) or not self.proxy.strip()):
27
+ raise ConfigurationError("proxy must be a nonempty proxy URL or None")
28
+
29
+
30
+ def cli_parser(
31
+ *, description: str | None = None, environ: Mapping[str, str] | None = None
32
+ ) -> argparse.ArgumentParser:
33
+ """Return a parser projects can extend before constructing their spiders.
34
+
35
+ Absent options stay absent so Python configuration can supply their values.
36
+ argparse converts environment defaults only when a CLI value doesn't replace
37
+ them. An invalid environment value therefore cannot override a valid flag.
38
+ """
39
+ env = os.environ if environ is None else environ
40
+ parser = argparse.ArgumentParser(description=description)
41
+ options: tuple[tuple[str, str, type, str], ...] = (
42
+ ("spider", "ZENX_LITE_SPIDER", str, "Spider to run"),
43
+ ("concurrency", "ZENX_LITE_CONCURRENCY", int, "Number of crawl workers"),
44
+ ("interval", "ZENX_LITE_INTERVAL", float, "Slots interval in seconds"),
45
+ ("log-level", "ZENX_LITE_LOG_LEVEL", str, "Logging level"),
46
+ ("dedup-size", "ZENX_LITE_DEDUP_SIZE", int, "Remembered item ID capacity"),
47
+ ("proxy", "ZENX_LITE_PROXY", str, "Proxy URL for managed sessions"),
48
+ )
49
+ for flag, variable, value_type, help_text in options:
50
+ default = env.get(variable, None if flag == "spider" else argparse.SUPPRESS)
51
+ parser.add_argument(
52
+ f"--{flag}", type=value_type, default=default,
53
+ help=f"{help_text} (environment: {variable})",
54
+ )
55
+ parser.add_argument("--once", action="store_true", help="Run one crawl, then shut down")
56
+ return parser