zenx-lite 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
zenx_lite/__init__.py ADDED
@@ -0,0 +1,8 @@
1
+ """One selected spider, explicitly composed dependencies, best-effort delivery."""
2
+
3
+ from .app import App
4
+ from .config import Settings
5
+ from .crawler import Crawler, Spider
6
+ from .http import Response
7
+
8
+ __all__ = ["App", "Crawler", "Response", "Settings", "Spider"]
zenx_lite/app.py ADDED
@@ -0,0 +1,390 @@
1
+ """Explicit application composition and a single runtime-owned lifecycle."""
2
+
3
+ lazy import argparse
4
+ import asyncio
5
+ import signal
6
+ from collections.abc import Awaitable, Callable, Collection, Mapping, Sequence
7
+ from dataclasses import replace
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from structlog.typing import FilteringBoundLogger
12
+
13
+ from .config import Settings, cli_parser
14
+ from .crawler import Crawler, Spider
15
+ from .delivery import Delivery, Sender
16
+ from .errors import ConfigurationError, FatalRuntimeError, SessionUnavailable, WorkerOverlap
17
+ from .logging import make_logger, run_logging
18
+ from .notifiers.base import Notifier
19
+ from .scheduler import Scheduler, Slots
20
+ from .sessions import Session, SessionManager, Sessions
21
+ from .destinations.base import DestinationAdapter
22
+ from .store import MemoryStore, Store
23
+
24
+ ONE_SHOT_GRACE = 60.0
25
+
26
+
27
+ def _consume(task: asyncio.Task[Any]) -> None:
28
+ if not task.cancelled():
29
+ task.exception()
30
+
31
+
32
+ class _Runtime:
33
+ def __init__(
34
+ self, *, project_name: str, spider: Spider, destinations: Mapping[str, DestinationAdapter],
35
+ settings: Settings, scheduler: Scheduler, store: Store, sessions: SessionManager,
36
+ once: bool, dedup_scope: str, notifier: Notifier | None, logger: FilteringBoundLogger,
37
+ ) -> None:
38
+ self.spider = spider
39
+ self.settings = settings
40
+ self.scheduler = scheduler
41
+ self.store = store
42
+ self.sessions = sessions
43
+ self.once = once
44
+ self.notifier = notifier
45
+ self._notifier_ready = False
46
+ self.stop = asyncio.Event()
47
+ self.logger = logger
48
+ self._worker_loggers = tuple(logger.bind(worker_id=worker_id) for worker_id in range(1 if once else settings.concurrency))
49
+ self.senders = {name: Sender(name, destination, self.logger) for name, destination in destinations.items()}
50
+ self.delivery = Delivery(project_name, dedup_scope, store, self.senders, self.logger)
51
+ self._busy: set[int] = set()
52
+ self._active: set[asyncio.Task[Any]] = set()
53
+ self._closers: list[Callable[[], Awaitable[None]]] = []
54
+ self._closing: list[asyncio.Task[None]] | None = None
55
+
56
+ async def _serve(self) -> None:
57
+ # Register cleanup before opening, including partially opened resources.
58
+ if self.notifier is not None:
59
+ self._closers.append(self.notifier.close)
60
+ try:
61
+ await self.notifier.open()
62
+ except Exception:
63
+ self.logger.exception("Notifier startup failed; continuing without telemetry")
64
+ else:
65
+ self._notifier_ready = True
66
+ self._closers.append(self.store.close)
67
+ await self.store.open()
68
+ self._closers.append(self.sessions.close)
69
+ await self.sessions.open(concurrency=1 if self.once else self.settings.concurrency, proxy=self.settings.proxy)
70
+ for sender in self.senders.values():
71
+ self._closers.append(sender.close)
72
+ await sender.open()
73
+ if self.stop.is_set():
74
+ return
75
+ if self.once:
76
+ await self.crawl(0)
77
+ else:
78
+ await self.scheduler.run(self.crawl, concurrency=self.settings.concurrency, stop=self.stop)
79
+
80
+ def _notify(self) -> None:
81
+ if self.notifier is None or not self._notifier_ready:
82
+ return
83
+ try:
84
+ self.notifier.notify()
85
+ except Exception:
86
+ self.logger.exception("Notifier failed; notification discarded")
87
+
88
+ async def crawl(self, worker_id: int) -> None:
89
+ if type(worker_id) is not int or not 0 <= worker_id < len(self._worker_loggers):
90
+ raise FatalRuntimeError("Scheduler supplied an invalid worker ID")
91
+ if worker_id in self._busy:
92
+ raise WorkerOverlap(f"Worker {worker_id} already has an active crawl")
93
+ if self.stop.is_set():
94
+ return
95
+ self._busy.add(worker_id)
96
+ logger = self._worker_loggers[worker_id]
97
+ task = asyncio.current_task()
98
+ assert task is not None
99
+ self._active.add(task)
100
+ crawler: Crawler | None = None
101
+ session: Session | None = None
102
+ try:
103
+ session = await self.sessions.acquire(worker_id)
104
+ if session is None:
105
+ raise SessionUnavailable(f"No session is ready for worker {worker_id}")
106
+ session.require_ready()
107
+ async with asyncio.TaskGroup() as tasks:
108
+ crawler = Crawler(
109
+ worker_id=worker_id, settings=self.settings,
110
+ logger=logger,
111
+ session=session, tasks=tasks, delivery=self.delivery,
112
+ )
113
+ await self.spider.crawl(crawler)
114
+ except Exception:
115
+ logger.exception("Crawl failed")
116
+ raise
117
+ finally:
118
+ if crawler is not None:
119
+ crawler._active = False
120
+ try:
121
+ if session is not None:
122
+ await self.sessions.release(worker_id, session)
123
+ finally:
124
+ self._busy.discard(worker_id)
125
+ self._active.discard(task)
126
+ self._notify()
127
+
128
+ async def run(self) -> None:
129
+ work = asyncio.create_task(self._serve(), name="zenx:execution")
130
+ stopped = asyncio.create_task(self.stop.wait(), name="zenx:stop")
131
+ cancelled = False
132
+ try:
133
+ await asyncio.wait((work, stopped), return_when=asyncio.FIRST_COMPLETED)
134
+ except asyncio.CancelledError:
135
+ cancelled = True
136
+ finally:
137
+ self.stop.set()
138
+ stopped.cancel()
139
+ await self._shutdown(work)
140
+ if cancelled:
141
+ raise asyncio.CancelledError
142
+ if work.done() and not work.cancelled():
143
+ work.result()
144
+
145
+ async def _graceful(self, work: asyncio.Task[None]) -> None:
146
+ await asyncio.gather(work, return_exceptions=True)
147
+ if self._active:
148
+ await asyncio.gather(*tuple(self._active), return_exceptions=True)
149
+ self.delivery.accepting = False
150
+ for sender in self.senders.values():
151
+ sender.stop_admission()
152
+ await asyncio.gather(*(sender.drain() for sender in self.senders.values()))
153
+ await asyncio.gather(*self._begin_close())
154
+
155
+ async def _close(self, closer: Callable[[], Awaitable[None]]) -> None:
156
+ try:
157
+ await closer()
158
+ except Exception:
159
+ self.logger.exception("Resource cleanup failed")
160
+
161
+ def _begin_close(self) -> list[asyncio.Task[None]]:
162
+ if self._closing is None:
163
+ self._notifier_ready = False
164
+ self._closing = [
165
+ asyncio.create_task(self._close(closer), name="zenx:cleanup")
166
+ for closer in reversed(self._closers)
167
+ ]
168
+ return self._closing
169
+
170
+ async def _shutdown(self, work: asyncio.Task[None]) -> None:
171
+ graceful: asyncio.Task[None] | None = None
172
+ if self.once:
173
+ graceful = asyncio.create_task(self._graceful(work), name="zenx:drain")
174
+ # asyncio.wait does not wait for cancellation acknowledgement past
175
+ # its timeout, unlike wait_for and a timeout context around cleanup.
176
+ done, _ = await asyncio.wait((graceful,), timeout=ONE_SHOT_GRACE)
177
+ if done:
178
+ graceful.result()
179
+ return
180
+ self.logger.warning("One-shot shutdown grace expired; discarding pending work")
181
+ graceful.cancel()
182
+ self.delivery.accepting = False
183
+ work.cancel()
184
+ active = tuple(self._active)
185
+ for task in active:
186
+ task.cancel()
187
+ for sender in self.senders.values():
188
+ sender.cancel()
189
+ closing = self._begin_close()
190
+ # Allow ready cancellation/finally callbacks to release resources. There
191
+ # is no I/O wait or new grace period on this immediate shutdown path.
192
+ for _ in range(8):
193
+ await asyncio.sleep(0)
194
+ pending = [work, *active, *closing]
195
+ if graceful is not None:
196
+ pending.append(graceful)
197
+ for task in pending:
198
+ if not task.done():
199
+ task.cancel()
200
+ task.add_done_callback(_consume)
201
+
202
+
203
+ class App:
204
+ def __init__(
205
+ self, *, project_name: str, spiders: Sequence[Spider], destinations: Mapping[str, DestinationAdapter],
206
+ settings: Settings | None = None, scheduler: Scheduler | None = None,
207
+ store: Store | None = None, sessions: SessionManager | None = None,
208
+ dedup_scope: str | None = None,
209
+ routes: Mapping[str, Collection[str]] | None = None,
210
+ notifiers: Mapping[str, Notifier] | None = None,
211
+ logger: FilteringBoundLogger | None = None,
212
+ ) -> None:
213
+ if not isinstance(project_name, str) or not project_name.strip():
214
+ raise ConfigurationError("project_name must be explicitly configured")
215
+ if dedup_scope is not None and (not isinstance(dedup_scope, str) or not dedup_scope.strip()):
216
+ raise ConfigurationError("dedup_scope must be a nonempty string or None")
217
+ self.project_name = project_name
218
+ self.dedup_scope = dedup_scope
219
+ self.spiders: dict[str, Spider] = {}
220
+ self._routes: dict[str, tuple[str, ...]] = {}
221
+ self.destinations = dict(destinations)
222
+ routes = dict(routes) if routes is not None else {}
223
+ for name in self.destinations:
224
+ if not isinstance(name, str) or not name:
225
+ raise ConfigurationError("Destination names must be nonempty strings")
226
+ for spider in spiders:
227
+ if not isinstance(spider.name, str) or not spider.name or spider.name in self.spiders:
228
+ raise ConfigurationError("Spiders must have distinct, nonempty names")
229
+ active = routes[spider.name] if spider.name in routes else getattr(spider, "destinations", ())
230
+ if not isinstance(active, Collection) or isinstance(active, str) or any(not isinstance(name, str) or name not in self.destinations for name in active):
231
+ raise ConfigurationError(f"Spider {spider.name} routes must reference registered destinations")
232
+ self._routes[spider.name] = tuple(dict.fromkeys(active))
233
+ self.spiders[spider.name] = spider
234
+ if not self.spiders:
235
+ raise ConfigurationError("Register at least one spider")
236
+ if any(name not in self.spiders for name in routes):
237
+ raise ConfigurationError("Routes reference an unknown spider")
238
+ self.notifiers = dict(notifiers) if notifiers is not None else {}
239
+ if any(name not in self.spiders for name in self.notifiers):
240
+ raise ConfigurationError("Notifiers reference an unknown spider")
241
+ self.settings = settings if settings is not None else Settings()
242
+ self.scheduler = scheduler if scheduler is not None else Slots()
243
+ self.store = store
244
+ self.sessions = sessions
245
+ self.logger = logger
246
+ self._runtime: _Runtime | None = None
247
+
248
+ @staticmethod
249
+ def cli_parser(*, description: str | None = None) -> argparse.ArgumentParser:
250
+ return cli_parser(description=description)
251
+
252
+ def stop(self) -> None:
253
+ """Request shutdown from the app's event loop (CLI signals call this)."""
254
+ if self._runtime is not None:
255
+ self._runtime.stop.set()
256
+
257
+ def _select(self, name: str | None) -> Spider:
258
+ if name is None and len(self.spiders) == 1:
259
+ return next(iter(self.spiders.values()))
260
+ if name is None:
261
+ raise ConfigurationError("Select one spider with --spider or run(spider=...)")
262
+ if name not in self.spiders:
263
+ raise ConfigurationError(f"Unknown spider: {name}")
264
+ return self.spiders[name]
265
+
266
+
267
+ def _validate_settings(self, settings: Settings) -> None:
268
+ settings.validate()
269
+ if self.logger is not None and settings.log_file is not None:
270
+ raise ConfigurationError("Supply a logger or a log_file")
271
+
272
+
273
+ async def run(self, *, spider: str | None = None, once: bool = False) -> None:
274
+ """Run explicit Python configuration without a CLI/environment overlay."""
275
+ await self._run(spider=spider, once=once, settings=self.settings, scheduler=self.scheduler, store=self.store)
276
+
277
+ async def _run(
278
+ self, *, spider: str | None, once: bool, settings: Settings,
279
+ scheduler: Scheduler, store: Store | None, cli: bool = False,
280
+ ) -> int:
281
+ if self._runtime is not None:
282
+ raise RuntimeError("This App is already running")
283
+ self._validate_settings(settings)
284
+ selected = self._select(spider)
285
+ path = Path(settings.log_file) if settings.log_file is not None else None
286
+ context = {"project": self.project_name, "spider": selected.name}
287
+ if settings.instance is not None:
288
+ context["instance"] = settings.instance
289
+ try:
290
+ async with run_logging(settings.log_level, path, settings.log_backup_count, self.logger, **context) as logger:
291
+ runtime = _Runtime(
292
+ project_name=self.project_name, spider=selected,
293
+ destinations={name: self.destinations[name] for name in self._routes[selected.name]},
294
+ settings=settings, scheduler=scheduler,
295
+ store=store if store is not None else MemoryStore(settings.dedup_size),
296
+ sessions=self.sessions if self.sessions is not None else Sessions(), once=once,
297
+ dedup_scope=selected.name if self.dedup_scope is None else self.dedup_scope,
298
+ notifier=self.notifiers.get(selected.name), logger=logger,
299
+ )
300
+ self._runtime = runtime
301
+ try:
302
+ await runtime.run()
303
+ except Exception:
304
+ if not cli:
305
+ raise
306
+ logger.exception("Application failed")
307
+ return 1
308
+ finally:
309
+ self._runtime = None
310
+ return 0
311
+
312
+ def cli(
313
+ self, argv: Sequence[str] | None = None, *, args: argparse.Namespace | None = None,
314
+ parser: argparse.ArgumentParser | None = None,
315
+ ) -> None:
316
+ """Run a script, optionally using a project-extended parser/namespace."""
317
+ if argv is not None and args is not None:
318
+ raise TypeError("Supply argv or an already parsed namespace, not both")
319
+ parser = parser if parser is not None else self.cli_parser()
320
+ args = args if args is not None else parser.parse_args(argv)
321
+ try:
322
+ overrides = {name: getattr(args, name) for name in ("concurrency", "dedup_size", "log_level", "log_file", "log_backup_count", "proxy") if hasattr(args, name)}
323
+ settings = replace(self.settings, **overrides)
324
+ self._validate_settings(settings)
325
+ spider = getattr(args, "spider", None)
326
+ self._select(spider)
327
+ scheduler = self.scheduler
328
+ if hasattr(args, "interval"):
329
+ if type(scheduler) is not Slots:
330
+ raise ConfigurationError("--interval / ZENX_LITE_INTERVAL require the Slots scheduler")
331
+ scheduler = Slots(args.interval)
332
+ store = self.store
333
+ if hasattr(args, "dedup_size") and store is not None:
334
+ if type(store) is not MemoryStore:
335
+ raise ConfigurationError("--dedup-size / ZENX_LITE_DEDUP_SIZE require the built-in MemoryStore")
336
+ store = MemoryStore(settings.dedup_size)
337
+ except ConfigurationError as exc:
338
+ parser.error(str(exc))
339
+ try:
340
+ self._cli_loop(spider, getattr(args, "once", False), settings, scheduler, store)
341
+ except Exception:
342
+ make_logger(settings.log_level).exception("Application failed")
343
+ raise SystemExit(1) from None
344
+
345
+ def _cli_loop(
346
+ self, spider: str | None, once: bool, settings: Settings,
347
+ scheduler: Scheduler, store: Store | None,
348
+ ) -> None:
349
+ try:
350
+ asyncio.get_running_loop()
351
+ except RuntimeError:
352
+ pass
353
+ else:
354
+ raise RuntimeError("app.cli() owns its loop; use await app.run() inside an event loop")
355
+ loop = asyncio.new_event_loop()
356
+ previous: dict[signal.Signals, Any] = {}
357
+ received: int | None = None
358
+
359
+ def stop(signum: signal.Signals) -> None:
360
+ nonlocal received
361
+ received = signum
362
+ self.stop()
363
+
364
+ try:
365
+ asyncio.set_event_loop(loop)
366
+ for signum in (signal.SIGINT, signal.SIGTERM):
367
+ try:
368
+ handler = signal.getsignal(signum)
369
+ loop.add_signal_handler(signum, stop, signum)
370
+ previous[signum] = handler
371
+ except (NotImplementedError, RuntimeError):
372
+ pass
373
+ status = loop.run_until_complete(self._run(spider=spider, once=once, settings=settings, scheduler=scheduler, store=store, cli=True))
374
+ finally:
375
+ for signum, handler in previous.items():
376
+ loop.remove_signal_handler(signum)
377
+ signal.signal(signum, handler)
378
+ # Do not introduce asyncio.run's unbounded cancellation/executor join
379
+ # after the runtime's shutdown deadline has already expired.
380
+ for task in asyncio.all_tasks(loop):
381
+ task.cancel()
382
+ task.add_done_callback(_consume)
383
+ for _ in range(3):
384
+ loop.run_until_complete(asyncio.sleep(0))
385
+ loop.close()
386
+ asyncio.set_event_loop(None)
387
+ if received is not None:
388
+ raise SystemExit(128 + received)
389
+ if status:
390
+ raise SystemExit(status)
zenx_lite/config.py ADDED
@@ -0,0 +1,68 @@
1
+ """Runtime settings and the deliberately small CLI/environment overlay."""
2
+
3
+ lazy import argparse
4
+ import os
5
+ from collections.abc import Mapping
6
+ from dataclasses import dataclass
7
+ from pathlib import Path
8
+
9
+ from .errors import ConfigurationError
10
+ from .logging import LOG_LEVELS
11
+
12
+
13
+ @dataclass(frozen=True, slots=True)
14
+ class Settings:
15
+ concurrency: int = 1
16
+ dedup_size: int = 500
17
+ log_level: str = "INFO"
18
+ proxy: str | None = None
19
+ log_file: str | Path | None = None
20
+ log_backup_count: int = 1
21
+ instance: str | None = None
22
+
23
+ def validate(self) -> None:
24
+ for name in ("concurrency", "dedup_size"):
25
+ value = getattr(self, name)
26
+ if type(value) is not int or value <= 0:
27
+ raise ConfigurationError(f"{name} must be a positive integer")
28
+ if not isinstance(self.log_level, str) or self.log_level.upper() not in LOG_LEVELS:
29
+ raise ConfigurationError("log_level must be a standard logging level")
30
+ if self.proxy is not None and (not isinstance(self.proxy, str) or not self.proxy.strip()):
31
+ raise ConfigurationError("proxy must be a nonempty proxy URL or None")
32
+ if self.log_file is not None and (not isinstance(self.log_file, (str, Path)) or not str(self.log_file).strip()):
33
+ raise ConfigurationError("log_file must be a nonempty path or None")
34
+ if type(self.log_backup_count) is not int or self.log_backup_count < 0:
35
+ raise ConfigurationError("log_backup_count must be a nonnegative integer")
36
+ if self.instance is not None and (not isinstance(self.instance, str) or not self.instance.strip()):
37
+ raise ConfigurationError("instance must be a nonempty string or None")
38
+
39
+
40
+ def cli_parser(
41
+ *, description: str | None = None, environ: Mapping[str, str] | None = None
42
+ ) -> argparse.ArgumentParser:
43
+ """Return a parser projects can extend before constructing their spiders.
44
+
45
+ Absent options stay absent so Python configuration can supply their values.
46
+ argparse converts environment defaults only when a CLI value doesn't replace
47
+ them. An invalid environment value therefore cannot override a valid flag.
48
+ """
49
+ env = os.environ if environ is None else environ
50
+ parser = argparse.ArgumentParser(description=description)
51
+ options: tuple[tuple[str, str, type, str], ...] = (
52
+ ("spider", "ZENX_LITE_SPIDER", str, "Spider to run"),
53
+ ("concurrency", "ZENX_LITE_CONCURRENCY", int, "Number of crawl workers"),
54
+ ("interval", "ZENX_LITE_INTERVAL", float, "Slots interval in seconds"),
55
+ ("log-level", "ZENX_LITE_LOG_LEVEL", str, "Logging level"),
56
+ ("log-file", "ZENX_LITE_LOG_FILE", str, "Write logs to a rotating file"),
57
+ ("log-backup-count", "ZENX_LITE_LOG_BACKUP_COUNT", int, "Number of rotated files to retain (0 keeps all)"),
58
+ ("dedup-size", "ZENX_LITE_DEDUP_SIZE", int, "Remembered item ID capacity"),
59
+ ("proxy", "ZENX_LITE_PROXY", str, "Proxy URL for managed sessions"),
60
+ )
61
+ for flag, variable, value_type, help_text in options:
62
+ default = env.get(variable, None if flag == "spider" else argparse.SUPPRESS)
63
+ parser.add_argument(
64
+ f"--{flag}", type=value_type, default=default,
65
+ help=f"{help_text} (environment: {variable})",
66
+ )
67
+ parser.add_argument("--once", action="store_true", help="Run one crawl, then shut down")
68
+ return parser