zenx-lite-python 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,359 @@
1
+ """Versioned project configuration, explicit resolution, and application construction."""
2
+
3
+ import inspect
4
+ import math
5
+ import sys
6
+ from collections.abc import Callable, Collection, Mapping, Sequence
7
+ from copy import deepcopy
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from .app import App
12
+ from .config import Settings
13
+ from .crawler import Spider
14
+ from .destinations.base import Destination, Transform
15
+ from .errors import ConfigurationError
16
+ from .notifiers.base import Notifier
17
+ from .scheduler import Slots
18
+ from .sessions import Sessions
19
+ from .store import MemoryStore, Store
20
+
21
+ type Data = dict[str, Any]
22
+ type SpiderFactory = Callable[..., Spider]
23
+
24
+ _SECTIONS = {"inputs", "runtime", "scheduler", "sessions", "dedup", "destinations", "notifier"}
25
+ _DEFAULTS: Data = {
26
+ "inputs": {},
27
+ "runtime": {"concurrency": 1, "log_level": "INFO", "proxy": None},
28
+ "scheduler": {"type": "slots", "interval": 1.0},
29
+ "sessions": {"options": {}, "proxies": []},
30
+ "dedup": {"scope": None, "store": {"type": "memory", "capacity": 500}},
31
+ "notifier": None,
32
+ }
33
+
34
+
35
+ def _mapping(value: Any, path: str, allowed: set[str] | None = None) -> Data:
36
+ if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
37
+ raise ConfigurationError(f"{path} must be a mapping with string keys")
38
+ if allowed is not None and (unknown := value.keys() - allowed):
39
+ raise ConfigurationError(f"{path} has unknown fields: {', '.join(sorted(unknown))}")
40
+ return value
41
+
42
+
43
+ def _text(value: Any, path: str) -> None:
44
+ if not isinstance(value, str) or not value.strip():
45
+ raise ConfigurationError(f"{path} must be a nonblank string")
46
+
47
+
48
+ def _positive(value: Any, path: str, *, integer: bool = False) -> None:
49
+ valid_type = type(value) is int if integer else type(value) in (int, float)
50
+ if not valid_type or (type(value) is float and not math.isfinite(value)) or value <= 0:
51
+ kind = "integer" if integer else "number"
52
+ raise ConfigurationError(f"{path} must be a positive finite {kind}")
53
+
54
+
55
+ def _names(value: Any, path: str) -> None:
56
+ if not isinstance(value, list):
57
+ raise ConfigurationError(f"{path} must be a list of names")
58
+ for name in value:
59
+ _text(name, path)
60
+ if len(value) != len(set(value)):
61
+ raise ConfigurationError(f"{path} must not contain duplicates")
62
+
63
+
64
+ def _data(value: Any, path: str = "config", parents: frozenset[int] = frozenset()) -> Any:
65
+ if isinstance(value, (dict, list)):
66
+ if id(value) in parents:
67
+ raise ConfigurationError(f"{path} contains a recursive reference")
68
+ parents = parents | {id(value)}
69
+ if isinstance(value, dict):
70
+ _mapping(value, path)
71
+ return {key: _data(item, f"{path}.{key}", parents) for key, item in value.items()}
72
+ return [_data(item, path, parents) for item in value]
73
+ if value is None or type(value) in (str, bool, int):
74
+ return value
75
+ if type(value) is float and math.isfinite(value):
76
+ return value
77
+ raise ConfigurationError(f"{path} must contain only plain data values")
78
+
79
+
80
+ def _merge(base: Data, override: Data, path: tuple[str, ...] = ()) -> Data:
81
+ if path in {("scheduler",), ("dedup", "store")} and "type" in override and override["type"] != base.get("type"):
82
+ return deepcopy(override)
83
+ result = deepcopy(base)
84
+ for key, value in override.items():
85
+ if isinstance(value, dict) and isinstance(result.get(key), dict):
86
+ result[key] = _merge(result[key], value, (*path, key))
87
+ else:
88
+ result[key] = deepcopy(value)
89
+ return result
90
+
91
+
92
+ def _options(data: Data, path: str, *, complete: bool = False) -> None:
93
+ _mapping(data, path, _SECTIONS)
94
+ if "inputs" in data:
95
+ _mapping(data["inputs"], f"{path}.inputs")
96
+ if "runtime" in data:
97
+ runtime = _mapping(data["runtime"], f"{path}.runtime", {"concurrency", "log_level", "proxy"})
98
+ if "concurrency" in runtime:
99
+ _positive(runtime["concurrency"], f"{path}.runtime.concurrency", integer=True)
100
+ if "log_level" in runtime:
101
+ _text(runtime["log_level"], f"{path}.runtime.log_level")
102
+ Settings(log_level=runtime["log_level"]).validate()
103
+ if runtime.get("proxy") is not None:
104
+ _text(runtime["proxy"], f"{path}.runtime.proxy")
105
+ if "scheduler" in data:
106
+ scheduler = _mapping(data["scheduler"], f"{path}.scheduler", {"type", "interval"})
107
+ if scheduler.get("type", "slots") != "slots":
108
+ raise ConfigurationError(f"{path}.scheduler.type must be slots")
109
+ if "interval" in scheduler:
110
+ _positive(scheduler["interval"], f"{path}.scheduler.interval")
111
+ if "sessions" in data:
112
+ sessions = _mapping(data["sessions"], f"{path}.sessions", {"options", "proxies"})
113
+ options = _mapping(sessions.get("options", {}), f"{path}.sessions.options")
114
+ if options.get("retry", 0) != 0:
115
+ raise ConfigurationError(f"{path}.sessions.options.retry must be zero")
116
+ if "proxies" in sessions:
117
+ _names(sessions["proxies"], f"{path}.sessions.proxies")
118
+ if "dedup" in data:
119
+ dedup = _mapping(data["dedup"], f"{path}.dedup", {"scope", "store"})
120
+ if dedup.get("scope") is not None:
121
+ _text(dedup["scope"], f"{path}.dedup.scope")
122
+ if "store" in dedup:
123
+ store_path = f"{path}.dedup.store"
124
+ store = _mapping(dedup["store"], store_path, {"type", "capacity", "url", "ttl_seconds"})
125
+ kind = store.get("type", "memory")
126
+ if kind not in ("memory", "redis"):
127
+ raise ConfigurationError(f"{store_path}.type must be memory or redis")
128
+ if "capacity" in store:
129
+ _positive(store["capacity"], f"{store_path}.capacity", integer=True)
130
+ if "url" in store:
131
+ _text(store["url"], f"{store_path}.url")
132
+ if not store["url"].startswith(("redis://", "rediss://", "unix://")):
133
+ raise ConfigurationError(f"{store_path}.url must use redis://, rediss://, or unix://")
134
+ if store.get("ttl_seconds") is not None:
135
+ _positive(store["ttl_seconds"], f"{store_path}.ttl_seconds", integer=True)
136
+ if complete:
137
+ fields = {"type", "capacity"} if kind == "memory" else {"type", "url", "ttl_seconds"}
138
+ _mapping(store, store_path, fields)
139
+ if kind == "redis" and not {"url", "ttl_seconds"} <= store.keys():
140
+ raise ConfigurationError(f"{store_path} requires url and explicit ttl_seconds")
141
+ if "destinations" in data:
142
+ _names(data["destinations"], f"{path}.destinations")
143
+ if data.get("notifier") is not None:
144
+ notifier_path = f"{path}.notifier"
145
+ notifier = _mapping(data["notifier"], notifier_path, {"app_id", "instance"})
146
+ if complete or "app_id" in notifier:
147
+ _text(notifier.get("app_id"), f"{notifier_path}.app_id")
148
+ if notifier.get("instance") is not None:
149
+ _text(notifier["instance"], f"{notifier_path}.instance")
150
+ if complete and "destinations" not in data:
151
+ raise ConfigurationError(f"{path}.destinations must be explicitly configured (use [] for no delivery)")
152
+
153
+
154
+ def _destination(data: Any, path: str) -> Data:
155
+ data = _mapping(data, path)
156
+ kind = data.get("type")
157
+ if kind == "jsonl":
158
+ _mapping(data, path, {"type", "path", "stream"})
159
+ if ("path" in data) == ("stream" in data):
160
+ raise ConfigurationError(f"{path} requires exactly one of path or stream")
161
+ if "path" in data:
162
+ _text(data["path"], f"{path}.path")
163
+ elif data["stream"] not in ("stdout", "stderr"):
164
+ raise ConfigurationError(f"{path}.stream must be stdout or stderr")
165
+ elif kind == "synoptic_source":
166
+ _mapping(data, path, {"type", "endpoint", "source_id", "api_key", "reconnect_delay", "connect_options"})
167
+ for field in ("endpoint", "source_id", "api_key"):
168
+ _text(data.get(field), f"{path}.{field}")
169
+ if not data["endpoint"].startswith(("ws://", "wss://")):
170
+ raise ConfigurationError(f"{path}.endpoint must use ws:// or wss://")
171
+ _positive(data.get("reconnect_delay", 1.0), f"{path}.reconnect_delay")
172
+ _mapping(data.get("connect_options", {}), f"{path}.connect_options")
173
+ else:
174
+ raise ConfigurationError(f"{path}.type must be jsonl or synoptic_source")
175
+ return data
176
+
177
+
178
+ class ProjectConfig:
179
+ """Validated project data; resolution returns an independent, reloadable snapshot."""
180
+
181
+ def __init__(self, data: Mapping[str, Any]) -> None:
182
+ if not isinstance(data, Mapping):
183
+ raise ConfigurationError("config must be a mapping")
184
+ data = _mapping(_data(dict(data)), "config", {"schema_version", "project_name", "defaults", "destinations", "spiders"})
185
+ if type(data.get("schema_version")) is not int or data["schema_version"] != 1:
186
+ raise ConfigurationError("schema_version must be 1")
187
+ _text(data.get("project_name"), "project_name")
188
+ self._project_name: str = data["project_name"]
189
+ definitions = _mapping(data.get("destinations", {}), "destinations")
190
+ self._destinations: Data = {}
191
+ for name, definition in definitions.items():
192
+ _text(name, "destinations key")
193
+ self._destinations[name] = _destination(definition, f"destinations.{name}")
194
+ defaults = _mapping(data.get("defaults", {}), "defaults")
195
+ _options(defaults, "defaults")
196
+ defaults = _merge(_DEFAULTS, defaults)
197
+ entries = _mapping(data.get("spiders"), "spiders")
198
+ if not entries:
199
+ raise ConfigurationError("spiders must contain at least one entry")
200
+ self._spiders: Data = {}
201
+ for name, entry in entries.items():
202
+ _text(name, "spiders key")
203
+ path = f"spiders.{name}"
204
+ entry = _mapping(entry, path, _SECTIONS | {"enabled"})
205
+ enabled = entry.get("enabled", True)
206
+ if type(enabled) is not bool:
207
+ raise ConfigurationError(f"{path}.enabled must be a boolean")
208
+ options = {key: value for key, value in entry.items() if key != "enabled"}
209
+ _options(options, path)
210
+ effective = _merge(defaults, options)
211
+ if enabled:
212
+ _options(effective, path, complete=True)
213
+ if any(destination not in self._destinations for destination in effective["destinations"]):
214
+ raise ConfigurationError(f"{path}.destinations references an undefined destination")
215
+ self._spiders[name] = {"enabled": enabled, **effective}
216
+
217
+ @property
218
+ def project_name(self) -> str:
219
+ return self._project_name
220
+
221
+ @property
222
+ def enabled_spiders(self) -> tuple[str, ...]:
223
+ return tuple(name for name, entry in self._spiders.items() if entry["enabled"])
224
+
225
+ def resolve(self, spider: str | None = None) -> Data:
226
+ if spider is None:
227
+ if len(self.enabled_spiders) != 1:
228
+ raise ConfigurationError("Select one enabled spider explicitly")
229
+ spider = self.enabled_spiders[0]
230
+ if spider not in self._spiders or not self._spiders[spider]["enabled"]:
231
+ raise ConfigurationError("Selected spider is unknown or disabled")
232
+ entry = self._spiders[spider]
233
+ return deepcopy({
234
+ "schema_version": 1, "project_name": self.project_name,
235
+ "destinations": {name: self._destinations[name] for name in entry["destinations"]},
236
+ "spiders": {spider: entry},
237
+ })
238
+
239
+
240
+ def load_config(path: str | Path) -> ProjectConfig:
241
+ try:
242
+ import yaml
243
+ except ImportError as exc:
244
+ raise ImportError("YAML loading requires zenx-lite-python's optional 'config' extra") from exc
245
+
246
+ class UniqueLoader(yaml.SafeLoader):
247
+ def construct_mapping(self, node: Any, deep: bool = False) -> Data:
248
+ if not isinstance(node, yaml.MappingNode):
249
+ raise ConfigurationError(f"Expected a YAML mapping (line {node.start_mark.line + 1})")
250
+ result: Data = {}
251
+ for key_node, value_node in node.value:
252
+ key = self.construct_object(key_node, deep=deep)
253
+ if not isinstance(key, str) or key in result:
254
+ raise ConfigurationError(f"YAML mappings require unique string keys (line {key_node.start_mark.line + 1})")
255
+ result[key] = self.construct_object(value_node, deep=deep)
256
+ return result
257
+
258
+ try:
259
+ with Path(path).open(encoding="utf-8") as stream:
260
+ data = yaml.load(stream, Loader=UniqueLoader)
261
+ except yaml.YAMLError as exc:
262
+ mark = getattr(exc, "problem_mark", None)
263
+ location = f" at line {mark.line + 1}, column {mark.column + 1}" if mark is not None else ""
264
+ raise ConfigurationError(f"Invalid YAML configuration{location}") from None
265
+ except UnicodeError:
266
+ raise ConfigurationError("Configuration must be UTF-8 text") from None
267
+ return ProjectConfig(_mapping(data, "config"))
268
+
269
+
270
+ def build_app(
271
+ config: ProjectConfig, *, spiders: Mapping[str, SpiderFactory],
272
+ spider: str | None = None, transforms: Mapping[str, Transform] | None = None,
273
+ ) -> App:
274
+ resolved = config.resolve(spider)
275
+ name, options = next(iter(resolved["spiders"].items()))
276
+ if name not in spiders or not callable(spiders[name]):
277
+ raise ConfigurationError(f"No registered factory for spider {name}")
278
+ factory = spiders[name]
279
+ try:
280
+ inspect.signature(factory).bind(**options["inputs"])
281
+ except (TypeError, ValueError) as exc:
282
+ raise ConfigurationError(f"spiders.{name}.inputs do not match the registered factory: {exc}") from None
283
+ instance = factory(**deepcopy(options["inputs"]))
284
+ if instance.name != name:
285
+ raise ConfigurationError(f"Registered factory must return a spider named {name}")
286
+ allowed = instance.destinations
287
+ if isinstance(allowed, str) or not isinstance(allowed, Collection) or any(route not in allowed for route in options["destinations"]):
288
+ raise ConfigurationError(f"spiders.{name}.destinations must be allowed by the spider")
289
+ transforms = dict(transforms) if transforms is not None else {}
290
+ if any(not isinstance(key, str) or not key.strip() or not callable(value) for key, value in transforms.items()):
291
+ raise ConfigurationError("Transforms must map destination names to callables")
292
+ destinations: dict[str, Destination] = {}
293
+ for route, definition in resolved["destinations"].items():
294
+ kwargs = {key: value for key, value in definition.items() if key != "type"}
295
+ if definition["type"] == "jsonl":
296
+ from .destinations.jsonl import JSONLines
297
+ target = kwargs["path"] if "path" in kwargs else getattr(sys, kwargs["stream"])
298
+ destinations[route] = JSONLines(target, transform=transforms.get(route))
299
+ else:
300
+ from .destinations.synoptic import SourceDestination
301
+ destinations[route] = SourceDestination(**kwargs, transform=transforms.get(route))
302
+ storage = options["dedup"]["store"]
303
+ store: Store
304
+ if storage["type"] == "memory":
305
+ store = MemoryStore(storage.get("capacity", 500))
306
+ else:
307
+ from .stores.redis import RedisStore
308
+ store = RedisStore(storage["url"], ttl_seconds=storage["ttl_seconds"])
309
+ notifiers: dict[str, Notifier] = {}
310
+ if options["notifier"] is not None:
311
+ from .notifiers.magent import MagentNotifier
312
+ notifiers[name] = MagentNotifier(
313
+ app_id=options["notifier"]["app_id"], instance=options["notifier"].get("instance"),
314
+ )
315
+ return App(
316
+ project_name=config.project_name, spiders=[instance], destinations=destinations,
317
+ settings=Settings(**options["runtime"], dedup_size=storage.get("capacity", 500)),
318
+ scheduler=Slots(interval=options["scheduler"]["interval"]),
319
+ sessions=Sessions(**options["sessions"]), store=store,
320
+ dedup_scope=options["dedup"]["scope"], routes={name: options["destinations"]},
321
+ notifiers=notifiers,
322
+ )
323
+
324
+
325
+ def run_project(
326
+ *, spiders: Mapping[str, SpiderFactory], transforms: Mapping[str, Transform] | None = None,
327
+ argv: Sequence[str] | None = None,
328
+ ) -> None:
329
+ """Load explicit project configuration and delegate execution to App.cli()."""
330
+ import argparse
331
+ parser = argparse.ArgumentParser(description="Run one configured spider.")
332
+ parser.add_argument("--config", required=True, help="YAML configuration path")
333
+ parser.add_argument("--spider", help="Enabled spider to run; optional when only one is enabled")
334
+ parser.add_argument("--once", action="store_true", help="Run one crawl, then shut down")
335
+ args = parser.parse_args(argv)
336
+ try:
337
+ app = build_app(load_config(args.config), spider=args.spider, spiders=spiders, transforms=transforms)
338
+ except (ConfigurationError, OSError, ImportError) as exc:
339
+ parser.error(str(exc))
340
+ app.cli(args=args, parser=parser)
341
+
342
+
343
+ def main() -> None:
344
+ import argparse
345
+ parser = argparse.ArgumentParser(description="Validate a Lite project configuration without starting spiders.")
346
+ parser.add_argument("config", help="YAML configuration path")
347
+ parser.add_argument("--spider", help="Also validate selection of this enabled spider")
348
+ args = parser.parse_args()
349
+ try:
350
+ config = load_config(args.config)
351
+ if args.spider is not None:
352
+ config.resolve(args.spider)
353
+ except (ConfigurationError, OSError, ImportError) as exc:
354
+ parser.error(str(exc))
355
+ print(f"Valid configuration; enabled spiders: {', '.join(config.enabled_spiders) or '(none)'}")
356
+
357
+
358
+ if __name__ == "__main__":
359
+ main()
zenx_lite/crawler.py ADDED
@@ -0,0 +1,69 @@
1
+ """The per-crawl handle; spiders need no framework base class."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import logging
7
+ from collections.abc import AsyncIterator, Collection, Coroutine
8
+ from contextlib import asynccontextmanager
9
+ from typing import Any, Protocol
10
+
11
+ from curl_cffi import Response as CurlResponse
12
+
13
+ from . import http
14
+ from .config import Settings
15
+ from .delivery import Delivery
16
+ from .errors import CrawlFinished
17
+ from .http import Response
18
+ from .sessions import Session
19
+
20
+
21
+ class Spider(Protocol):
22
+ name: str
23
+ destinations: Collection[str]
24
+
25
+ async def crawl(self, crawler: Crawler) -> None: ...
26
+
27
+
28
+ class Crawler:
29
+ def __init__(
30
+ self, *, worker_id: int, settings: Settings, logger: logging.Logger,
31
+ session: Session, tasks: asyncio.TaskGroup, delivery: Delivery,
32
+ ) -> None:
33
+ self.worker_id = worker_id
34
+ self.settings = settings
35
+ self.logger = logger
36
+ self._session = session
37
+ self._tasks = tasks
38
+ self._delivery = delivery
39
+ self._active = True
40
+
41
+ def _require_active(self) -> None:
42
+ if not self._active:
43
+ raise CrawlFinished("This crawler's crawl has finished")
44
+
45
+ async def emit(
46
+ self, item_id: str, data: dict[str, Any], *, to: Collection[str] | None = None,
47
+ ) -> None:
48
+ self._require_active()
49
+ await self._delivery.emit(item_id, data, to=to)
50
+
51
+ async def request(self, url: str, method: str = "GET", **kwargs: Any) -> Response:
52
+ self._require_active()
53
+ return await http.request(self._session, url, method, **kwargs)
54
+
55
+ @asynccontextmanager
56
+ async def stream(self, url: str, method: str = "GET", **kwargs: Any) -> AsyncIterator[CurlResponse]:
57
+ self._require_active()
58
+ async with http.stream(self._session, url, method, **kwargs) as response:
59
+ yield response
60
+
61
+ def create_task[T](
62
+ self, coro: Coroutine[Any, Any, T], *, name: str | None = None,
63
+ ) -> asyncio.Task[T]:
64
+ try:
65
+ self._require_active()
66
+ return self._tasks.create_task(coro, name=name)
67
+ except RuntimeError:
68
+ coro.close()
69
+ raise
zenx_lite/delivery.py ADDED
@@ -0,0 +1,139 @@
1
+ """Immediate fan-out with independent, unbounded, best-effort senders."""
2
+
3
+ import asyncio
4
+ import logging
5
+ from collections import deque
6
+ from collections.abc import Collection, Mapping
7
+ from copy import deepcopy
8
+ from dataclasses import dataclass
9
+ from typing import Any
10
+
11
+ from .destinations.base import Destination
12
+ from .store import Store
13
+
14
+
15
+ @dataclass(frozen=True, slots=True)
16
+ class DeliveryStats:
17
+ pending: int
18
+ oldest_pending_age: float
19
+ sent: int
20
+ failures: int
21
+
22
+
23
+ class Sender:
24
+ def __init__(self, name: str, destination: Destination, logger: logging.Logger) -> None:
25
+ self.name = name
26
+ self.destination = destination
27
+ self.logger = logger
28
+ self.queue: asyncio.Queue[bytes] = asyncio.Queue()
29
+ self._times: deque[float] = deque()
30
+ self._task: asyncio.Task[None] | None = None
31
+ self._accepting = True
32
+ self._current = False
33
+ self.sent = 0
34
+ self.failures = 0
35
+
36
+ async def open(self) -> None:
37
+ await self.destination.open()
38
+ self._task = asyncio.create_task(self._run(), name=f"zenx:destination:{self.name}")
39
+
40
+ def enqueue(self, payload: bytes) -> None:
41
+ if not self._accepting:
42
+ raise RuntimeError("Destination admission is closed")
43
+ if not isinstance(payload, bytes):
44
+ raise TypeError("Destination.prepare() must return immutable bytes")
45
+ self._times.append(asyncio.get_running_loop().time())
46
+ self.queue.put_nowait(payload)
47
+
48
+ def stop_admission(self) -> None:
49
+ self._accepting = False
50
+
51
+ async def _run(self) -> None:
52
+ while True:
53
+ payload = await self.queue.get()
54
+ self._current = True
55
+ try:
56
+ await self.destination.ready()
57
+ await self.destination.send(payload)
58
+ self.sent += 1
59
+ except Exception:
60
+ self.failures += 1
61
+ self.logger.exception("Destination %s failed; payload discarded", self.name)
62
+ finally:
63
+ self._current = False
64
+ self._times.popleft()
65
+ self.queue.task_done()
66
+ # A recovered connection may accept many buffered sends without
67
+ # suspending. Keep backlog draining fair to other destinations and crawls.
68
+ await asyncio.sleep(0)
69
+
70
+ async def drain(self) -> None:
71
+ await self.queue.join()
72
+
73
+ def cancel(self) -> None:
74
+ self.stop_admission()
75
+ if self._task is not None:
76
+ self._task.cancel()
77
+ while not self.queue.empty():
78
+ self.queue.get_nowait()
79
+ self.queue.task_done()
80
+ # The in-flight entry is removed by the sender's finally clause.
81
+ while len(self._times) > int(self._current):
82
+ self._times.pop()
83
+
84
+ async def close(self) -> None:
85
+ self.cancel()
86
+ try:
87
+ if self._task is not None:
88
+ await asyncio.gather(self._task, return_exceptions=True)
89
+ finally:
90
+ await self.destination.close()
91
+
92
+ def stats(self) -> DeliveryStats:
93
+ now = asyncio.get_running_loop().time()
94
+ return DeliveryStats(
95
+ pending=len(self._times),
96
+ oldest_pending_age=max(0.0, now - self._times[0]) if self._times else 0.0,
97
+ sent=self.sent, failures=self.failures,
98
+ )
99
+
100
+
101
+ class Delivery:
102
+ def __init__(
103
+ self, project_name: str, dedup_scope: str, store: Store,
104
+ senders: Mapping[str, Sender], logger: logging.Logger,
105
+ ) -> None:
106
+ self.project_name = project_name
107
+ self.dedup_scope = dedup_scope
108
+ self.store = store
109
+ self.senders = dict(senders)
110
+ self.logger = logger
111
+ self.accepting = True
112
+
113
+ async def emit(
114
+ self, item_id: str, data: dict[str, Any], *, to: Collection[str] | None = None,
115
+ ) -> None:
116
+ if not self.accepting:
117
+ raise RuntimeError("Delivery admission is closed")
118
+ if not isinstance(item_id, str) or not item_id:
119
+ raise ValueError("item_id must be a nonempty string")
120
+ if not isinstance(data, dict):
121
+ raise TypeError("Item data must be a dictionary")
122
+ if isinstance(to, str):
123
+ raise TypeError("to must be a collection of destination names, not a string")
124
+ names = tuple(self.senders) if to is None else tuple(to)
125
+ if any(not isinstance(name, str) or name not in self.senders for name in names):
126
+ raise ValueError("Routes must be among the selected spider's configured destinations")
127
+ if not await self.store.add((self.project_name, self.dedup_scope, item_id)):
128
+ return
129
+ for name in dict.fromkeys(names):
130
+ sender = self.senders[name]
131
+ try:
132
+ # Isolate even a mutating transform from other destinations and
133
+ # from the caller. The queue only retains serialized bytes.
134
+ payload = sender.destination.prepare(item_id, deepcopy(data))
135
+ sender.enqueue(payload)
136
+ except Exception:
137
+ sender.failures += 1
138
+ self.logger.exception("Destination %s could not prepare/admit item %s", name, item_id)
139
+ await asyncio.sleep(0)
@@ -0,0 +1,5 @@
1
+ """Destinations are imported explicitly; network adapters remain optional."""
2
+
3
+ from .base import Destination
4
+
5
+ __all__ = ["Destination"]
@@ -0,0 +1,32 @@
1
+ from collections.abc import Callable
2
+ from typing import Any, Protocol
3
+
4
+ type Item = dict[str, Any]
5
+ type Transform = Callable[[Item], Item]
6
+
7
+
8
+ class Destination(Protocol):
9
+ """Each destination has one runtime-owned queue and sender.
10
+
11
+ prepare() must return owned, immutable bytes without destination I/O.
12
+ ready() may recover a connection, but must never send a payload.
13
+ send() makes one attempt; neither it nor close() may replay that attempt.
14
+ close() must be cancellation-cooperative and release resources in finally.
15
+ """
16
+
17
+ async def open(self) -> None: ...
18
+
19
+ def prepare(self, item_id: str, data: Item) -> bytes: ...
20
+
21
+ async def ready(self) -> None: ...
22
+
23
+ async def send(self, payload: bytes) -> None: ...
24
+
25
+ async def close(self) -> None: ...
26
+
27
+
28
+ def transform_item(data: Item, transform: Transform | None) -> Item:
29
+ result = data if transform is None else transform(data)
30
+ if not isinstance(result, dict):
31
+ raise TypeError("A destination transform must return a dictionary")
32
+ return result