zenx-lite-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenx_lite/__init__.py +8 -0
- zenx_lite/app.py +377 -0
- zenx_lite/config.py +56 -0
- zenx_lite/configuration.py +359 -0
- zenx_lite/crawler.py +69 -0
- zenx_lite/delivery.py +139 -0
- zenx_lite/destinations/__init__.py +5 -0
- zenx_lite/destinations/base.py +32 -0
- zenx_lite/destinations/jsonl.py +132 -0
- zenx_lite/destinations/synoptic/__init__.py +5 -0
- zenx_lite/destinations/synoptic/source.py +136 -0
- zenx_lite/errors.py +21 -0
- zenx_lite/http.py +130 -0
- zenx_lite/notifiers/__init__.py +5 -0
- zenx_lite/notifiers/base.py +15 -0
- zenx_lite/notifiers/magent.py +86 -0
- zenx_lite/scheduler.py +63 -0
- zenx_lite/sessions.py +102 -0
- zenx_lite/store.py +45 -0
- zenx_lite/stores/__init__.py +1 -0
- zenx_lite/stores/redis.py +54 -0
- zenx_lite_python-0.1.0.dist-info/METADATA +14 -0
- zenx_lite_python-0.1.0.dist-info/RECORD +24 -0
- zenx_lite_python-0.1.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
"""Versioned project configuration, explicit resolution, and application construction."""
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
import math
|
|
5
|
+
import sys
|
|
6
|
+
from collections.abc import Callable, Collection, Mapping, Sequence
|
|
7
|
+
from copy import deepcopy
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from .app import App
|
|
12
|
+
from .config import Settings
|
|
13
|
+
from .crawler import Spider
|
|
14
|
+
from .destinations.base import Destination, Transform
|
|
15
|
+
from .errors import ConfigurationError
|
|
16
|
+
from .notifiers.base import Notifier
|
|
17
|
+
from .scheduler import Slots
|
|
18
|
+
from .sessions import Sessions
|
|
19
|
+
from .store import MemoryStore, Store
|
|
20
|
+
|
|
21
|
+
type Data = dict[str, Any]
|
|
22
|
+
type SpiderFactory = Callable[..., Spider]
|
|
23
|
+
|
|
24
|
+
_SECTIONS = {"inputs", "runtime", "scheduler", "sessions", "dedup", "destinations", "notifier"}
|
|
25
|
+
_DEFAULTS: Data = {
|
|
26
|
+
"inputs": {},
|
|
27
|
+
"runtime": {"concurrency": 1, "log_level": "INFO", "proxy": None},
|
|
28
|
+
"scheduler": {"type": "slots", "interval": 1.0},
|
|
29
|
+
"sessions": {"options": {}, "proxies": []},
|
|
30
|
+
"dedup": {"scope": None, "store": {"type": "memory", "capacity": 500}},
|
|
31
|
+
"notifier": None,
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _mapping(value: Any, path: str, allowed: set[str] | None = None) -> Data:
|
|
36
|
+
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
|
37
|
+
raise ConfigurationError(f"{path} must be a mapping with string keys")
|
|
38
|
+
if allowed is not None and (unknown := value.keys() - allowed):
|
|
39
|
+
raise ConfigurationError(f"{path} has unknown fields: {', '.join(sorted(unknown))}")
|
|
40
|
+
return value
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _text(value: Any, path: str) -> None:
|
|
44
|
+
if not isinstance(value, str) or not value.strip():
|
|
45
|
+
raise ConfigurationError(f"{path} must be a nonblank string")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _positive(value: Any, path: str, *, integer: bool = False) -> None:
|
|
49
|
+
valid_type = type(value) is int if integer else type(value) in (int, float)
|
|
50
|
+
if not valid_type or (type(value) is float and not math.isfinite(value)) or value <= 0:
|
|
51
|
+
kind = "integer" if integer else "number"
|
|
52
|
+
raise ConfigurationError(f"{path} must be a positive finite {kind}")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _names(value: Any, path: str) -> None:
|
|
56
|
+
if not isinstance(value, list):
|
|
57
|
+
raise ConfigurationError(f"{path} must be a list of names")
|
|
58
|
+
for name in value:
|
|
59
|
+
_text(name, path)
|
|
60
|
+
if len(value) != len(set(value)):
|
|
61
|
+
raise ConfigurationError(f"{path} must not contain duplicates")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _data(value: Any, path: str = "config", parents: frozenset[int] = frozenset()) -> Any:
|
|
65
|
+
if isinstance(value, (dict, list)):
|
|
66
|
+
if id(value) in parents:
|
|
67
|
+
raise ConfigurationError(f"{path} contains a recursive reference")
|
|
68
|
+
parents = parents | {id(value)}
|
|
69
|
+
if isinstance(value, dict):
|
|
70
|
+
_mapping(value, path)
|
|
71
|
+
return {key: _data(item, f"{path}.{key}", parents) for key, item in value.items()}
|
|
72
|
+
return [_data(item, path, parents) for item in value]
|
|
73
|
+
if value is None or type(value) in (str, bool, int):
|
|
74
|
+
return value
|
|
75
|
+
if type(value) is float and math.isfinite(value):
|
|
76
|
+
return value
|
|
77
|
+
raise ConfigurationError(f"{path} must contain only plain data values")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _merge(base: Data, override: Data, path: tuple[str, ...] = ()) -> Data:
|
|
81
|
+
if path in {("scheduler",), ("dedup", "store")} and "type" in override and override["type"] != base.get("type"):
|
|
82
|
+
return deepcopy(override)
|
|
83
|
+
result = deepcopy(base)
|
|
84
|
+
for key, value in override.items():
|
|
85
|
+
if isinstance(value, dict) and isinstance(result.get(key), dict):
|
|
86
|
+
result[key] = _merge(result[key], value, (*path, key))
|
|
87
|
+
else:
|
|
88
|
+
result[key] = deepcopy(value)
|
|
89
|
+
return result
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _options(data: Data, path: str, *, complete: bool = False) -> None:
|
|
93
|
+
_mapping(data, path, _SECTIONS)
|
|
94
|
+
if "inputs" in data:
|
|
95
|
+
_mapping(data["inputs"], f"{path}.inputs")
|
|
96
|
+
if "runtime" in data:
|
|
97
|
+
runtime = _mapping(data["runtime"], f"{path}.runtime", {"concurrency", "log_level", "proxy"})
|
|
98
|
+
if "concurrency" in runtime:
|
|
99
|
+
_positive(runtime["concurrency"], f"{path}.runtime.concurrency", integer=True)
|
|
100
|
+
if "log_level" in runtime:
|
|
101
|
+
_text(runtime["log_level"], f"{path}.runtime.log_level")
|
|
102
|
+
Settings(log_level=runtime["log_level"]).validate()
|
|
103
|
+
if runtime.get("proxy") is not None:
|
|
104
|
+
_text(runtime["proxy"], f"{path}.runtime.proxy")
|
|
105
|
+
if "scheduler" in data:
|
|
106
|
+
scheduler = _mapping(data["scheduler"], f"{path}.scheduler", {"type", "interval"})
|
|
107
|
+
if scheduler.get("type", "slots") != "slots":
|
|
108
|
+
raise ConfigurationError(f"{path}.scheduler.type must be slots")
|
|
109
|
+
if "interval" in scheduler:
|
|
110
|
+
_positive(scheduler["interval"], f"{path}.scheduler.interval")
|
|
111
|
+
if "sessions" in data:
|
|
112
|
+
sessions = _mapping(data["sessions"], f"{path}.sessions", {"options", "proxies"})
|
|
113
|
+
options = _mapping(sessions.get("options", {}), f"{path}.sessions.options")
|
|
114
|
+
if options.get("retry", 0) != 0:
|
|
115
|
+
raise ConfigurationError(f"{path}.sessions.options.retry must be zero")
|
|
116
|
+
if "proxies" in sessions:
|
|
117
|
+
_names(sessions["proxies"], f"{path}.sessions.proxies")
|
|
118
|
+
if "dedup" in data:
|
|
119
|
+
dedup = _mapping(data["dedup"], f"{path}.dedup", {"scope", "store"})
|
|
120
|
+
if dedup.get("scope") is not None:
|
|
121
|
+
_text(dedup["scope"], f"{path}.dedup.scope")
|
|
122
|
+
if "store" in dedup:
|
|
123
|
+
store_path = f"{path}.dedup.store"
|
|
124
|
+
store = _mapping(dedup["store"], store_path, {"type", "capacity", "url", "ttl_seconds"})
|
|
125
|
+
kind = store.get("type", "memory")
|
|
126
|
+
if kind not in ("memory", "redis"):
|
|
127
|
+
raise ConfigurationError(f"{store_path}.type must be memory or redis")
|
|
128
|
+
if "capacity" in store:
|
|
129
|
+
_positive(store["capacity"], f"{store_path}.capacity", integer=True)
|
|
130
|
+
if "url" in store:
|
|
131
|
+
_text(store["url"], f"{store_path}.url")
|
|
132
|
+
if not store["url"].startswith(("redis://", "rediss://", "unix://")):
|
|
133
|
+
raise ConfigurationError(f"{store_path}.url must use redis://, rediss://, or unix://")
|
|
134
|
+
if store.get("ttl_seconds") is not None:
|
|
135
|
+
_positive(store["ttl_seconds"], f"{store_path}.ttl_seconds", integer=True)
|
|
136
|
+
if complete:
|
|
137
|
+
fields = {"type", "capacity"} if kind == "memory" else {"type", "url", "ttl_seconds"}
|
|
138
|
+
_mapping(store, store_path, fields)
|
|
139
|
+
if kind == "redis" and not {"url", "ttl_seconds"} <= store.keys():
|
|
140
|
+
raise ConfigurationError(f"{store_path} requires url and explicit ttl_seconds")
|
|
141
|
+
if "destinations" in data:
|
|
142
|
+
_names(data["destinations"], f"{path}.destinations")
|
|
143
|
+
if data.get("notifier") is not None:
|
|
144
|
+
notifier_path = f"{path}.notifier"
|
|
145
|
+
notifier = _mapping(data["notifier"], notifier_path, {"app_id", "instance"})
|
|
146
|
+
if complete or "app_id" in notifier:
|
|
147
|
+
_text(notifier.get("app_id"), f"{notifier_path}.app_id")
|
|
148
|
+
if notifier.get("instance") is not None:
|
|
149
|
+
_text(notifier["instance"], f"{notifier_path}.instance")
|
|
150
|
+
if complete and "destinations" not in data:
|
|
151
|
+
raise ConfigurationError(f"{path}.destinations must be explicitly configured (use [] for no delivery)")
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _destination(data: Any, path: str) -> Data:
|
|
155
|
+
data = _mapping(data, path)
|
|
156
|
+
kind = data.get("type")
|
|
157
|
+
if kind == "jsonl":
|
|
158
|
+
_mapping(data, path, {"type", "path", "stream"})
|
|
159
|
+
if ("path" in data) == ("stream" in data):
|
|
160
|
+
raise ConfigurationError(f"{path} requires exactly one of path or stream")
|
|
161
|
+
if "path" in data:
|
|
162
|
+
_text(data["path"], f"{path}.path")
|
|
163
|
+
elif data["stream"] not in ("stdout", "stderr"):
|
|
164
|
+
raise ConfigurationError(f"{path}.stream must be stdout or stderr")
|
|
165
|
+
elif kind == "synoptic_source":
|
|
166
|
+
_mapping(data, path, {"type", "endpoint", "source_id", "api_key", "reconnect_delay", "connect_options"})
|
|
167
|
+
for field in ("endpoint", "source_id", "api_key"):
|
|
168
|
+
_text(data.get(field), f"{path}.{field}")
|
|
169
|
+
if not data["endpoint"].startswith(("ws://", "wss://")):
|
|
170
|
+
raise ConfigurationError(f"{path}.endpoint must use ws:// or wss://")
|
|
171
|
+
_positive(data.get("reconnect_delay", 1.0), f"{path}.reconnect_delay")
|
|
172
|
+
_mapping(data.get("connect_options", {}), f"{path}.connect_options")
|
|
173
|
+
else:
|
|
174
|
+
raise ConfigurationError(f"{path}.type must be jsonl or synoptic_source")
|
|
175
|
+
return data
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
class ProjectConfig:
|
|
179
|
+
"""Validated project data; resolution returns an independent, reloadable snapshot."""
|
|
180
|
+
|
|
181
|
+
def __init__(self, data: Mapping[str, Any]) -> None:
|
|
182
|
+
if not isinstance(data, Mapping):
|
|
183
|
+
raise ConfigurationError("config must be a mapping")
|
|
184
|
+
data = _mapping(_data(dict(data)), "config", {"schema_version", "project_name", "defaults", "destinations", "spiders"})
|
|
185
|
+
if type(data.get("schema_version")) is not int or data["schema_version"] != 1:
|
|
186
|
+
raise ConfigurationError("schema_version must be 1")
|
|
187
|
+
_text(data.get("project_name"), "project_name")
|
|
188
|
+
self._project_name: str = data["project_name"]
|
|
189
|
+
definitions = _mapping(data.get("destinations", {}), "destinations")
|
|
190
|
+
self._destinations: Data = {}
|
|
191
|
+
for name, definition in definitions.items():
|
|
192
|
+
_text(name, "destinations key")
|
|
193
|
+
self._destinations[name] = _destination(definition, f"destinations.{name}")
|
|
194
|
+
defaults = _mapping(data.get("defaults", {}), "defaults")
|
|
195
|
+
_options(defaults, "defaults")
|
|
196
|
+
defaults = _merge(_DEFAULTS, defaults)
|
|
197
|
+
entries = _mapping(data.get("spiders"), "spiders")
|
|
198
|
+
if not entries:
|
|
199
|
+
raise ConfigurationError("spiders must contain at least one entry")
|
|
200
|
+
self._spiders: Data = {}
|
|
201
|
+
for name, entry in entries.items():
|
|
202
|
+
_text(name, "spiders key")
|
|
203
|
+
path = f"spiders.{name}"
|
|
204
|
+
entry = _mapping(entry, path, _SECTIONS | {"enabled"})
|
|
205
|
+
enabled = entry.get("enabled", True)
|
|
206
|
+
if type(enabled) is not bool:
|
|
207
|
+
raise ConfigurationError(f"{path}.enabled must be a boolean")
|
|
208
|
+
options = {key: value for key, value in entry.items() if key != "enabled"}
|
|
209
|
+
_options(options, path)
|
|
210
|
+
effective = _merge(defaults, options)
|
|
211
|
+
if enabled:
|
|
212
|
+
_options(effective, path, complete=True)
|
|
213
|
+
if any(destination not in self._destinations for destination in effective["destinations"]):
|
|
214
|
+
raise ConfigurationError(f"{path}.destinations references an undefined destination")
|
|
215
|
+
self._spiders[name] = {"enabled": enabled, **effective}
|
|
216
|
+
|
|
217
|
+
@property
|
|
218
|
+
def project_name(self) -> str:
|
|
219
|
+
return self._project_name
|
|
220
|
+
|
|
221
|
+
@property
|
|
222
|
+
def enabled_spiders(self) -> tuple[str, ...]:
|
|
223
|
+
return tuple(name for name, entry in self._spiders.items() if entry["enabled"])
|
|
224
|
+
|
|
225
|
+
def resolve(self, spider: str | None = None) -> Data:
|
|
226
|
+
if spider is None:
|
|
227
|
+
if len(self.enabled_spiders) != 1:
|
|
228
|
+
raise ConfigurationError("Select one enabled spider explicitly")
|
|
229
|
+
spider = self.enabled_spiders[0]
|
|
230
|
+
if spider not in self._spiders or not self._spiders[spider]["enabled"]:
|
|
231
|
+
raise ConfigurationError("Selected spider is unknown or disabled")
|
|
232
|
+
entry = self._spiders[spider]
|
|
233
|
+
return deepcopy({
|
|
234
|
+
"schema_version": 1, "project_name": self.project_name,
|
|
235
|
+
"destinations": {name: self._destinations[name] for name in entry["destinations"]},
|
|
236
|
+
"spiders": {spider: entry},
|
|
237
|
+
})
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def load_config(path: str | Path) -> ProjectConfig:
|
|
241
|
+
try:
|
|
242
|
+
import yaml
|
|
243
|
+
except ImportError as exc:
|
|
244
|
+
raise ImportError("YAML loading requires zenx-lite-python's optional 'config' extra") from exc
|
|
245
|
+
|
|
246
|
+
class UniqueLoader(yaml.SafeLoader):
|
|
247
|
+
def construct_mapping(self, node: Any, deep: bool = False) -> Data:
|
|
248
|
+
if not isinstance(node, yaml.MappingNode):
|
|
249
|
+
raise ConfigurationError(f"Expected a YAML mapping (line {node.start_mark.line + 1})")
|
|
250
|
+
result: Data = {}
|
|
251
|
+
for key_node, value_node in node.value:
|
|
252
|
+
key = self.construct_object(key_node, deep=deep)
|
|
253
|
+
if not isinstance(key, str) or key in result:
|
|
254
|
+
raise ConfigurationError(f"YAML mappings require unique string keys (line {key_node.start_mark.line + 1})")
|
|
255
|
+
result[key] = self.construct_object(value_node, deep=deep)
|
|
256
|
+
return result
|
|
257
|
+
|
|
258
|
+
try:
|
|
259
|
+
with Path(path).open(encoding="utf-8") as stream:
|
|
260
|
+
data = yaml.load(stream, Loader=UniqueLoader)
|
|
261
|
+
except yaml.YAMLError as exc:
|
|
262
|
+
mark = getattr(exc, "problem_mark", None)
|
|
263
|
+
location = f" at line {mark.line + 1}, column {mark.column + 1}" if mark is not None else ""
|
|
264
|
+
raise ConfigurationError(f"Invalid YAML configuration{location}") from None
|
|
265
|
+
except UnicodeError:
|
|
266
|
+
raise ConfigurationError("Configuration must be UTF-8 text") from None
|
|
267
|
+
return ProjectConfig(_mapping(data, "config"))
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def build_app(
|
|
271
|
+
config: ProjectConfig, *, spiders: Mapping[str, SpiderFactory],
|
|
272
|
+
spider: str | None = None, transforms: Mapping[str, Transform] | None = None,
|
|
273
|
+
) -> App:
|
|
274
|
+
resolved = config.resolve(spider)
|
|
275
|
+
name, options = next(iter(resolved["spiders"].items()))
|
|
276
|
+
if name not in spiders or not callable(spiders[name]):
|
|
277
|
+
raise ConfigurationError(f"No registered factory for spider {name}")
|
|
278
|
+
factory = spiders[name]
|
|
279
|
+
try:
|
|
280
|
+
inspect.signature(factory).bind(**options["inputs"])
|
|
281
|
+
except (TypeError, ValueError) as exc:
|
|
282
|
+
raise ConfigurationError(f"spiders.{name}.inputs do not match the registered factory: {exc}") from None
|
|
283
|
+
instance = factory(**deepcopy(options["inputs"]))
|
|
284
|
+
if instance.name != name:
|
|
285
|
+
raise ConfigurationError(f"Registered factory must return a spider named {name}")
|
|
286
|
+
allowed = instance.destinations
|
|
287
|
+
if isinstance(allowed, str) or not isinstance(allowed, Collection) or any(route not in allowed for route in options["destinations"]):
|
|
288
|
+
raise ConfigurationError(f"spiders.{name}.destinations must be allowed by the spider")
|
|
289
|
+
transforms = dict(transforms) if transforms is not None else {}
|
|
290
|
+
if any(not isinstance(key, str) or not key.strip() or not callable(value) for key, value in transforms.items()):
|
|
291
|
+
raise ConfigurationError("Transforms must map destination names to callables")
|
|
292
|
+
destinations: dict[str, Destination] = {}
|
|
293
|
+
for route, definition in resolved["destinations"].items():
|
|
294
|
+
kwargs = {key: value for key, value in definition.items() if key != "type"}
|
|
295
|
+
if definition["type"] == "jsonl":
|
|
296
|
+
from .destinations.jsonl import JSONLines
|
|
297
|
+
target = kwargs["path"] if "path" in kwargs else getattr(sys, kwargs["stream"])
|
|
298
|
+
destinations[route] = JSONLines(target, transform=transforms.get(route))
|
|
299
|
+
else:
|
|
300
|
+
from .destinations.synoptic import SourceDestination
|
|
301
|
+
destinations[route] = SourceDestination(**kwargs, transform=transforms.get(route))
|
|
302
|
+
storage = options["dedup"]["store"]
|
|
303
|
+
store: Store
|
|
304
|
+
if storage["type"] == "memory":
|
|
305
|
+
store = MemoryStore(storage.get("capacity", 500))
|
|
306
|
+
else:
|
|
307
|
+
from .stores.redis import RedisStore
|
|
308
|
+
store = RedisStore(storage["url"], ttl_seconds=storage["ttl_seconds"])
|
|
309
|
+
notifiers: dict[str, Notifier] = {}
|
|
310
|
+
if options["notifier"] is not None:
|
|
311
|
+
from .notifiers.magent import MagentNotifier
|
|
312
|
+
notifiers[name] = MagentNotifier(
|
|
313
|
+
app_id=options["notifier"]["app_id"], instance=options["notifier"].get("instance"),
|
|
314
|
+
)
|
|
315
|
+
return App(
|
|
316
|
+
project_name=config.project_name, spiders=[instance], destinations=destinations,
|
|
317
|
+
settings=Settings(**options["runtime"], dedup_size=storage.get("capacity", 500)),
|
|
318
|
+
scheduler=Slots(interval=options["scheduler"]["interval"]),
|
|
319
|
+
sessions=Sessions(**options["sessions"]), store=store,
|
|
320
|
+
dedup_scope=options["dedup"]["scope"], routes={name: options["destinations"]},
|
|
321
|
+
notifiers=notifiers,
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def run_project(
|
|
326
|
+
*, spiders: Mapping[str, SpiderFactory], transforms: Mapping[str, Transform] | None = None,
|
|
327
|
+
argv: Sequence[str] | None = None,
|
|
328
|
+
) -> None:
|
|
329
|
+
"""Load explicit project configuration and delegate execution to App.cli()."""
|
|
330
|
+
import argparse
|
|
331
|
+
parser = argparse.ArgumentParser(description="Run one configured spider.")
|
|
332
|
+
parser.add_argument("--config", required=True, help="YAML configuration path")
|
|
333
|
+
parser.add_argument("--spider", help="Enabled spider to run; optional when only one is enabled")
|
|
334
|
+
parser.add_argument("--once", action="store_true", help="Run one crawl, then shut down")
|
|
335
|
+
args = parser.parse_args(argv)
|
|
336
|
+
try:
|
|
337
|
+
app = build_app(load_config(args.config), spider=args.spider, spiders=spiders, transforms=transforms)
|
|
338
|
+
except (ConfigurationError, OSError, ImportError) as exc:
|
|
339
|
+
parser.error(str(exc))
|
|
340
|
+
app.cli(args=args, parser=parser)
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def main() -> None:
|
|
344
|
+
import argparse
|
|
345
|
+
parser = argparse.ArgumentParser(description="Validate a Lite project configuration without starting spiders.")
|
|
346
|
+
parser.add_argument("config", help="YAML configuration path")
|
|
347
|
+
parser.add_argument("--spider", help="Also validate selection of this enabled spider")
|
|
348
|
+
args = parser.parse_args()
|
|
349
|
+
try:
|
|
350
|
+
config = load_config(args.config)
|
|
351
|
+
if args.spider is not None:
|
|
352
|
+
config.resolve(args.spider)
|
|
353
|
+
except (ConfigurationError, OSError, ImportError) as exc:
|
|
354
|
+
parser.error(str(exc))
|
|
355
|
+
print(f"Valid configuration; enabled spiders: {', '.join(config.enabled_spiders) or '(none)'}")
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
if __name__ == "__main__":
|
|
359
|
+
main()
|
zenx_lite/crawler.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""The per-crawl handle; spiders need no framework base class."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
from collections.abc import AsyncIterator, Collection, Coroutine
|
|
8
|
+
from contextlib import asynccontextmanager
|
|
9
|
+
from typing import Any, Protocol
|
|
10
|
+
|
|
11
|
+
from curl_cffi import Response as CurlResponse
|
|
12
|
+
|
|
13
|
+
from . import http
|
|
14
|
+
from .config import Settings
|
|
15
|
+
from .delivery import Delivery
|
|
16
|
+
from .errors import CrawlFinished
|
|
17
|
+
from .http import Response
|
|
18
|
+
from .sessions import Session
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Spider(Protocol):
|
|
22
|
+
name: str
|
|
23
|
+
destinations: Collection[str]
|
|
24
|
+
|
|
25
|
+
async def crawl(self, crawler: Crawler) -> None: ...
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Crawler:
|
|
29
|
+
def __init__(
|
|
30
|
+
self, *, worker_id: int, settings: Settings, logger: logging.Logger,
|
|
31
|
+
session: Session, tasks: asyncio.TaskGroup, delivery: Delivery,
|
|
32
|
+
) -> None:
|
|
33
|
+
self.worker_id = worker_id
|
|
34
|
+
self.settings = settings
|
|
35
|
+
self.logger = logger
|
|
36
|
+
self._session = session
|
|
37
|
+
self._tasks = tasks
|
|
38
|
+
self._delivery = delivery
|
|
39
|
+
self._active = True
|
|
40
|
+
|
|
41
|
+
def _require_active(self) -> None:
|
|
42
|
+
if not self._active:
|
|
43
|
+
raise CrawlFinished("This crawler's crawl has finished")
|
|
44
|
+
|
|
45
|
+
async def emit(
|
|
46
|
+
self, item_id: str, data: dict[str, Any], *, to: Collection[str] | None = None,
|
|
47
|
+
) -> None:
|
|
48
|
+
self._require_active()
|
|
49
|
+
await self._delivery.emit(item_id, data, to=to)
|
|
50
|
+
|
|
51
|
+
async def request(self, url: str, method: str = "GET", **kwargs: Any) -> Response:
|
|
52
|
+
self._require_active()
|
|
53
|
+
return await http.request(self._session, url, method, **kwargs)
|
|
54
|
+
|
|
55
|
+
@asynccontextmanager
|
|
56
|
+
async def stream(self, url: str, method: str = "GET", **kwargs: Any) -> AsyncIterator[CurlResponse]:
|
|
57
|
+
self._require_active()
|
|
58
|
+
async with http.stream(self._session, url, method, **kwargs) as response:
|
|
59
|
+
yield response
|
|
60
|
+
|
|
61
|
+
def create_task[T](
|
|
62
|
+
self, coro: Coroutine[Any, Any, T], *, name: str | None = None,
|
|
63
|
+
) -> asyncio.Task[T]:
|
|
64
|
+
try:
|
|
65
|
+
self._require_active()
|
|
66
|
+
return self._tasks.create_task(coro, name=name)
|
|
67
|
+
except RuntimeError:
|
|
68
|
+
coro.close()
|
|
69
|
+
raise
|
zenx_lite/delivery.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Immediate fan-out with independent, unbounded, best-effort senders."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
from collections import deque
|
|
6
|
+
from collections.abc import Collection, Mapping
|
|
7
|
+
from copy import deepcopy
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from .destinations.base import Destination
|
|
12
|
+
from .store import Store
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True, slots=True)
|
|
16
|
+
class DeliveryStats:
|
|
17
|
+
pending: int
|
|
18
|
+
oldest_pending_age: float
|
|
19
|
+
sent: int
|
|
20
|
+
failures: int
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Sender:
|
|
24
|
+
def __init__(self, name: str, destination: Destination, logger: logging.Logger) -> None:
|
|
25
|
+
self.name = name
|
|
26
|
+
self.destination = destination
|
|
27
|
+
self.logger = logger
|
|
28
|
+
self.queue: asyncio.Queue[bytes] = asyncio.Queue()
|
|
29
|
+
self._times: deque[float] = deque()
|
|
30
|
+
self._task: asyncio.Task[None] | None = None
|
|
31
|
+
self._accepting = True
|
|
32
|
+
self._current = False
|
|
33
|
+
self.sent = 0
|
|
34
|
+
self.failures = 0
|
|
35
|
+
|
|
36
|
+
async def open(self) -> None:
|
|
37
|
+
await self.destination.open()
|
|
38
|
+
self._task = asyncio.create_task(self._run(), name=f"zenx:destination:{self.name}")
|
|
39
|
+
|
|
40
|
+
def enqueue(self, payload: bytes) -> None:
|
|
41
|
+
if not self._accepting:
|
|
42
|
+
raise RuntimeError("Destination admission is closed")
|
|
43
|
+
if not isinstance(payload, bytes):
|
|
44
|
+
raise TypeError("Destination.prepare() must return immutable bytes")
|
|
45
|
+
self._times.append(asyncio.get_running_loop().time())
|
|
46
|
+
self.queue.put_nowait(payload)
|
|
47
|
+
|
|
48
|
+
def stop_admission(self) -> None:
|
|
49
|
+
self._accepting = False
|
|
50
|
+
|
|
51
|
+
async def _run(self) -> None:
|
|
52
|
+
while True:
|
|
53
|
+
payload = await self.queue.get()
|
|
54
|
+
self._current = True
|
|
55
|
+
try:
|
|
56
|
+
await self.destination.ready()
|
|
57
|
+
await self.destination.send(payload)
|
|
58
|
+
self.sent += 1
|
|
59
|
+
except Exception:
|
|
60
|
+
self.failures += 1
|
|
61
|
+
self.logger.exception("Destination %s failed; payload discarded", self.name)
|
|
62
|
+
finally:
|
|
63
|
+
self._current = False
|
|
64
|
+
self._times.popleft()
|
|
65
|
+
self.queue.task_done()
|
|
66
|
+
# A recovered connection may accept many buffered sends without
|
|
67
|
+
# suspending. Keep backlog draining fair to other destinations and crawls.
|
|
68
|
+
await asyncio.sleep(0)
|
|
69
|
+
|
|
70
|
+
async def drain(self) -> None:
|
|
71
|
+
await self.queue.join()
|
|
72
|
+
|
|
73
|
+
def cancel(self) -> None:
|
|
74
|
+
self.stop_admission()
|
|
75
|
+
if self._task is not None:
|
|
76
|
+
self._task.cancel()
|
|
77
|
+
while not self.queue.empty():
|
|
78
|
+
self.queue.get_nowait()
|
|
79
|
+
self.queue.task_done()
|
|
80
|
+
# The in-flight entry is removed by the sender's finally clause.
|
|
81
|
+
while len(self._times) > int(self._current):
|
|
82
|
+
self._times.pop()
|
|
83
|
+
|
|
84
|
+
async def close(self) -> None:
|
|
85
|
+
self.cancel()
|
|
86
|
+
try:
|
|
87
|
+
if self._task is not None:
|
|
88
|
+
await asyncio.gather(self._task, return_exceptions=True)
|
|
89
|
+
finally:
|
|
90
|
+
await self.destination.close()
|
|
91
|
+
|
|
92
|
+
def stats(self) -> DeliveryStats:
|
|
93
|
+
now = asyncio.get_running_loop().time()
|
|
94
|
+
return DeliveryStats(
|
|
95
|
+
pending=len(self._times),
|
|
96
|
+
oldest_pending_age=max(0.0, now - self._times[0]) if self._times else 0.0,
|
|
97
|
+
sent=self.sent, failures=self.failures,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class Delivery:
|
|
102
|
+
def __init__(
|
|
103
|
+
self, project_name: str, dedup_scope: str, store: Store,
|
|
104
|
+
senders: Mapping[str, Sender], logger: logging.Logger,
|
|
105
|
+
) -> None:
|
|
106
|
+
self.project_name = project_name
|
|
107
|
+
self.dedup_scope = dedup_scope
|
|
108
|
+
self.store = store
|
|
109
|
+
self.senders = dict(senders)
|
|
110
|
+
self.logger = logger
|
|
111
|
+
self.accepting = True
|
|
112
|
+
|
|
113
|
+
async def emit(
|
|
114
|
+
self, item_id: str, data: dict[str, Any], *, to: Collection[str] | None = None,
|
|
115
|
+
) -> None:
|
|
116
|
+
if not self.accepting:
|
|
117
|
+
raise RuntimeError("Delivery admission is closed")
|
|
118
|
+
if not isinstance(item_id, str) or not item_id:
|
|
119
|
+
raise ValueError("item_id must be a nonempty string")
|
|
120
|
+
if not isinstance(data, dict):
|
|
121
|
+
raise TypeError("Item data must be a dictionary")
|
|
122
|
+
if isinstance(to, str):
|
|
123
|
+
raise TypeError("to must be a collection of destination names, not a string")
|
|
124
|
+
names = tuple(self.senders) if to is None else tuple(to)
|
|
125
|
+
if any(not isinstance(name, str) or name not in self.senders for name in names):
|
|
126
|
+
raise ValueError("Routes must be among the selected spider's configured destinations")
|
|
127
|
+
if not await self.store.add((self.project_name, self.dedup_scope, item_id)):
|
|
128
|
+
return
|
|
129
|
+
for name in dict.fromkeys(names):
|
|
130
|
+
sender = self.senders[name]
|
|
131
|
+
try:
|
|
132
|
+
# Isolate even a mutating transform from other destinations and
|
|
133
|
+
# from the caller. The queue only retains serialized bytes.
|
|
134
|
+
payload = sender.destination.prepare(item_id, deepcopy(data))
|
|
135
|
+
sender.enqueue(payload)
|
|
136
|
+
except Exception:
|
|
137
|
+
sender.failures += 1
|
|
138
|
+
self.logger.exception("Destination %s could not prepare/admit item %s", name, item_id)
|
|
139
|
+
await asyncio.sleep(0)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from typing import Any, Protocol
|
|
3
|
+
|
|
4
|
+
type Item = dict[str, Any]
|
|
5
|
+
type Transform = Callable[[Item], Item]
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Destination(Protocol):
|
|
9
|
+
"""Each destination has one runtime-owned queue and sender.
|
|
10
|
+
|
|
11
|
+
prepare() must return owned, immutable bytes without destination I/O.
|
|
12
|
+
ready() may recover a connection, but must never send a payload.
|
|
13
|
+
send() makes one attempt; neither it nor close() may replay that attempt.
|
|
14
|
+
close() must be cancellation-cooperative and release resources in finally.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
async def open(self) -> None: ...
|
|
18
|
+
|
|
19
|
+
def prepare(self, item_id: str, data: Item) -> bytes: ...
|
|
20
|
+
|
|
21
|
+
async def ready(self) -> None: ...
|
|
22
|
+
|
|
23
|
+
async def send(self, payload: bytes) -> None: ...
|
|
24
|
+
|
|
25
|
+
async def close(self) -> None: ...
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def transform_item(data: Item, transform: Transform | None) -> Item:
|
|
29
|
+
result = data if transform is None else transform(data)
|
|
30
|
+
if not isinstance(result, dict):
|
|
31
|
+
raise TypeError("A destination transform must return a dictionary")
|
|
32
|
+
return result
|