zenx-lite-python 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,132 @@
1
+ """JSONL output without blocking the crawl's event loop on local writes."""
2
+
3
+ import asyncio
4
+ import io
5
+ import os
6
+ import queue
7
+ import sys
8
+ import threading
9
+ from dataclasses import dataclass
10
+ from typing import BinaryIO, TextIO
11
+
12
+ import orjson
13
+
14
+ from .base import Item, Transform, transform_item
15
+
16
+
17
+ @dataclass(slots=True)
18
+ class _Operation:
19
+ kind: str
20
+ payload: bytes | None
21
+ result: asyncio.Future[None]
22
+
23
+
24
+ class JSONLines:
25
+ """Append UTF-8 JSON lines to a path or write to a supplied file object.
26
+
27
+ Supplied files remain caller-owned. A daemon writer keeps a blocked disk or
28
+ pipe outside the event loop and cannot extend CLI shutdown by holding an
29
+ executor thread alive. Each send is written and flushed immediately.
30
+ """
31
+
32
+ def __init__(
33
+ self, target: str | os.PathLike[str] | BinaryIO | TextIO,
34
+ *, transform: Transform | None = None,
35
+ ) -> None:
36
+ self.target = target
37
+ self.transform = transform
38
+ self._operations: queue.Queue[_Operation | None] = queue.Queue()
39
+ self._abort = threading.Event()
40
+ self._thread: threading.Thread | None = None
41
+
42
+ async def open(self) -> None:
43
+ if self._thread is not None and self._thread.is_alive():
44
+ raise RuntimeError("JSONLines is already open")
45
+ self._operations = queue.Queue()
46
+ self._abort = threading.Event()
47
+ self._thread = threading.Thread(target=self._writer, name="zenx-jsonl", daemon=True)
48
+ self._thread.start()
49
+ await self._submit("open")
50
+
51
+ def prepare(self, item_id: str, data: Item) -> bytes:
52
+ return orjson.dumps(transform_item(data, self.transform), option=orjson.OPT_APPEND_NEWLINE)
53
+
54
+ async def ready(self) -> None:
55
+ pass
56
+
57
+ async def send(self, payload: bytes) -> None:
58
+ await self._submit("write", payload)
59
+
60
+ async def close(self) -> None:
61
+ if self._thread is None:
62
+ return
63
+ try:
64
+ if self._thread.is_alive():
65
+ await self._submit("close")
66
+ finally:
67
+ self._abort.set()
68
+ self._operations.put_nowait(None)
69
+
70
+ async def _submit(self, kind: str, payload: bytes | None = None) -> None:
71
+ if self._abort.is_set():
72
+ raise RuntimeError("JSONLines is closed")
73
+ future = asyncio.get_running_loop().create_future()
74
+ self._operations.put_nowait(_Operation(kind, payload, future))
75
+ await future
76
+
77
+ @staticmethod
78
+ def _complete(future: asyncio.Future[None], error: Exception | None) -> None:
79
+ if not future.done():
80
+ if error is None:
81
+ future.set_result(None)
82
+ else:
83
+ future.set_exception(error)
84
+
85
+ def _writer(self) -> None:
86
+ stream: BinaryIO | TextIO | None = None
87
+ owned = False
88
+ try:
89
+ while not self._abort.is_set():
90
+ operation = self._operations.get()
91
+ if operation is None:
92
+ return
93
+ error: Exception | None = None
94
+ try:
95
+ if operation.kind == "open":
96
+ if isinstance(self.target, (str, os.PathLike)):
97
+ stream = open(self.target, "ab", buffering=0)
98
+ owned = True
99
+ elif self.target is sys.stdout or self.target is sys.stderr:
100
+ # Avoid holding Python's standard-stream buffer lock
101
+ # in a blocked daemon thread during interpreter exit.
102
+ stream = os.fdopen(os.dup(self.target.fileno()), "ab", buffering=0)
103
+ owned = True
104
+ else:
105
+ stream = self.target
106
+ elif operation.kind == "write":
107
+ if stream is None or operation.payload is None:
108
+ raise RuntimeError("JSONLines is not open")
109
+ content = operation.payload.decode("utf-8") if isinstance(stream, io.TextIOBase) else operation.payload
110
+ offset = 0
111
+ while offset < len(content):
112
+ written = stream.write(content[offset:])
113
+ if written is None or written <= 0:
114
+ raise OSError("JSONLines write made no progress")
115
+ offset += written
116
+ stream.flush()
117
+ elif operation.kind == "close" and owned and stream is not None:
118
+ stream.close()
119
+ stream = None
120
+ except Exception as exc:
121
+ error = exc
122
+ loop = operation.result.get_loop()
123
+ try:
124
+ loop.call_soon_threadsafe(self._complete, operation.result, error)
125
+ except RuntimeError:
126
+ # The CLI has already exhausted its shutdown budget.
127
+ return
128
+ if operation.kind == "close":
129
+ return
130
+ finally:
131
+ if owned and stream is not None:
132
+ stream.close()
@@ -0,0 +1,5 @@
1
+ """Optional Synoptic Source integration; install the 'source' extra."""
2
+
3
+ from .source import SourceDestination
4
+
5
+ __all__ = ["SourceDestination"]
@@ -0,0 +1,136 @@
1
+ """Authenticated Source WebSocket connections with no payload replay."""
2
+
3
+ import asyncio
4
+ import logging
5
+ import math
6
+ from contextlib import suppress
7
+ from typing import Any
8
+
9
+ import orjson
10
+
11
+ try:
12
+ from websockets.asyncio.client import ClientConnection, connect
13
+ from websockets.protocol import State
14
+ except ImportError as exc:
15
+ raise ImportError("SourceDestination requires zenx-lite-python's optional 'source' extra") from exc
16
+
17
+ from ...errors import ConfigurationError
18
+ from ..base import Item, Transform, transform_item
19
+
20
+
21
+ class SourceDestination:
22
+ def __init__(
23
+ self, endpoint: str, *, api_key: str, source_id: str,
24
+ transform: Transform | None = None, reconnect_delay: float = 1.0,
25
+ connect_options: dict[str, Any] | None = None,
26
+ ) -> None:
27
+ if not endpoint.startswith(("ws://", "wss://")):
28
+ raise ConfigurationError("Source endpoint must be a ws:// or wss:// URL")
29
+ if not api_key or not source_id:
30
+ raise ConfigurationError("Source api_key and source_id are required")
31
+ if not math.isfinite(reconnect_delay) or reconnect_delay <= 0:
32
+ raise ConfigurationError("reconnect_delay must be finite and positive")
33
+ self.endpoint = endpoint
34
+ self.api_key = api_key
35
+ self.source_id = source_id
36
+ self.transform = transform
37
+ self.reconnect_delay = reconnect_delay
38
+ self._options = dict(connect_options or {})
39
+ self._connection: ClientConnection | None = None
40
+ self._listener: asyncio.Task[None] | None = None
41
+ self._closed = True
42
+ self._logger = logging.getLogger("zenx_lite.source")
43
+
44
+ async def open(self) -> None:
45
+ self._closed = False
46
+ await self._connect()
47
+
48
+ async def _connect(self) -> None:
49
+ options = self._options.copy()
50
+ headers = dict(options.pop("additional_headers", {}))
51
+ headers["x-api-key"] = self.api_key
52
+ connection = await connect(self.endpoint, additional_headers=headers, **options)
53
+ self._connection = connection
54
+ try:
55
+ welcome = orjson.loads(await connection.recv())
56
+ if not isinstance(welcome, dict) or welcome.get("type") != "connected":
57
+ raise ConfigurationError("Unexpected Synoptic Source welcome frame")
58
+ sources = welcome.get("sources") or []
59
+ if not isinstance(sources, list) or not any(
60
+ isinstance(source, dict) and source.get("sourceId") == self.source_id
61
+ for source in sources
62
+ ):
63
+ raise ConfigurationError("Source API key is not authorized for the configured source_id")
64
+ except BaseException:
65
+ connection.transport.abort()
66
+ self._connection = None
67
+ raise
68
+ self._listener = asyncio.create_task(self._listen(connection), name="zenx:source:listen")
69
+
70
+ async def _listen(self, connection: ClientConnection) -> None:
71
+ try:
72
+ async for frame in connection:
73
+ # Responses are diagnostic only. They never control replay or
74
+ # admission, and payload contents/credentials aren't logged.
75
+ try:
76
+ message = orjson.loads(frame)
77
+ kind = message.get("type") if isinstance(message, dict) else None
78
+ self._logger.debug("Source response type=%s", kind)
79
+ except orjson.JSONDecodeError:
80
+ self._logger.warning("Source returned a non-JSON response")
81
+ except Exception:
82
+ self._logger.warning("Source connection closed")
83
+
84
+ def prepare(self, item_id: str, data: Item) -> bytes:
85
+ return orjson.dumps({
86
+ "sourceId": self.source_id,
87
+ "idempotencyKey": item_id,
88
+ "data": transform_item(data, self.transform),
89
+ })
90
+
91
+ async def ready(self) -> None:
92
+ while not self._closed:
93
+ if (
94
+ self._connection is not None
95
+ and self._connection.state is State.OPEN
96
+ and not self._connection.transport.is_closing()
97
+ ):
98
+ return
99
+ await self._disconnect()
100
+ try:
101
+ await self._connect()
102
+ return
103
+ except Exception:
104
+ self._logger.warning("Source reconnect failed; unattempted payloads remain queued")
105
+ await asyncio.sleep(self.reconnect_delay)
106
+ raise RuntimeError("SourceDestination is closed")
107
+
108
+ async def send(self, payload: bytes) -> None:
109
+ connection = self._connection
110
+ if connection is None:
111
+ raise ConnectionError("Source connection is unavailable")
112
+ try:
113
+ # UTF-8 bytes are sent as a WebSocket text frame, not a binary frame.
114
+ await connection.send(payload, text=True)
115
+ except BaseException:
116
+ connection.transport.abort()
117
+ self._connection = None
118
+ raise
119
+
120
+ async def _disconnect(self) -> None:
121
+ connection, self._connection = self._connection, None
122
+ listener, self._listener = self._listener, None
123
+ if connection is not None:
124
+ connection.transport.abort()
125
+ if listener is not None:
126
+ listener.cancel()
127
+ with suppress(asyncio.CancelledError):
128
+ await listener
129
+
130
+ async def close(self) -> None:
131
+ self._closed = True
132
+ try:
133
+ if self._connection is not None:
134
+ await self._connection.close()
135
+ finally:
136
+ await self._disconnect()
zenx_lite/errors.py ADDED
@@ -0,0 +1,21 @@
1
+ """Exceptions shared by the runtime and its extension contracts."""
2
+
3
+
4
+ class ConfigurationError(ValueError):
5
+ """Application configuration is invalid; no crawl should start."""
6
+
7
+
8
+ class FatalRuntimeError(RuntimeError):
9
+ """The scheduler must propagate this error and stop execution."""
10
+
11
+
12
+ class WorkerOverlap(FatalRuntimeError):
13
+ """A strategy requested a crawl on an already occupied worker."""
14
+
15
+
16
+ class SessionUnavailable(RuntimeError):
17
+ """No usable session is available for this worker's crawl."""
18
+
19
+
20
+ class CrawlFinished(RuntimeError):
21
+ """A crawler handle was used outside its owning crawl."""
zenx_lite/http.py ADDED
@@ -0,0 +1,130 @@
1
+ """Buffered responses and a thin context around native curl streaming."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import time
7
+ from collections.abc import AsyncIterator, Mapping
8
+ from contextlib import asynccontextmanager
9
+ from dataclasses import dataclass, field
10
+ from typing import Any
11
+
12
+ import orjson
13
+ lazy import parsel
14
+ from curl_cffi import Response as CurlResponse
15
+ from curl_cffi.requests import Headers
16
+ from curl_cffi.requests.exceptions import RequestException
17
+
18
+ from .sessions import Session
19
+
20
+ RETIRE_STATUSES = frozenset({401, 403, 429})
21
+
22
+
23
+ @dataclass(frozen=True, slots=True)
24
+ class Response:
25
+ url: str
26
+ status: int
27
+ headers: Headers
28
+ body: bytes
29
+ elapsed: float
30
+ encoding: str
31
+ # Keep dataclass field inspection from resolving the lazy parser import.
32
+ _selector: "parsel.Selector | None" = field(default=None, init=False, repr=False, compare=False)
33
+
34
+ @property
35
+ def text(self) -> str:
36
+ return self.body.decode(self.encoding, errors="replace")
37
+
38
+ def json(self) -> Any:
39
+ return orjson.loads(self.body)
40
+
41
+ def xpath(
42
+ self, query: str, namespaces: Mapping[str, str] | None = None,
43
+ **variables: Any,
44
+ ) -> parsel.SelectorList[parsel.Selector]:
45
+ """Select HTML/XML nodes, parsing the response only on first use."""
46
+ selector = self._selector
47
+ if selector is None:
48
+ content_type = self.headers.get("Content-Type", "").split(";", 1)[0].strip().lower()
49
+ kind = "xml" if content_type in {"application/xml", "text/xml"} or content_type.endswith("+xml") else None
50
+ selector = parsel.Selector(text=self.text, type=kind, base_url=self.url)
51
+ object.__setattr__(self, "_selector", selector)
52
+ return selector.xpath(query, namespaces=namespaces, **variables)
53
+
54
+
55
+ def _options(options: dict[str, Any]) -> dict[str, Any]:
56
+ if "session" in options:
57
+ raise TypeError("Framework requests use the worker's session")
58
+ if "stream" in options:
59
+ raise TypeError("Use crawler.stream() for streaming responses")
60
+ if "json" in options and options["json"] is not None:
61
+ if options.get("data") is not None or options.get("content") is not None:
62
+ raise TypeError("json cannot be combined with data or content")
63
+ options["data"] = orjson.dumps(options.pop("json"))
64
+ headers = Headers(options.get("headers"))
65
+ headers.setdefault("Content-Type", "application/json")
66
+ options["headers"] = headers
67
+ return options
68
+
69
+
70
+ async def request(session: Session, url: str, method: str = "GET", **kwargs: Any) -> Response:
71
+ session.require_ready()
72
+ options = _options(kwargs)
73
+ started = time.monotonic()
74
+ try:
75
+ raw = await session.transport.request(method, url, **options)
76
+ except (Exception, asyncio.CancelledError):
77
+ session.retired = True
78
+ raise
79
+ if raw.status_code in RETIRE_STATUSES:
80
+ session.retired = True
81
+ return Response(
82
+ url=raw.url, status=raw.status_code, headers=raw.headers,
83
+ body=raw.content, elapsed=time.monotonic() - started,
84
+ encoding=raw.encoding or "utf-8",
85
+ )
86
+
87
+
88
+ async def _close_stream(response: CurlResponse) -> None:
89
+ # curl's aclose() only waits for the download. Cancel it explicitly on early
90
+ # exit so an unread/infinite body cannot keep the worker occupied forever.
91
+ task = response.astream_task
92
+ if response.quit_now is not None:
93
+ response.quit_now.set()
94
+ if isinstance(task, asyncio.Task) and not task.done():
95
+ task.cancel()
96
+ if task is not None:
97
+ # Consume the download's intentional cancellation, but let cancellation
98
+ # of this caller propagate. A broad suppress would swallow both.
99
+ result, = await asyncio.gather(task, return_exceptions=True)
100
+ if isinstance(result, asyncio.CancelledError):
101
+ return
102
+ if isinstance(result, BaseException):
103
+ raise result
104
+ await response.aclose()
105
+
106
+
107
+ @asynccontextmanager
108
+ async def stream(
109
+ session: Session, url: str, method: str = "GET", **kwargs: Any,
110
+ ) -> AsyncIterator[CurlResponse]:
111
+ session.require_ready()
112
+ options = _options(kwargs)
113
+ try:
114
+ response = await session.transport.request(method, url, stream=True, **options)
115
+ except (Exception, asyncio.CancelledError):
116
+ session.retired = True
117
+ raise
118
+ if response.status_code in RETIRE_STATUSES:
119
+ session.retired = True
120
+ try:
121
+ yield response
122
+ except (RequestException, asyncio.CancelledError):
123
+ session.retired = True
124
+ raise
125
+ finally:
126
+ try:
127
+ await _close_stream(response)
128
+ except (RequestException, asyncio.CancelledError):
129
+ session.retired = True
130
+ raise
@@ -0,0 +1,5 @@
1
+ """Successful-crawl notification; import concrete adapters explicitly."""
2
+
3
+ from .base import Notifier
4
+
5
+ __all__ = ["Notifier"]
@@ -0,0 +1,15 @@
1
+ from typing import Protocol
2
+
3
+
4
+ class Notifier(Protocol):
5
+ """Report successful crawls without blocking execution.
6
+
7
+ open() must return promptly; notify() must perform no blocking I/O.
8
+ close() must cooperate with cancellation and the runtime shutdown budget.
9
+ """
10
+
11
+ async def open(self) -> None: ...
12
+
13
+ def notify(self) -> None: ...
14
+
15
+ async def close(self) -> None: ...
@@ -0,0 +1,86 @@
1
+ """Best-effort heartbeats over Magent's local Unix socket."""
2
+
3
+ import asyncio
4
+ import logging
5
+ import math
6
+ import os
7
+
8
+ import orjson
9
+
10
+ from ..errors import ConfigurationError
11
+
12
+
13
+ class MagentNotifier:
14
+ def __init__(
15
+ self, app_id: str, *, instance: str | None = None,
16
+ socket_path: str = "/tmp/itxpmonitor.sock", timeout: float = 1.0,
17
+ ) -> None:
18
+ for name, value in (("app_id", app_id), ("socket_path", socket_path)):
19
+ if not isinstance(value, str) or not value.strip():
20
+ raise ConfigurationError(f"Magent {name} must be a nonblank string")
21
+ if instance is not None and (not isinstance(instance, str) or not instance.strip()):
22
+ raise ConfigurationError("Magent instance must be a nonblank string or None")
23
+ if type(timeout) not in (int, float) or not math.isfinite(timeout) or timeout <= 0:
24
+ raise ConfigurationError("Magent timeout must be a positive finite number")
25
+ self.app_id = app_id
26
+ self.instance = instance
27
+ self.socket_path = socket_path
28
+ self.timeout = timeout
29
+ self._queue: asyncio.Queue[None] = asyncio.Queue()
30
+ self._reader: asyncio.StreamReader | None = None
31
+ self._writer: asyncio.StreamWriter | None = None
32
+ self._task: asyncio.Task[None] | None = None
33
+ self._accepting = False
34
+
35
+ async def open(self) -> None:
36
+ if self._task is not None:
37
+ raise RuntimeError("MagentNotifier is already open")
38
+ self._queue = asyncio.Queue()
39
+ self._accepting = True
40
+ self._task = asyncio.create_task(self._send(), name="zenx:notifier:magent")
41
+
42
+ def notify(self) -> None:
43
+ if self._accepting:
44
+ self._queue.put_nowait(None)
45
+
46
+ def _disconnect(self) -> None:
47
+ if self._writer is not None:
48
+ self._writer.transport.abort()
49
+ self._reader = self._writer = None
50
+
51
+ async def _send(self) -> None:
52
+ while True:
53
+ await self._queue.get()
54
+ try:
55
+ async with asyncio.timeout(self.timeout):
56
+ if self._writer is None:
57
+ self._reader, self._writer = await asyncio.open_unix_connection(self.socket_path)
58
+ elif self._reader is not None and self._reader.at_eof():
59
+ raise ConnectionError("Magent collector disconnected")
60
+ payload = {"type": "heartbeat", "app_id": self.app_id, "pid": os.getpid()}
61
+ if self.instance is not None:
62
+ payload["instance"] = self.instance
63
+ self._writer.write(orjson.dumps(payload, option=orjson.OPT_APPEND_NEWLINE))
64
+ await self._writer.drain()
65
+ except Exception:
66
+ self._disconnect()
67
+ logging.getLogger("zenx_lite.magent").warning("Magent heartbeat failed; discarded")
68
+ finally:
69
+ self._queue.task_done()
70
+ await asyncio.sleep(0)
71
+
72
+ async def close(self) -> None:
73
+ self._accepting = False
74
+ task, self._task = self._task, None
75
+ try:
76
+ if task is not None:
77
+ await self._queue.join()
78
+ finally:
79
+ if task is not None:
80
+ task.cancel()
81
+ self._disconnect()
82
+ while not self._queue.empty():
83
+ self._queue.get_nowait()
84
+ self._queue.task_done()
85
+ if task is not None:
86
+ await asyncio.gather(task, return_exceptions=True)
zenx_lite/scheduler.py ADDED
@@ -0,0 +1,63 @@
1
+ """Scheduling policy is independent of workers, HTTP, and delivery."""
2
+
3
+ import asyncio
4
+ import math
5
+ from collections.abc import Awaitable, Callable
6
+ from dataclasses import dataclass
7
+ from typing import Protocol
8
+
9
+ from .errors import ConfigurationError, FatalRuntimeError
10
+
11
+ type Crawl = Callable[[int], Awaitable[None]]
12
+
13
+
14
+ class Scheduler(Protocol):
15
+ async def run(self, crawl: Crawl, *, concurrency: int, stop: asyncio.Event) -> None: ...
16
+
17
+
18
+ @dataclass(frozen=True, slots=True)
19
+ class Slots:
20
+ interval: float = 1.0
21
+
22
+ def __post_init__(self) -> None:
23
+ if isinstance(self.interval, bool) or not isinstance(self.interval, (int, float)) or not math.isfinite(self.interval) or self.interval <= 0:
24
+ raise ConfigurationError("Slots interval must be a finite positive number")
25
+
26
+ async def run(self, crawl: Crawl, *, concurrency: int, stop: asyncio.Event) -> None:
27
+ if type(concurrency) is not int or concurrency <= 0:
28
+ raise ConfigurationError("concurrency must be a positive integer")
29
+ origin = asyncio.get_running_loop().time()
30
+ async with asyncio.TaskGroup() as tasks:
31
+ for worker_id in range(concurrency):
32
+ tasks.create_task(self._worker(crawl, worker_id, concurrency, origin, stop))
33
+
34
+ async def _worker(
35
+ self, crawl: Crawl, worker_id: int, concurrency: int,
36
+ origin: float, stop: asyncio.Event,
37
+ ) -> None:
38
+ loop = asyncio.get_running_loop()
39
+ first = origin + worker_id * self.interval / concurrency
40
+ target = first
41
+ while not stop.is_set():
42
+ delay = target - loop.time()
43
+ if delay > 0:
44
+ try:
45
+ async with asyncio.timeout(delay):
46
+ await stop.wait()
47
+ return
48
+ except TimeoutError:
49
+ pass
50
+ if stop.is_set():
51
+ return
52
+ try:
53
+ await crawl(worker_id)
54
+ except FatalRuntimeError:
55
+ raise
56
+ except Exception as exc:
57
+ if isinstance(exc, BaseExceptionGroup) and exc.subgroup(FatalRuntimeError) is not None:
58
+ raise
59
+ # The execution callback already logged this crawl's failure.
60
+ pass
61
+ # Recompute from the origin, never from completion time or missed ticks.
62
+ step = max(1, math.floor((loop.time() - first) / self.interval) + 1)
63
+ target = first + step * self.interval