zenx-lite-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenx_lite/__init__.py +8 -0
- zenx_lite/app.py +377 -0
- zenx_lite/config.py +56 -0
- zenx_lite/configuration.py +359 -0
- zenx_lite/crawler.py +69 -0
- zenx_lite/delivery.py +139 -0
- zenx_lite/destinations/__init__.py +5 -0
- zenx_lite/destinations/base.py +32 -0
- zenx_lite/destinations/jsonl.py +132 -0
- zenx_lite/destinations/synoptic/__init__.py +5 -0
- zenx_lite/destinations/synoptic/source.py +136 -0
- zenx_lite/errors.py +21 -0
- zenx_lite/http.py +130 -0
- zenx_lite/notifiers/__init__.py +5 -0
- zenx_lite/notifiers/base.py +15 -0
- zenx_lite/notifiers/magent.py +86 -0
- zenx_lite/scheduler.py +63 -0
- zenx_lite/sessions.py +102 -0
- zenx_lite/store.py +45 -0
- zenx_lite/stores/__init__.py +1 -0
- zenx_lite/stores/redis.py +54 -0
- zenx_lite_python-0.1.0.dist-info/METADATA +14 -0
- zenx_lite_python-0.1.0.dist-info/RECORD +24 -0
- zenx_lite_python-0.1.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""JSONL output without blocking the crawl's event loop on local writes."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import io
|
|
5
|
+
import os
|
|
6
|
+
import queue
|
|
7
|
+
import sys
|
|
8
|
+
import threading
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from typing import BinaryIO, TextIO
|
|
11
|
+
|
|
12
|
+
import orjson
|
|
13
|
+
|
|
14
|
+
from .base import Item, Transform, transform_item
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(slots=True)
|
|
18
|
+
class _Operation:
|
|
19
|
+
kind: str
|
|
20
|
+
payload: bytes | None
|
|
21
|
+
result: asyncio.Future[None]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class JSONLines:
|
|
25
|
+
"""Append UTF-8 JSON lines to a path or write to a supplied file object.
|
|
26
|
+
|
|
27
|
+
Supplied files remain caller-owned. A daemon writer keeps a blocked disk or
|
|
28
|
+
pipe outside the event loop and cannot extend CLI shutdown by holding an
|
|
29
|
+
executor thread alive. Each send is written and flushed immediately.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(
|
|
33
|
+
self, target: str | os.PathLike[str] | BinaryIO | TextIO,
|
|
34
|
+
*, transform: Transform | None = None,
|
|
35
|
+
) -> None:
|
|
36
|
+
self.target = target
|
|
37
|
+
self.transform = transform
|
|
38
|
+
self._operations: queue.Queue[_Operation | None] = queue.Queue()
|
|
39
|
+
self._abort = threading.Event()
|
|
40
|
+
self._thread: threading.Thread | None = None
|
|
41
|
+
|
|
42
|
+
async def open(self) -> None:
|
|
43
|
+
if self._thread is not None and self._thread.is_alive():
|
|
44
|
+
raise RuntimeError("JSONLines is already open")
|
|
45
|
+
self._operations = queue.Queue()
|
|
46
|
+
self._abort = threading.Event()
|
|
47
|
+
self._thread = threading.Thread(target=self._writer, name="zenx-jsonl", daemon=True)
|
|
48
|
+
self._thread.start()
|
|
49
|
+
await self._submit("open")
|
|
50
|
+
|
|
51
|
+
def prepare(self, item_id: str, data: Item) -> bytes:
|
|
52
|
+
return orjson.dumps(transform_item(data, self.transform), option=orjson.OPT_APPEND_NEWLINE)
|
|
53
|
+
|
|
54
|
+
async def ready(self) -> None:
|
|
55
|
+
pass
|
|
56
|
+
|
|
57
|
+
async def send(self, payload: bytes) -> None:
|
|
58
|
+
await self._submit("write", payload)
|
|
59
|
+
|
|
60
|
+
async def close(self) -> None:
|
|
61
|
+
if self._thread is None:
|
|
62
|
+
return
|
|
63
|
+
try:
|
|
64
|
+
if self._thread.is_alive():
|
|
65
|
+
await self._submit("close")
|
|
66
|
+
finally:
|
|
67
|
+
self._abort.set()
|
|
68
|
+
self._operations.put_nowait(None)
|
|
69
|
+
|
|
70
|
+
async def _submit(self, kind: str, payload: bytes | None = None) -> None:
|
|
71
|
+
if self._abort.is_set():
|
|
72
|
+
raise RuntimeError("JSONLines is closed")
|
|
73
|
+
future = asyncio.get_running_loop().create_future()
|
|
74
|
+
self._operations.put_nowait(_Operation(kind, payload, future))
|
|
75
|
+
await future
|
|
76
|
+
|
|
77
|
+
@staticmethod
|
|
78
|
+
def _complete(future: asyncio.Future[None], error: Exception | None) -> None:
|
|
79
|
+
if not future.done():
|
|
80
|
+
if error is None:
|
|
81
|
+
future.set_result(None)
|
|
82
|
+
else:
|
|
83
|
+
future.set_exception(error)
|
|
84
|
+
|
|
85
|
+
def _writer(self) -> None:
|
|
86
|
+
stream: BinaryIO | TextIO | None = None
|
|
87
|
+
owned = False
|
|
88
|
+
try:
|
|
89
|
+
while not self._abort.is_set():
|
|
90
|
+
operation = self._operations.get()
|
|
91
|
+
if operation is None:
|
|
92
|
+
return
|
|
93
|
+
error: Exception | None = None
|
|
94
|
+
try:
|
|
95
|
+
if operation.kind == "open":
|
|
96
|
+
if isinstance(self.target, (str, os.PathLike)):
|
|
97
|
+
stream = open(self.target, "ab", buffering=0)
|
|
98
|
+
owned = True
|
|
99
|
+
elif self.target is sys.stdout or self.target is sys.stderr:
|
|
100
|
+
# Avoid holding Python's standard-stream buffer lock
|
|
101
|
+
# in a blocked daemon thread during interpreter exit.
|
|
102
|
+
stream = os.fdopen(os.dup(self.target.fileno()), "ab", buffering=0)
|
|
103
|
+
owned = True
|
|
104
|
+
else:
|
|
105
|
+
stream = self.target
|
|
106
|
+
elif operation.kind == "write":
|
|
107
|
+
if stream is None or operation.payload is None:
|
|
108
|
+
raise RuntimeError("JSONLines is not open")
|
|
109
|
+
content = operation.payload.decode("utf-8") if isinstance(stream, io.TextIOBase) else operation.payload
|
|
110
|
+
offset = 0
|
|
111
|
+
while offset < len(content):
|
|
112
|
+
written = stream.write(content[offset:])
|
|
113
|
+
if written is None or written <= 0:
|
|
114
|
+
raise OSError("JSONLines write made no progress")
|
|
115
|
+
offset += written
|
|
116
|
+
stream.flush()
|
|
117
|
+
elif operation.kind == "close" and owned and stream is not None:
|
|
118
|
+
stream.close()
|
|
119
|
+
stream = None
|
|
120
|
+
except Exception as exc:
|
|
121
|
+
error = exc
|
|
122
|
+
loop = operation.result.get_loop()
|
|
123
|
+
try:
|
|
124
|
+
loop.call_soon_threadsafe(self._complete, operation.result, error)
|
|
125
|
+
except RuntimeError:
|
|
126
|
+
# The CLI has already exhausted its shutdown budget.
|
|
127
|
+
return
|
|
128
|
+
if operation.kind == "close":
|
|
129
|
+
return
|
|
130
|
+
finally:
|
|
131
|
+
if owned and stream is not None:
|
|
132
|
+
stream.close()
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""Authenticated Source WebSocket connections with no payload replay."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
import math
|
|
6
|
+
from contextlib import suppress
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import orjson
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
from websockets.asyncio.client import ClientConnection, connect
|
|
13
|
+
from websockets.protocol import State
|
|
14
|
+
except ImportError as exc:
|
|
15
|
+
raise ImportError("SourceDestination requires zenx-lite-python's optional 'source' extra") from exc
|
|
16
|
+
|
|
17
|
+
from ...errors import ConfigurationError
|
|
18
|
+
from ..base import Item, Transform, transform_item
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class SourceDestination:
|
|
22
|
+
def __init__(
|
|
23
|
+
self, endpoint: str, *, api_key: str, source_id: str,
|
|
24
|
+
transform: Transform | None = None, reconnect_delay: float = 1.0,
|
|
25
|
+
connect_options: dict[str, Any] | None = None,
|
|
26
|
+
) -> None:
|
|
27
|
+
if not endpoint.startswith(("ws://", "wss://")):
|
|
28
|
+
raise ConfigurationError("Source endpoint must be a ws:// or wss:// URL")
|
|
29
|
+
if not api_key or not source_id:
|
|
30
|
+
raise ConfigurationError("Source api_key and source_id are required")
|
|
31
|
+
if not math.isfinite(reconnect_delay) or reconnect_delay <= 0:
|
|
32
|
+
raise ConfigurationError("reconnect_delay must be finite and positive")
|
|
33
|
+
self.endpoint = endpoint
|
|
34
|
+
self.api_key = api_key
|
|
35
|
+
self.source_id = source_id
|
|
36
|
+
self.transform = transform
|
|
37
|
+
self.reconnect_delay = reconnect_delay
|
|
38
|
+
self._options = dict(connect_options or {})
|
|
39
|
+
self._connection: ClientConnection | None = None
|
|
40
|
+
self._listener: asyncio.Task[None] | None = None
|
|
41
|
+
self._closed = True
|
|
42
|
+
self._logger = logging.getLogger("zenx_lite.source")
|
|
43
|
+
|
|
44
|
+
async def open(self) -> None:
|
|
45
|
+
self._closed = False
|
|
46
|
+
await self._connect()
|
|
47
|
+
|
|
48
|
+
async def _connect(self) -> None:
|
|
49
|
+
options = self._options.copy()
|
|
50
|
+
headers = dict(options.pop("additional_headers", {}))
|
|
51
|
+
headers["x-api-key"] = self.api_key
|
|
52
|
+
connection = await connect(self.endpoint, additional_headers=headers, **options)
|
|
53
|
+
self._connection = connection
|
|
54
|
+
try:
|
|
55
|
+
welcome = orjson.loads(await connection.recv())
|
|
56
|
+
if not isinstance(welcome, dict) or welcome.get("type") != "connected":
|
|
57
|
+
raise ConfigurationError("Unexpected Synoptic Source welcome frame")
|
|
58
|
+
sources = welcome.get("sources") or []
|
|
59
|
+
if not isinstance(sources, list) or not any(
|
|
60
|
+
isinstance(source, dict) and source.get("sourceId") == self.source_id
|
|
61
|
+
for source in sources
|
|
62
|
+
):
|
|
63
|
+
raise ConfigurationError("Source API key is not authorized for the configured source_id")
|
|
64
|
+
except BaseException:
|
|
65
|
+
connection.transport.abort()
|
|
66
|
+
self._connection = None
|
|
67
|
+
raise
|
|
68
|
+
self._listener = asyncio.create_task(self._listen(connection), name="zenx:source:listen")
|
|
69
|
+
|
|
70
|
+
async def _listen(self, connection: ClientConnection) -> None:
|
|
71
|
+
try:
|
|
72
|
+
async for frame in connection:
|
|
73
|
+
# Responses are diagnostic only. They never control replay or
|
|
74
|
+
# admission, and payload contents/credentials aren't logged.
|
|
75
|
+
try:
|
|
76
|
+
message = orjson.loads(frame)
|
|
77
|
+
kind = message.get("type") if isinstance(message, dict) else None
|
|
78
|
+
self._logger.debug("Source response type=%s", kind)
|
|
79
|
+
except orjson.JSONDecodeError:
|
|
80
|
+
self._logger.warning("Source returned a non-JSON response")
|
|
81
|
+
except Exception:
|
|
82
|
+
self._logger.warning("Source connection closed")
|
|
83
|
+
|
|
84
|
+
def prepare(self, item_id: str, data: Item) -> bytes:
|
|
85
|
+
return orjson.dumps({
|
|
86
|
+
"sourceId": self.source_id,
|
|
87
|
+
"idempotencyKey": item_id,
|
|
88
|
+
"data": transform_item(data, self.transform),
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
async def ready(self) -> None:
|
|
92
|
+
while not self._closed:
|
|
93
|
+
if (
|
|
94
|
+
self._connection is not None
|
|
95
|
+
and self._connection.state is State.OPEN
|
|
96
|
+
and not self._connection.transport.is_closing()
|
|
97
|
+
):
|
|
98
|
+
return
|
|
99
|
+
await self._disconnect()
|
|
100
|
+
try:
|
|
101
|
+
await self._connect()
|
|
102
|
+
return
|
|
103
|
+
except Exception:
|
|
104
|
+
self._logger.warning("Source reconnect failed; unattempted payloads remain queued")
|
|
105
|
+
await asyncio.sleep(self.reconnect_delay)
|
|
106
|
+
raise RuntimeError("SourceDestination is closed")
|
|
107
|
+
|
|
108
|
+
async def send(self, payload: bytes) -> None:
|
|
109
|
+
connection = self._connection
|
|
110
|
+
if connection is None:
|
|
111
|
+
raise ConnectionError("Source connection is unavailable")
|
|
112
|
+
try:
|
|
113
|
+
# UTF-8 bytes are sent as a WebSocket text frame, not a binary frame.
|
|
114
|
+
await connection.send(payload, text=True)
|
|
115
|
+
except BaseException:
|
|
116
|
+
connection.transport.abort()
|
|
117
|
+
self._connection = None
|
|
118
|
+
raise
|
|
119
|
+
|
|
120
|
+
async def _disconnect(self) -> None:
|
|
121
|
+
connection, self._connection = self._connection, None
|
|
122
|
+
listener, self._listener = self._listener, None
|
|
123
|
+
if connection is not None:
|
|
124
|
+
connection.transport.abort()
|
|
125
|
+
if listener is not None:
|
|
126
|
+
listener.cancel()
|
|
127
|
+
with suppress(asyncio.CancelledError):
|
|
128
|
+
await listener
|
|
129
|
+
|
|
130
|
+
async def close(self) -> None:
|
|
131
|
+
self._closed = True
|
|
132
|
+
try:
|
|
133
|
+
if self._connection is not None:
|
|
134
|
+
await self._connection.close()
|
|
135
|
+
finally:
|
|
136
|
+
await self._disconnect()
|
zenx_lite/errors.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Exceptions shared by the runtime and its extension contracts."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ConfigurationError(ValueError):
|
|
5
|
+
"""Application configuration is invalid; no crawl should start."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class FatalRuntimeError(RuntimeError):
|
|
9
|
+
"""The scheduler must propagate this error and stop execution."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class WorkerOverlap(FatalRuntimeError):
|
|
13
|
+
"""A strategy requested a crawl on an already occupied worker."""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class SessionUnavailable(RuntimeError):
|
|
17
|
+
"""No usable session is available for this worker's crawl."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class CrawlFinished(RuntimeError):
|
|
21
|
+
"""A crawler handle was used outside its owning crawl."""
|
zenx_lite/http.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Buffered responses and a thin context around native curl streaming."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import time
|
|
7
|
+
from collections.abc import AsyncIterator, Mapping
|
|
8
|
+
from contextlib import asynccontextmanager
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import orjson
|
|
13
|
+
lazy import parsel
|
|
14
|
+
from curl_cffi import Response as CurlResponse
|
|
15
|
+
from curl_cffi.requests import Headers
|
|
16
|
+
from curl_cffi.requests.exceptions import RequestException
|
|
17
|
+
|
|
18
|
+
from .sessions import Session
|
|
19
|
+
|
|
20
|
+
RETIRE_STATUSES = frozenset({401, 403, 429})
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True, slots=True)
|
|
24
|
+
class Response:
|
|
25
|
+
url: str
|
|
26
|
+
status: int
|
|
27
|
+
headers: Headers
|
|
28
|
+
body: bytes
|
|
29
|
+
elapsed: float
|
|
30
|
+
encoding: str
|
|
31
|
+
# Keep dataclass field inspection from resolving the lazy parser import.
|
|
32
|
+
_selector: "parsel.Selector | None" = field(default=None, init=False, repr=False, compare=False)
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def text(self) -> str:
|
|
36
|
+
return self.body.decode(self.encoding, errors="replace")
|
|
37
|
+
|
|
38
|
+
def json(self) -> Any:
|
|
39
|
+
return orjson.loads(self.body)
|
|
40
|
+
|
|
41
|
+
def xpath(
|
|
42
|
+
self, query: str, namespaces: Mapping[str, str] | None = None,
|
|
43
|
+
**variables: Any,
|
|
44
|
+
) -> parsel.SelectorList[parsel.Selector]:
|
|
45
|
+
"""Select HTML/XML nodes, parsing the response only on first use."""
|
|
46
|
+
selector = self._selector
|
|
47
|
+
if selector is None:
|
|
48
|
+
content_type = self.headers.get("Content-Type", "").split(";", 1)[0].strip().lower()
|
|
49
|
+
kind = "xml" if content_type in {"application/xml", "text/xml"} or content_type.endswith("+xml") else None
|
|
50
|
+
selector = parsel.Selector(text=self.text, type=kind, base_url=self.url)
|
|
51
|
+
object.__setattr__(self, "_selector", selector)
|
|
52
|
+
return selector.xpath(query, namespaces=namespaces, **variables)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _options(options: dict[str, Any]) -> dict[str, Any]:
|
|
56
|
+
if "session" in options:
|
|
57
|
+
raise TypeError("Framework requests use the worker's session")
|
|
58
|
+
if "stream" in options:
|
|
59
|
+
raise TypeError("Use crawler.stream() for streaming responses")
|
|
60
|
+
if "json" in options and options["json"] is not None:
|
|
61
|
+
if options.get("data") is not None or options.get("content") is not None:
|
|
62
|
+
raise TypeError("json cannot be combined with data or content")
|
|
63
|
+
options["data"] = orjson.dumps(options.pop("json"))
|
|
64
|
+
headers = Headers(options.get("headers"))
|
|
65
|
+
headers.setdefault("Content-Type", "application/json")
|
|
66
|
+
options["headers"] = headers
|
|
67
|
+
return options
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
async def request(session: Session, url: str, method: str = "GET", **kwargs: Any) -> Response:
|
|
71
|
+
session.require_ready()
|
|
72
|
+
options = _options(kwargs)
|
|
73
|
+
started = time.monotonic()
|
|
74
|
+
try:
|
|
75
|
+
raw = await session.transport.request(method, url, **options)
|
|
76
|
+
except (Exception, asyncio.CancelledError):
|
|
77
|
+
session.retired = True
|
|
78
|
+
raise
|
|
79
|
+
if raw.status_code in RETIRE_STATUSES:
|
|
80
|
+
session.retired = True
|
|
81
|
+
return Response(
|
|
82
|
+
url=raw.url, status=raw.status_code, headers=raw.headers,
|
|
83
|
+
body=raw.content, elapsed=time.monotonic() - started,
|
|
84
|
+
encoding=raw.encoding or "utf-8",
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
async def _close_stream(response: CurlResponse) -> None:
|
|
89
|
+
# curl's aclose() only waits for the download. Cancel it explicitly on early
|
|
90
|
+
# exit so an unread/infinite body cannot keep the worker occupied forever.
|
|
91
|
+
task = response.astream_task
|
|
92
|
+
if response.quit_now is not None:
|
|
93
|
+
response.quit_now.set()
|
|
94
|
+
if isinstance(task, asyncio.Task) and not task.done():
|
|
95
|
+
task.cancel()
|
|
96
|
+
if task is not None:
|
|
97
|
+
# Consume the download's intentional cancellation, but let cancellation
|
|
98
|
+
# of this caller propagate. A broad suppress would swallow both.
|
|
99
|
+
result, = await asyncio.gather(task, return_exceptions=True)
|
|
100
|
+
if isinstance(result, asyncio.CancelledError):
|
|
101
|
+
return
|
|
102
|
+
if isinstance(result, BaseException):
|
|
103
|
+
raise result
|
|
104
|
+
await response.aclose()
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@asynccontextmanager
|
|
108
|
+
async def stream(
|
|
109
|
+
session: Session, url: str, method: str = "GET", **kwargs: Any,
|
|
110
|
+
) -> AsyncIterator[CurlResponse]:
|
|
111
|
+
session.require_ready()
|
|
112
|
+
options = _options(kwargs)
|
|
113
|
+
try:
|
|
114
|
+
response = await session.transport.request(method, url, stream=True, **options)
|
|
115
|
+
except (Exception, asyncio.CancelledError):
|
|
116
|
+
session.retired = True
|
|
117
|
+
raise
|
|
118
|
+
if response.status_code in RETIRE_STATUSES:
|
|
119
|
+
session.retired = True
|
|
120
|
+
try:
|
|
121
|
+
yield response
|
|
122
|
+
except (RequestException, asyncio.CancelledError):
|
|
123
|
+
session.retired = True
|
|
124
|
+
raise
|
|
125
|
+
finally:
|
|
126
|
+
try:
|
|
127
|
+
await _close_stream(response)
|
|
128
|
+
except (RequestException, asyncio.CancelledError):
|
|
129
|
+
session.retired = True
|
|
130
|
+
raise
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from typing import Protocol
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Notifier(Protocol):
|
|
5
|
+
"""Report successful crawls without blocking execution.
|
|
6
|
+
|
|
7
|
+
open() must return promptly; notify() must perform no blocking I/O.
|
|
8
|
+
close() must cooperate with cancellation and the runtime shutdown budget.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
async def open(self) -> None: ...
|
|
12
|
+
|
|
13
|
+
def notify(self) -> None: ...
|
|
14
|
+
|
|
15
|
+
async def close(self) -> None: ...
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Best-effort heartbeats over Magent's local Unix socket."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
import math
|
|
6
|
+
import os
|
|
7
|
+
|
|
8
|
+
import orjson
|
|
9
|
+
|
|
10
|
+
from ..errors import ConfigurationError
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class MagentNotifier:
|
|
14
|
+
def __init__(
|
|
15
|
+
self, app_id: str, *, instance: str | None = None,
|
|
16
|
+
socket_path: str = "/tmp/itxpmonitor.sock", timeout: float = 1.0,
|
|
17
|
+
) -> None:
|
|
18
|
+
for name, value in (("app_id", app_id), ("socket_path", socket_path)):
|
|
19
|
+
if not isinstance(value, str) or not value.strip():
|
|
20
|
+
raise ConfigurationError(f"Magent {name} must be a nonblank string")
|
|
21
|
+
if instance is not None and (not isinstance(instance, str) or not instance.strip()):
|
|
22
|
+
raise ConfigurationError("Magent instance must be a nonblank string or None")
|
|
23
|
+
if type(timeout) not in (int, float) or not math.isfinite(timeout) or timeout <= 0:
|
|
24
|
+
raise ConfigurationError("Magent timeout must be a positive finite number")
|
|
25
|
+
self.app_id = app_id
|
|
26
|
+
self.instance = instance
|
|
27
|
+
self.socket_path = socket_path
|
|
28
|
+
self.timeout = timeout
|
|
29
|
+
self._queue: asyncio.Queue[None] = asyncio.Queue()
|
|
30
|
+
self._reader: asyncio.StreamReader | None = None
|
|
31
|
+
self._writer: asyncio.StreamWriter | None = None
|
|
32
|
+
self._task: asyncio.Task[None] | None = None
|
|
33
|
+
self._accepting = False
|
|
34
|
+
|
|
35
|
+
async def open(self) -> None:
|
|
36
|
+
if self._task is not None:
|
|
37
|
+
raise RuntimeError("MagentNotifier is already open")
|
|
38
|
+
self._queue = asyncio.Queue()
|
|
39
|
+
self._accepting = True
|
|
40
|
+
self._task = asyncio.create_task(self._send(), name="zenx:notifier:magent")
|
|
41
|
+
|
|
42
|
+
def notify(self) -> None:
|
|
43
|
+
if self._accepting:
|
|
44
|
+
self._queue.put_nowait(None)
|
|
45
|
+
|
|
46
|
+
def _disconnect(self) -> None:
|
|
47
|
+
if self._writer is not None:
|
|
48
|
+
self._writer.transport.abort()
|
|
49
|
+
self._reader = self._writer = None
|
|
50
|
+
|
|
51
|
+
async def _send(self) -> None:
|
|
52
|
+
while True:
|
|
53
|
+
await self._queue.get()
|
|
54
|
+
try:
|
|
55
|
+
async with asyncio.timeout(self.timeout):
|
|
56
|
+
if self._writer is None:
|
|
57
|
+
self._reader, self._writer = await asyncio.open_unix_connection(self.socket_path)
|
|
58
|
+
elif self._reader is not None and self._reader.at_eof():
|
|
59
|
+
raise ConnectionError("Magent collector disconnected")
|
|
60
|
+
payload = {"type": "heartbeat", "app_id": self.app_id, "pid": os.getpid()}
|
|
61
|
+
if self.instance is not None:
|
|
62
|
+
payload["instance"] = self.instance
|
|
63
|
+
self._writer.write(orjson.dumps(payload, option=orjson.OPT_APPEND_NEWLINE))
|
|
64
|
+
await self._writer.drain()
|
|
65
|
+
except Exception:
|
|
66
|
+
self._disconnect()
|
|
67
|
+
logging.getLogger("zenx_lite.magent").warning("Magent heartbeat failed; discarded")
|
|
68
|
+
finally:
|
|
69
|
+
self._queue.task_done()
|
|
70
|
+
await asyncio.sleep(0)
|
|
71
|
+
|
|
72
|
+
async def close(self) -> None:
|
|
73
|
+
self._accepting = False
|
|
74
|
+
task, self._task = self._task, None
|
|
75
|
+
try:
|
|
76
|
+
if task is not None:
|
|
77
|
+
await self._queue.join()
|
|
78
|
+
finally:
|
|
79
|
+
if task is not None:
|
|
80
|
+
task.cancel()
|
|
81
|
+
self._disconnect()
|
|
82
|
+
while not self._queue.empty():
|
|
83
|
+
self._queue.get_nowait()
|
|
84
|
+
self._queue.task_done()
|
|
85
|
+
if task is not None:
|
|
86
|
+
await asyncio.gather(task, return_exceptions=True)
|
zenx_lite/scheduler.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Scheduling policy is independent of workers, HTTP, and delivery."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import math
|
|
5
|
+
from collections.abc import Awaitable, Callable
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Protocol
|
|
8
|
+
|
|
9
|
+
from .errors import ConfigurationError, FatalRuntimeError
|
|
10
|
+
|
|
11
|
+
type Crawl = Callable[[int], Awaitable[None]]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Scheduler(Protocol):
|
|
15
|
+
async def run(self, crawl: Crawl, *, concurrency: int, stop: asyncio.Event) -> None: ...
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True, slots=True)
|
|
19
|
+
class Slots:
|
|
20
|
+
interval: float = 1.0
|
|
21
|
+
|
|
22
|
+
def __post_init__(self) -> None:
|
|
23
|
+
if isinstance(self.interval, bool) or not isinstance(self.interval, (int, float)) or not math.isfinite(self.interval) or self.interval <= 0:
|
|
24
|
+
raise ConfigurationError("Slots interval must be a finite positive number")
|
|
25
|
+
|
|
26
|
+
async def run(self, crawl: Crawl, *, concurrency: int, stop: asyncio.Event) -> None:
|
|
27
|
+
if type(concurrency) is not int or concurrency <= 0:
|
|
28
|
+
raise ConfigurationError("concurrency must be a positive integer")
|
|
29
|
+
origin = asyncio.get_running_loop().time()
|
|
30
|
+
async with asyncio.TaskGroup() as tasks:
|
|
31
|
+
for worker_id in range(concurrency):
|
|
32
|
+
tasks.create_task(self._worker(crawl, worker_id, concurrency, origin, stop))
|
|
33
|
+
|
|
34
|
+
async def _worker(
|
|
35
|
+
self, crawl: Crawl, worker_id: int, concurrency: int,
|
|
36
|
+
origin: float, stop: asyncio.Event,
|
|
37
|
+
) -> None:
|
|
38
|
+
loop = asyncio.get_running_loop()
|
|
39
|
+
first = origin + worker_id * self.interval / concurrency
|
|
40
|
+
target = first
|
|
41
|
+
while not stop.is_set():
|
|
42
|
+
delay = target - loop.time()
|
|
43
|
+
if delay > 0:
|
|
44
|
+
try:
|
|
45
|
+
async with asyncio.timeout(delay):
|
|
46
|
+
await stop.wait()
|
|
47
|
+
return
|
|
48
|
+
except TimeoutError:
|
|
49
|
+
pass
|
|
50
|
+
if stop.is_set():
|
|
51
|
+
return
|
|
52
|
+
try:
|
|
53
|
+
await crawl(worker_id)
|
|
54
|
+
except FatalRuntimeError:
|
|
55
|
+
raise
|
|
56
|
+
except Exception as exc:
|
|
57
|
+
if isinstance(exc, BaseExceptionGroup) and exc.subgroup(FatalRuntimeError) is not None:
|
|
58
|
+
raise
|
|
59
|
+
# The execution callback already logged this crawl's failure.
|
|
60
|
+
pass
|
|
61
|
+
# Recompute from the origin, never from completion time or missed ticks.
|
|
62
|
+
step = max(1, math.floor((loop.time() - first) / self.interval) + 1)
|
|
63
|
+
target = first + step * self.interval
|