cli-watch-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cli_watch_kit/__init__.py +27 -0
- cli_watch_kit/app.py +175 -0
- cli_watch_kit/atomic.py +37 -0
- cli_watch_kit/budget.py +209 -0
- cli_watch_kit/changelog.py +87 -0
- cli_watch_kit/claude.py +195 -0
- cli_watch_kit/cli.py +410 -0
- cli_watch_kit/client.py +625 -0
- cli_watch_kit/daemon/__init__.py +25 -0
- cli_watch_kit/daemon/cache.py +48 -0
- cli_watch_kit/daemon/engine.py +874 -0
- cli_watch_kit/daemon/lifecycle.py +387 -0
- cli_watch_kit/daemon/scheduler.py +85 -0
- cli_watch_kit/daemon/server.py +116 -0
- cli_watch_kit/daemon/transport.py +324 -0
- cli_watch_kit/durations.py +28 -0
- cli_watch_kit/events.py +182 -0
- cli_watch_kit/locks.py +93 -0
- cli_watch_kit/paths.py +114 -0
- cli_watch_kit/py.typed +0 -0
- cli_watch_kit/sinks.py +106 -0
- cli_watch_kit/source.py +291 -0
- cli_watch_kit/timers.py +67 -0
- cli_watch_kit-0.1.0.dist-info/METADATA +207 -0
- cli_watch_kit-0.1.0.dist-info/RECORD +27 -0
- cli_watch_kit-0.1.0.dist-info/WHEEL +4 -0
- cli_watch_kit-0.1.0.dist-info/licenses/LICENSE +21 -0
cli_watch_kit/client.py
ADDED
|
@@ -0,0 +1,625 @@
|
|
|
1
|
+
"""The watcher: subscribe through a backend, poll it for events, write the changelog, deliver to a sink.
|
|
2
|
+
|
|
3
|
+
Two backends speak the same requests. :class:`DaemonBackend` talks to the shared per-user daemon and starts a new one
|
|
4
|
+
when it disappears. :class:`InProcessBackend` (`--no-daemon`) runs the same engine and request handler in a thread of
|
|
5
|
+
this process, with a private budget.
|
|
6
|
+
|
|
7
|
+
The watcher numbers the events it reports itself (`seq`), strictly increasing across daemon restarts and across
|
|
8
|
+
watcher runs on the same changelog. After a daemon restart it subscribes again with its baseline, so the new daemon
|
|
9
|
+
reports only what changed, and it drops the new `watch_started` when the snapshot did not move.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import asyncio
|
|
15
|
+
import concurrent.futures
|
|
16
|
+
import contextlib
|
|
17
|
+
import os
|
|
18
|
+
import signal
|
|
19
|
+
import sys
|
|
20
|
+
import threading
|
|
21
|
+
import time
|
|
22
|
+
import uuid
|
|
23
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
24
|
+
from dataclasses import dataclass, field, replace
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from types import FrameType
|
|
27
|
+
from typing import Any, Protocol
|
|
28
|
+
|
|
29
|
+
from cli_watch_kit.app import WatchApp
|
|
30
|
+
from cli_watch_kit.changelog import Changelog
|
|
31
|
+
from cli_watch_kit.daemon.engine import Engine
|
|
32
|
+
from cli_watch_kit.daemon.lifecycle import DEFAULT_REQUEST_TIMEOUT, HUNG_AFTER, ensure_daemon, kill_hung_daemon
|
|
33
|
+
from cli_watch_kit.daemon.server import handle
|
|
34
|
+
from cli_watch_kit.daemon.transport import (
|
|
35
|
+
DaemonFiles,
|
|
36
|
+
DaemonHung,
|
|
37
|
+
DaemonMismatch,
|
|
38
|
+
DaemonNotTrusted,
|
|
39
|
+
DaemonStartFailed,
|
|
40
|
+
DaemonTimeout,
|
|
41
|
+
DaemonUnavailable,
|
|
42
|
+
request,
|
|
43
|
+
)
|
|
44
|
+
from cli_watch_kit.durations import format_duration
|
|
45
|
+
from cli_watch_kit.events import WATCH_ENDED, WATCH_STARTED, Event, utc_now_iso
|
|
46
|
+
from cli_watch_kit.sinks import Sink, StatusOutput
|
|
47
|
+
from cli_watch_kit.source import NotFound, Params, Target
|
|
48
|
+
from cli_watch_kit.timers import ClientTimer, IdleTimer
|
|
49
|
+
|
|
50
|
+
OP_TIMEOUT = 660.0
|
|
51
|
+
"""An extra op may wait up to `Engine.await_budget`'s 600 s for the account's budget, then download something large."""
|
|
52
|
+
RETRYABLE_REASONS = frozenset({"error", "rate_limited"})
|
|
53
|
+
"""Reasons of a refused subscription that may pass by themselves: the source could not reach its API."""
|
|
54
|
+
EXIT_OK = 0
|
|
55
|
+
EXIT_ERROR = 1
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
STUCK_WRITE = 1.0
|
|
59
|
+
"""Seconds a write to the sink may take before a stop signal breaks it as stuck."""
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class _Interrupted(Exception):
|
|
63
|
+
"""Raised by the signal handler out of a write that blocks, so a stop is not stuck behind it."""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class BackendError(RuntimeError):
|
|
67
|
+
"""The daemon refused a request. `reason` is the source's error reason, when the source raised it."""
|
|
68
|
+
|
|
69
|
+
def __init__(self, message: str, reason: str | None = None) -> None:
|
|
70
|
+
super().__init__(message)
|
|
71
|
+
self.reason = reason
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class Backend(Protocol):
|
|
75
|
+
def describe(self) -> str: ...
|
|
76
|
+
|
|
77
|
+
def subscribe(
|
|
78
|
+
self,
|
|
79
|
+
client_id: str,
|
|
80
|
+
target: Target,
|
|
81
|
+
events: frozenset[str],
|
|
82
|
+
baseline: Any,
|
|
83
|
+
params: Params,
|
|
84
|
+
source_options: Mapping[str, Any],
|
|
85
|
+
) -> tuple[str, int]: ...
|
|
86
|
+
|
|
87
|
+
def poll(self, client_id: str, cursors: dict[str, int], acks: dict[str, int]) -> dict[str, Any]:
|
|
88
|
+
"""New events after the cursors. `acks` passes back the `acks` of the last answer that arrived."""
|
|
89
|
+
...
|
|
90
|
+
|
|
91
|
+
def unsubscribe(self, client_id: str, subscription_id: str) -> None: ...
|
|
92
|
+
|
|
93
|
+
def call(self, op: str, payload: Mapping[str, Any], params: Params | None = None) -> dict[str, Any]:
|
|
94
|
+
"""An extra op of the source, run by the daemon within its budget."""
|
|
95
|
+
...
|
|
96
|
+
|
|
97
|
+
def begin_shutdown(self) -> None:
|
|
98
|
+
"""The watcher is stopping: from now on, never start a daemon."""
|
|
99
|
+
...
|
|
100
|
+
|
|
101
|
+
def close(self) -> None: ...
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _check(response: dict[str, Any]) -> dict[str, Any]:
|
|
105
|
+
if not response.get("ok"):
|
|
106
|
+
raise BackendError(str(response.get("error", "the daemon refused the request")), response.get("reason"))
|
|
107
|
+
return response
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _subscribe_payload(
|
|
111
|
+
client_id: str,
|
|
112
|
+
target: Target,
|
|
113
|
+
events: frozenset[str],
|
|
114
|
+
baseline: Any,
|
|
115
|
+
params: Params,
|
|
116
|
+
source_options: Mapping[str, Any],
|
|
117
|
+
) -> dict[str, Any]:
|
|
118
|
+
options: dict[str, Any] = {"events": sorted(events)}
|
|
119
|
+
if baseline is not None:
|
|
120
|
+
options["baseline"] = baseline
|
|
121
|
+
payload: dict[str, Any] = {
|
|
122
|
+
"op": "subscribe",
|
|
123
|
+
"client_id": client_id,
|
|
124
|
+
"target": target.to_wire(),
|
|
125
|
+
"options": options,
|
|
126
|
+
}
|
|
127
|
+
if params:
|
|
128
|
+
payload["params"] = dict(params)
|
|
129
|
+
if source_options:
|
|
130
|
+
payload["source_options"] = dict(source_options)
|
|
131
|
+
return payload
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class DaemonBackend:
|
|
135
|
+
def __init__(self, app: WatchApp, files: DaemonFiles | None = None) -> None:
|
|
136
|
+
self.app = app
|
|
137
|
+
self.files = files or DaemonFiles.for_app(app)
|
|
138
|
+
self.hello = ensure_daemon(app, self.files)
|
|
139
|
+
self._shutting_down = False
|
|
140
|
+
self.timeout = app.env_float("REQUEST_TIMEOUT", DEFAULT_REQUEST_TIMEOUT)
|
|
141
|
+
self._timeouts = 0
|
|
142
|
+
|
|
143
|
+
def describe(self) -> str:
|
|
144
|
+
return f"pid {self.hello.get('pid')}, endpoint {self.files.describe_endpoint()}, log {self.files.log}"
|
|
145
|
+
|
|
146
|
+
def _call(self, payload: dict[str, Any], timeout: float | None = None) -> dict[str, Any]:
|
|
147
|
+
timeout = self.timeout if timeout is None else timeout
|
|
148
|
+
secret = bool(payload.get("params"))
|
|
149
|
+
try:
|
|
150
|
+
if secret:
|
|
151
|
+
self.files.check_trusted()
|
|
152
|
+
response = request(self.files, payload, timeout=timeout)
|
|
153
|
+
except DaemonNotTrusted:
|
|
154
|
+
raise
|
|
155
|
+
except DaemonUnavailable:
|
|
156
|
+
if self._shutting_down:
|
|
157
|
+
raise
|
|
158
|
+
# The daemon exited (idle, crash, drained or stopped by hand). Start a new one; subscriptions then come
|
|
159
|
+
# back as unknown and the watcher subscribes again.
|
|
160
|
+
self.hello = ensure_daemon(self.app, self.files)
|
|
161
|
+
if secret:
|
|
162
|
+
self.files.check_trusted()
|
|
163
|
+
response = request(self.files, payload, timeout=timeout)
|
|
164
|
+
error = str(response.get("error", ""))
|
|
165
|
+
if not response.get("ok") and error.startswith("unknown op"):
|
|
166
|
+
raise DaemonMismatch(
|
|
167
|
+
f"the daemon (pid {self.hello.get('pid')}, {self.app.name} {self.hello.get('app_version')}) answered"
|
|
168
|
+
f" {error}: it runs other code than this watcher. Run `{self.app.command} daemon stop`; the running"
|
|
169
|
+
" watchers then start a current daemon and subscribe again"
|
|
170
|
+
)
|
|
171
|
+
return _check(response)
|
|
172
|
+
|
|
173
|
+
def subscribe(
|
|
174
|
+
self,
|
|
175
|
+
client_id: str,
|
|
176
|
+
target: Target,
|
|
177
|
+
events: frozenset[str],
|
|
178
|
+
baseline: Any,
|
|
179
|
+
params: Params,
|
|
180
|
+
source_options: Mapping[str, Any],
|
|
181
|
+
) -> tuple[str, int]:
|
|
182
|
+
response = self._call(_subscribe_payload(client_id, target, events, baseline, params, source_options))
|
|
183
|
+
return str(response["subscription_id"]), int(response["cursor"])
|
|
184
|
+
|
|
185
|
+
def poll(self, client_id: str, cursors: dict[str, int], acks: dict[str, int]) -> dict[str, Any]:
|
|
186
|
+
"""A poll; a daemon that let `HUNG_AFTER` polls in a row time out is taken for hung and replaced.
|
|
187
|
+
|
|
188
|
+
It is killed only when its pid is verified (see `kill_hung_daemon`), else this raises `DaemonHung`. The
|
|
189
|
+
answer of the new daemon names the subscription unknown, and the watcher subscribes again."""
|
|
190
|
+
payload = {"op": "poll", "client_id": client_id, "cursors": cursors, "acks": acks}
|
|
191
|
+
try:
|
|
192
|
+
response = self._call(payload)
|
|
193
|
+
except DaemonTimeout:
|
|
194
|
+
self._timeouts += 1
|
|
195
|
+
if self._timeouts < HUNG_AFTER or self._shutting_down:
|
|
196
|
+
raise
|
|
197
|
+
self._timeouts = 0
|
|
198
|
+
if kill_hung_daemon(self.app, self.files) is None:
|
|
199
|
+
pid = self.files.owner_pid()
|
|
200
|
+
raise DaemonHung(
|
|
201
|
+
f"the daemon (pid {pid if pid is not None else 'unknown'}) did not answer {HUNG_AFTER} polls in a"
|
|
202
|
+
f" row and could not be verified and killed; run `{self.app.command} daemon stop`"
|
|
203
|
+
) from None
|
|
204
|
+
self.hello = ensure_daemon(self.app, self.files)
|
|
205
|
+
response = self._call(payload)
|
|
206
|
+
self._timeouts = 0
|
|
207
|
+
return response
|
|
208
|
+
|
|
209
|
+
def call(self, op: str, payload: Mapping[str, Any], params: Params | None = None) -> dict[str, Any]:
|
|
210
|
+
body: dict[str, Any] = {**payload, "op": op}
|
|
211
|
+
if params:
|
|
212
|
+
body["params"] = dict(params)
|
|
213
|
+
return self._call(body, OP_TIMEOUT)
|
|
214
|
+
|
|
215
|
+
def unsubscribe(self, client_id: str, subscription_id: str) -> None:
|
|
216
|
+
"""Best effort: one short attempt, never a spawn. If the daemon is gone, the lease expiry cleans up."""
|
|
217
|
+
payload = {"op": "unsubscribe", "client_id": client_id, "subscription_id": subscription_id}
|
|
218
|
+
with contextlib.suppress(DaemonUnavailable, OSError, ValueError):
|
|
219
|
+
request(self.files, payload, timeout=1.0)
|
|
220
|
+
|
|
221
|
+
def begin_shutdown(self) -> None:
|
|
222
|
+
self._shutting_down = True
|
|
223
|
+
|
|
224
|
+
def close(self) -> None:
|
|
225
|
+
pass
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
class InProcessBackend:
|
|
229
|
+
"""The engine and the daemon's request handler on their own event loop in a daemon thread."""
|
|
230
|
+
|
|
231
|
+
def __init__(self, app: WatchApp, engine: Engine | None = None) -> None:
|
|
232
|
+
self.app = app
|
|
233
|
+
self.engine = engine or Engine.for_app(app)
|
|
234
|
+
self._identity = app.identity()
|
|
235
|
+
self._loop = asyncio.new_event_loop()
|
|
236
|
+
self._stop: asyncio.Event | None = None
|
|
237
|
+
self._ready = threading.Event()
|
|
238
|
+
self._thread = threading.Thread(target=self._run, name=f"{app.name}-engine", daemon=True)
|
|
239
|
+
self._thread.start()
|
|
240
|
+
self._ready.wait()
|
|
241
|
+
|
|
242
|
+
def _run(self) -> None:
|
|
243
|
+
asyncio.set_event_loop(self._loop)
|
|
244
|
+
|
|
245
|
+
async def main() -> None:
|
|
246
|
+
self._stop = asyncio.Event()
|
|
247
|
+
self._ready.set()
|
|
248
|
+
await self.engine.run(self._stop)
|
|
249
|
+
|
|
250
|
+
self._loop.run_until_complete(main())
|
|
251
|
+
|
|
252
|
+
def _call(self, payload: dict[str, Any], timeout: float = 30.0) -> dict[str, Any]:
|
|
253
|
+
# A private stop event: `stop` and `drain` from inside the process must not end the engine under the watcher.
|
|
254
|
+
async def invoke() -> dict[str, Any]:
|
|
255
|
+
return await handle(self.engine, payload, asyncio.Event(), self._identity)
|
|
256
|
+
|
|
257
|
+
future: concurrent.futures.Future[dict[str, Any]] = asyncio.run_coroutine_threadsafe(invoke(), self._loop)
|
|
258
|
+
return _check(future.result(timeout=timeout))
|
|
259
|
+
|
|
260
|
+
def describe(self) -> str:
|
|
261
|
+
return "in-process (--no-daemon), private budget"
|
|
262
|
+
|
|
263
|
+
def subscribe(
|
|
264
|
+
self,
|
|
265
|
+
client_id: str,
|
|
266
|
+
target: Target,
|
|
267
|
+
events: frozenset[str],
|
|
268
|
+
baseline: Any,
|
|
269
|
+
params: Params,
|
|
270
|
+
source_options: Mapping[str, Any],
|
|
271
|
+
) -> tuple[str, int]:
|
|
272
|
+
response = self._call(_subscribe_payload(client_id, target, events, baseline, params, source_options))
|
|
273
|
+
return str(response["subscription_id"]), int(response["cursor"])
|
|
274
|
+
|
|
275
|
+
def poll(self, client_id: str, cursors: dict[str, int], acks: dict[str, int]) -> dict[str, Any]:
|
|
276
|
+
return self._call({"op": "poll", "client_id": client_id, "cursors": cursors, "acks": acks})
|
|
277
|
+
|
|
278
|
+
def call(self, op: str, payload: Mapping[str, Any], params: Params | None = None) -> dict[str, Any]:
|
|
279
|
+
body: dict[str, Any] = {**payload, "op": op}
|
|
280
|
+
if params:
|
|
281
|
+
body["params"] = dict(params)
|
|
282
|
+
return self._call(body, OP_TIMEOUT)
|
|
283
|
+
|
|
284
|
+
def unsubscribe(self, client_id: str, subscription_id: str) -> None:
|
|
285
|
+
self._call({"op": "unsubscribe", "client_id": client_id, "subscription_id": subscription_id})
|
|
286
|
+
|
|
287
|
+
def begin_shutdown(self) -> None:
|
|
288
|
+
pass
|
|
289
|
+
|
|
290
|
+
def close(self) -> None:
|
|
291
|
+
stop = self._stop
|
|
292
|
+
if stop is not None:
|
|
293
|
+
self._loop.call_soon_threadsafe(stop.set)
|
|
294
|
+
self._thread.join(timeout=10)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def default_backend(app: WatchApp, no_daemon: bool) -> Backend:
|
|
298
|
+
if no_daemon:
|
|
299
|
+
return InProcessBackend(app)
|
|
300
|
+
return DaemonBackend(app)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
class Writer(Protocol):
|
|
304
|
+
"""An app's writer of the resource dir: the state file from each snapshot, artifacts attached to events."""
|
|
305
|
+
|
|
306
|
+
def snapshot(self, data: Any) -> None: ...
|
|
307
|
+
|
|
308
|
+
def attach(self, events: list[Event]) -> list[Event]: ...
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
@dataclass(frozen=True)
|
|
312
|
+
class WriterContext:
|
|
313
|
+
"""What an app's writer is made from: the base dir and the resource dir of the watch, its target, the backend
|
|
314
|
+
(for the source's extra ops, such as fetching a log the daemon caches) and the status output."""
|
|
315
|
+
|
|
316
|
+
base_dir: Path
|
|
317
|
+
resource_dir: Path
|
|
318
|
+
target: Target
|
|
319
|
+
backend: Backend
|
|
320
|
+
status: StatusOutput
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
@dataclass(frozen=True)
|
|
324
|
+
class WatchSettings:
|
|
325
|
+
"""What one watch reports and when it ends.
|
|
326
|
+
|
|
327
|
+
`events` are the kinds reported, lifecycle and threshold kinds included; `subscribed` are the kinds asked of the
|
|
328
|
+
daemon (default: `events` without the thresholds, which the caller passes via `threshold_kinds`). `params` (secrets
|
|
329
|
+
such as an API key) and `source_options` (what the source should do for this watcher) go with the subscription
|
|
330
|
+
to the daemon. A `watch_ended` whose reason is in `error_reasons` exits with 1.
|
|
331
|
+
"""
|
|
332
|
+
|
|
333
|
+
target: Target
|
|
334
|
+
events: frozenset[str]
|
|
335
|
+
until: str | None = None
|
|
336
|
+
timeout: float | None = None
|
|
337
|
+
idle_after: float | None = None
|
|
338
|
+
params: Mapping[str, Any] = field(default_factory=dict)
|
|
339
|
+
source_options: Mapping[str, Any] = field(default_factory=dict)
|
|
340
|
+
threshold_kinds: frozenset[str] = frozenset()
|
|
341
|
+
subscribed: frozenset[str] | None = None
|
|
342
|
+
error_reasons: frozenset[str] = frozenset({NotFound.reason})
|
|
343
|
+
|
|
344
|
+
@property
|
|
345
|
+
def subscribed_events(self) -> frozenset[str]:
|
|
346
|
+
if self.subscribed is not None:
|
|
347
|
+
return self.subscribed
|
|
348
|
+
return self.events - self.threshold_kinds
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
class Watcher:
|
|
352
|
+
"""One watch: subscribes, polls the backend, numbers and delivers the events, and ends.
|
|
353
|
+
|
|
354
|
+
It does not lock its resource dir. `run_watch` does, so that a second watcher of the same resource is refused;
|
|
355
|
+
a CLI that builds Watcher objects itself must make sure no two of them share a changelog.
|
|
356
|
+
"""
|
|
357
|
+
|
|
358
|
+
def __init__(
|
|
359
|
+
self,
|
|
360
|
+
settings: WatchSettings,
|
|
361
|
+
backend: Backend,
|
|
362
|
+
sink: Sink,
|
|
363
|
+
changelog: Changelog,
|
|
364
|
+
status: StatusOutput,
|
|
365
|
+
*,
|
|
366
|
+
writer: Writer | None = None,
|
|
367
|
+
timers: Sequence[ClientTimer] = (),
|
|
368
|
+
clock: Callable[[], float] = time.monotonic,
|
|
369
|
+
retry_delays: tuple[float, float] = (1.0, 60.0),
|
|
370
|
+
) -> None:
|
|
371
|
+
self.settings = settings
|
|
372
|
+
self.backend = backend
|
|
373
|
+
self.sink = sink
|
|
374
|
+
self.changelog = changelog
|
|
375
|
+
self.status = status
|
|
376
|
+
self.writer = writer
|
|
377
|
+
self.timers = list(timers)
|
|
378
|
+
self.clock = clock
|
|
379
|
+
self.retry_delays = retry_delays
|
|
380
|
+
"""The first and the longest wait between attempts to subscribe again."""
|
|
381
|
+
self.client_id = uuid.uuid4().hex
|
|
382
|
+
self._stop_reason: str | None = None
|
|
383
|
+
self._emitting_since: float | None = None
|
|
384
|
+
self._stuck = False
|
|
385
|
+
self._wakeup = threading.Event()
|
|
386
|
+
self._subscription: str | None = None
|
|
387
|
+
# The daemon's sequence number of the last event received. It restarts when the daemon restarts, so it is
|
|
388
|
+
# only the poll cursor. Reported events carry the watcher's own `_seq`, which never goes back.
|
|
389
|
+
self._cursor = 0
|
|
390
|
+
self._acks: dict[str, int] = {}
|
|
391
|
+
self._seq = 0
|
|
392
|
+
self._snapshot_hash: str | None = None
|
|
393
|
+
|
|
394
|
+
def request_stop(self, reason: str = "stopped") -> None:
|
|
395
|
+
self._stop_reason = reason
|
|
396
|
+
self.backend.begin_shutdown()
|
|
397
|
+
self._wakeup.set()
|
|
398
|
+
|
|
399
|
+
def _install_signal_handlers(self) -> dict[int, Any]:
|
|
400
|
+
"""Turn SIGTERM and SIGINT into a stop with `watch_ended`. Return the handlers to put back afterwards."""
|
|
401
|
+
if threading.current_thread() is not threading.main_thread():
|
|
402
|
+
return {}
|
|
403
|
+
|
|
404
|
+
def handler(signum: int, frame: FrameType | None) -> None:
|
|
405
|
+
self.request_stop(f"signal {signal.Signals(signum).name}")
|
|
406
|
+
started = self._emitting_since
|
|
407
|
+
if started is None:
|
|
408
|
+
return
|
|
409
|
+
if time.monotonic() - started >= STUCK_WRITE:
|
|
410
|
+
# A write to a reader that stopped reading (a full pipe) would block the stop forever.
|
|
411
|
+
raise _Interrupted
|
|
412
|
+
|
|
413
|
+
# A write in progress is let finish; if it still runs when this fires, it is stuck, and the signal
|
|
414
|
+
# comes again to break it.
|
|
415
|
+
def again() -> None:
|
|
416
|
+
if self._emitting_since == started:
|
|
417
|
+
os.kill(os.getpid(), signum)
|
|
418
|
+
|
|
419
|
+
timer = threading.Timer(STUCK_WRITE, again)
|
|
420
|
+
timer.daemon = True
|
|
421
|
+
timer.start()
|
|
422
|
+
|
|
423
|
+
return {sig: signal.signal(sig, handler) for sig in (signal.SIGTERM, signal.SIGINT)}
|
|
424
|
+
|
|
425
|
+
def _subscribe(self, resubscribe: bool) -> None:
|
|
426
|
+
baseline = None
|
|
427
|
+
if resubscribe:
|
|
428
|
+
stored = self.changelog.load_baseline()
|
|
429
|
+
baseline = stored.snapshot if stored else None
|
|
430
|
+
s = self.settings
|
|
431
|
+
self._subscription, self._cursor = self.backend.subscribe(
|
|
432
|
+
self.client_id, s.target, s.subscribed_events, baseline, s.params, s.source_options
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
def _resubscribe(self, deadline: float | None) -> bool:
|
|
436
|
+
"""Subscribe again with the baseline, retrying with a backoff while the trouble may pass (a daemon that is
|
|
437
|
+
just restarting, an API that is unreachable after a suspend). Return whether it worked; on a stop or the
|
|
438
|
+
deadline it gives up and the loop ends the watch."""
|
|
439
|
+
delay, longest = self.retry_delays
|
|
440
|
+
while True:
|
|
441
|
+
try:
|
|
442
|
+
self._subscribe(resubscribe=True)
|
|
443
|
+
return True
|
|
444
|
+
except BackendError as exc:
|
|
445
|
+
if exc.reason not in RETRYABLE_REASONS:
|
|
446
|
+
raise
|
|
447
|
+
error: Exception = exc
|
|
448
|
+
except (DaemonStartFailed, DaemonNotTrusted, DaemonHung):
|
|
449
|
+
raise
|
|
450
|
+
except (DaemonUnavailable, TimeoutError) as exc:
|
|
451
|
+
error = exc
|
|
452
|
+
if isinstance(exc, DaemonTimeout):
|
|
453
|
+
# The daemon may have created the subscription after all. A new client id lets that one expire
|
|
454
|
+
# with its lease, instead of being renewed by this watcher's polls for nothing.
|
|
455
|
+
self.client_id = uuid.uuid4().hex
|
|
456
|
+
if deadline is not None and self.clock() + delay >= deadline:
|
|
457
|
+
self.request_stop("timeout")
|
|
458
|
+
return False
|
|
459
|
+
self.status.line(f"subscribing again failed: {error}; trying again in {format_duration(delay)}")
|
|
460
|
+
self._wakeup.wait(delay)
|
|
461
|
+
if self._stop_reason is not None:
|
|
462
|
+
return False
|
|
463
|
+
delay = min(delay * 2, longest)
|
|
464
|
+
|
|
465
|
+
def _next_seq(self) -> int:
|
|
466
|
+
self._seq += 1
|
|
467
|
+
return self._seq
|
|
468
|
+
|
|
469
|
+
def _end_event(self, reason: str, **data: Any) -> Event:
|
|
470
|
+
key = self.settings.target.key
|
|
471
|
+
return Event(WATCH_ENDED, key, self._next_seq(), utc_now_iso(), {"reason": reason, **data})
|
|
472
|
+
|
|
473
|
+
def _deliver(self, events: list[Event]) -> None:
|
|
474
|
+
if not events:
|
|
475
|
+
return
|
|
476
|
+
self.changelog.append(events)
|
|
477
|
+
self._to_sink(lambda: self.sink.emit(events))
|
|
478
|
+
|
|
479
|
+
def _to_sink(self, write: Callable[[], None]) -> None:
|
|
480
|
+
"""Run a write to the sink (an emit, a tick that sends a buffered batch) that a stop signal may break when it
|
|
481
|
+
blocks. A sink that blocked once gets nothing more; the changelog keeps every event."""
|
|
482
|
+
if self._stuck:
|
|
483
|
+
return
|
|
484
|
+
self._emitting_since = time.monotonic()
|
|
485
|
+
try:
|
|
486
|
+
write()
|
|
487
|
+
except _Interrupted:
|
|
488
|
+
self._stuck = True
|
|
489
|
+
finally:
|
|
490
|
+
self._emitting_since = None
|
|
491
|
+
|
|
492
|
+
def _abandon_stdout(self) -> None:
|
|
493
|
+
"""The reader of stdout stopped reading. Python flushes stdout at exit, which would block on the same full
|
|
494
|
+
pipe, so point this process's stdout at /dev/null. Only this process's fd changes: the pipe, which other
|
|
495
|
+
processes may share, keeps its mode."""
|
|
496
|
+
self.status.line("the event stream stopped taking events; they are in the changelog")
|
|
497
|
+
if self.sink.name != "stdout":
|
|
498
|
+
return
|
|
499
|
+
with contextlib.suppress(OSError, ValueError):
|
|
500
|
+
devnull = os.open(os.devnull, os.O_WRONLY)
|
|
501
|
+
os.dup2(devnull, sys.stdout.fileno())
|
|
502
|
+
os.close(devnull)
|
|
503
|
+
|
|
504
|
+
def _finish(self, event: Event) -> None:
|
|
505
|
+
self._deliver([event])
|
|
506
|
+
self.status.line(f"ended: {event.data.get('reason')}")
|
|
507
|
+
|
|
508
|
+
def run(self) -> int:
|
|
509
|
+
previous_handlers = self._install_signal_handlers()
|
|
510
|
+
s = self.settings
|
|
511
|
+
deadline = self.clock() + s.timeout if s.timeout is not None else None
|
|
512
|
+
stored = self.changelog.load_baseline()
|
|
513
|
+
self._snapshot_hash = stored.snapshot_hash if stored else None
|
|
514
|
+
# Continue the changelog's numbering, so events.jsonl stays strictly increasing across watcher runs too.
|
|
515
|
+
self._seq = self.changelog.last_seq()
|
|
516
|
+
resubscribed = False
|
|
517
|
+
idle = IdleTimer(s.idle_after, s.threshold_kinds, self.clock())
|
|
518
|
+
try:
|
|
519
|
+
self._subscribe(resubscribe=False)
|
|
520
|
+
while True:
|
|
521
|
+
if self._stop_reason is not None:
|
|
522
|
+
self._finish(self._end_event(self._stop_reason))
|
|
523
|
+
return EXIT_OK
|
|
524
|
+
assert self._subscription is not None
|
|
525
|
+
self._to_sink(self.sink.tick)
|
|
526
|
+
try:
|
|
527
|
+
response = self.backend.poll(self.client_id, {self._subscription: self._cursor}, self._acks)
|
|
528
|
+
except DaemonTimeout as exc:
|
|
529
|
+
# A busy daemon: polling is safe to repeat, so ask again rather than give up the watch.
|
|
530
|
+
if deadline is not None and self.clock() >= deadline:
|
|
531
|
+
self._finish(self._end_event("timeout"))
|
|
532
|
+
return EXIT_OK
|
|
533
|
+
self.status.line(f"{exc}; polling again ({HUNG_AFTER} in a row and the daemon is replaced)")
|
|
534
|
+
self._wakeup.wait(1.0)
|
|
535
|
+
continue
|
|
536
|
+
self._acks = {k: int(v) for k, v in (response.get("acks") or {}).items()}
|
|
537
|
+
if self._subscription in response.get("unknown", []):
|
|
538
|
+
self.status.line("daemon restarted, subscribing again")
|
|
539
|
+
if self._resubscribe(deadline):
|
|
540
|
+
resubscribed = True
|
|
541
|
+
continue
|
|
542
|
+
previous_hash = self._snapshot_hash
|
|
543
|
+
snapshot = (response.get("snapshots") or {}).get(self._subscription)
|
|
544
|
+
if snapshot:
|
|
545
|
+
self._snapshot_hash = str(snapshot["hash"])
|
|
546
|
+
self.changelog.save_baseline(self._snapshot_hash, snapshot.get("data"))
|
|
547
|
+
if self.writer is not None:
|
|
548
|
+
self.writer.snapshot(snapshot.get("data"))
|
|
549
|
+
for timer in self.timers:
|
|
550
|
+
timer.snapshot(snapshot.get("data"), self.clock())
|
|
551
|
+
raw = [Event.from_json(e) for e in (response.get("events") or {}).get(self._subscription) or []]
|
|
552
|
+
for event in raw:
|
|
553
|
+
self._cursor = max(self._cursor, event.seq)
|
|
554
|
+
events = [e for e in raw if e.kind in s.events]
|
|
555
|
+
if resubscribed and any(e.kind == WATCH_STARTED for e in events):
|
|
556
|
+
# After a daemon restart, the new watch_started repeats what we already know unless the snapshot
|
|
557
|
+
# moved; changes in between arrive as normal events, diffed against our stored snapshot.
|
|
558
|
+
events = [
|
|
559
|
+
e
|
|
560
|
+
for e in events
|
|
561
|
+
if not (e.kind == WATCH_STARTED and e.data.get("snapshot_hash") == previous_hash)
|
|
562
|
+
]
|
|
563
|
+
resubscribed = False
|
|
564
|
+
if self.writer is not None:
|
|
565
|
+
events = self.writer.attach(events)
|
|
566
|
+
now = self.clock()
|
|
567
|
+
for timer in self.timers:
|
|
568
|
+
timer.reported(events, now)
|
|
569
|
+
events += [Event(kind, s.target.key, 0, utc_now_iso(), dict(data)) for kind, data in timer.due(now)]
|
|
570
|
+
idle.reported(events, now)
|
|
571
|
+
events = [replace(e, seq=self._next_seq()) for e in events]
|
|
572
|
+
ended = next((e for e in events if e.kind == WATCH_ENDED), None)
|
|
573
|
+
hit = next((e for e in events if s.until and e.kind == s.until), None)
|
|
574
|
+
self._deliver(events)
|
|
575
|
+
if ended is not None:
|
|
576
|
+
reason = str(ended.data.get("reason"))
|
|
577
|
+
already = f" (already {reason} when the watch started)" if ended.data.get("already_ended") else ""
|
|
578
|
+
self.status.line(f"ended: {reason}{already}")
|
|
579
|
+
return EXIT_ERROR if reason in s.error_reasons else EXIT_OK
|
|
580
|
+
if hit is not None:
|
|
581
|
+
self._finish(self._end_event("until", event=hit.kind, seq=hit.seq))
|
|
582
|
+
return EXIT_OK
|
|
583
|
+
if not self.sink.alive():
|
|
584
|
+
self._finish(self._end_event("session_gone"))
|
|
585
|
+
return EXIT_OK
|
|
586
|
+
if deadline is not None and self.clock() >= deadline:
|
|
587
|
+
self._finish(self._end_event("timeout"))
|
|
588
|
+
return EXIT_OK
|
|
589
|
+
if idle.idle(self.clock()):
|
|
590
|
+
self._finish(self._end_event("idle", idle_after=format_duration(s.idle_after or 0.0)))
|
|
591
|
+
return EXIT_OK
|
|
592
|
+
self._wakeup.wait(self._wait(response, deadline, idle))
|
|
593
|
+
except (BackendError, DaemonMismatch, DaemonUnavailable, TimeoutError) as exc:
|
|
594
|
+
if self._stop_reason is not None:
|
|
595
|
+
# Stopped while a request was in flight, and the daemon is gone: end like any other stop.
|
|
596
|
+
self._finish(self._end_event(self._stop_reason))
|
|
597
|
+
return EXIT_OK
|
|
598
|
+
self.status.line(f"error: {exc}")
|
|
599
|
+
return EXIT_ERROR
|
|
600
|
+
finally:
|
|
601
|
+
self.backend.begin_shutdown()
|
|
602
|
+
if self._subscription is not None:
|
|
603
|
+
with contextlib.suppress(BackendError, DaemonUnavailable, OSError, TimeoutError):
|
|
604
|
+
self.backend.unsubscribe(self.client_id, self._subscription)
|
|
605
|
+
try:
|
|
606
|
+
if self._stuck:
|
|
607
|
+
self._abandon_stdout()
|
|
608
|
+
else:
|
|
609
|
+
self.sink.close()
|
|
610
|
+
finally:
|
|
611
|
+
for sig, previous in previous_handlers.items():
|
|
612
|
+
# `None` means a handler not installed from Python; the default is the closest one to put back.
|
|
613
|
+
signal.signal(sig, previous if previous is not None else signal.SIG_DFL)
|
|
614
|
+
|
|
615
|
+
def _wait(self, response: Mapping[str, Any], deadline: float | None, idle: IdleTimer) -> float:
|
|
616
|
+
now = self.clock()
|
|
617
|
+
wait = int(response.get("next_poll_ms", 5000)) / 1000.0
|
|
618
|
+
if deadline is not None:
|
|
619
|
+
wait = max(0.0, min(wait, deadline - now))
|
|
620
|
+
candidates = [t.seconds_until_next(now) for t in self.timers]
|
|
621
|
+
candidates += [idle.seconds_until_next(now), self.sink.seconds_until_flush()]
|
|
622
|
+
for candidate in candidates:
|
|
623
|
+
if candidate is not None:
|
|
624
|
+
wait = min(wait, candidate)
|
|
625
|
+
return max(0.0, wait)
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""The shared polling daemon. An app's daemon module is two lines:
|
|
2
|
+
|
|
3
|
+
```python
|
|
4
|
+
from cli_watch_kit.daemon import main
|
|
5
|
+
from my_cli.watch import APP
|
|
6
|
+
|
|
7
|
+
if __name__ == "__main__":
|
|
8
|
+
main(APP)
|
|
9
|
+
```
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import sys
|
|
15
|
+
from typing import TYPE_CHECKING, NoReturn
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from cli_watch_kit.app import WatchApp
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def main(app: WatchApp) -> NoReturn:
|
|
22
|
+
"""Run the daemon of `app`. Watchers start it with `python -m <app.daemon_module>`; it is not run by hand."""
|
|
23
|
+
from cli_watch_kit.daemon.lifecycle import run_daemon
|
|
24
|
+
|
|
25
|
+
sys.exit(run_daemon(app, sys.argv[1:]))
|