cli-watch-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cli_watch_kit/__init__.py +27 -0
- cli_watch_kit/app.py +175 -0
- cli_watch_kit/atomic.py +37 -0
- cli_watch_kit/budget.py +209 -0
- cli_watch_kit/changelog.py +87 -0
- cli_watch_kit/claude.py +195 -0
- cli_watch_kit/cli.py +410 -0
- cli_watch_kit/client.py +625 -0
- cli_watch_kit/daemon/__init__.py +25 -0
- cli_watch_kit/daemon/cache.py +48 -0
- cli_watch_kit/daemon/engine.py +874 -0
- cli_watch_kit/daemon/lifecycle.py +387 -0
- cli_watch_kit/daemon/scheduler.py +85 -0
- cli_watch_kit/daemon/server.py +116 -0
- cli_watch_kit/daemon/transport.py +324 -0
- cli_watch_kit/durations.py +28 -0
- cli_watch_kit/events.py +182 -0
- cli_watch_kit/locks.py +93 -0
- cli_watch_kit/paths.py +114 -0
- cli_watch_kit/py.typed +0 -0
- cli_watch_kit/sinks.py +106 -0
- cli_watch_kit/source.py +291 -0
- cli_watch_kit/timers.py +67 -0
- cli_watch_kit-0.1.0.dist-info/METADATA +207 -0
- cli_watch_kit-0.1.0.dist-info/RECORD +27 -0
- cli_watch_kit-0.1.0.dist-info/WHEEL +4 -0
- cli_watch_kit-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
"""Daemon lifecycle: the daemon's own main loop, and the watcher side that finds, starts or replaces it.
|
|
2
|
+
|
|
3
|
+
There is one daemon per user, app and pair of protocol versions. It holds an exclusive lock on its lock file for its
|
|
4
|
+
whole life, and exits after `idle_timeout` seconds without subscriptions. Watchers start it on demand; the spawn lock
|
|
5
|
+
makes concurrent watchers start one daemon, not one each.
|
|
6
|
+
|
|
7
|
+
The daemon is shared by every watcher of the user, so it must not be a descendant of the watcher that started it:
|
|
8
|
+
tools that stop a process together with its whole tree (Claude Code's task stop does) would take it down too. A new
|
|
9
|
+
session is not enough, because the parent stays the same. On POSIX the spawned process therefore forks once more and
|
|
10
|
+
exits as soon as the daemon is ready, so the daemon is reparented to init or the nearest subreaper.
|
|
11
|
+
|
|
12
|
+
**Upgrades.** The protocol versions in the file names keep incompatible daemons apart. Within one protocol, a daemon
|
|
13
|
+
may still run older code than a watcher, for example after `git pull` in an editable install. `hello` reports the
|
|
14
|
+
daemon's code stamp; a watcher whose code is newer asks the daemon to `drain` (stop), and starts a current one. The
|
|
15
|
+
watchers of the old daemon then resubscribe from their baselines and lose nothing. A watcher with older code than the
|
|
16
|
+
daemon uses it as it is.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import asyncio
|
|
22
|
+
import contextlib
|
|
23
|
+
import itertools
|
|
24
|
+
import logging
|
|
25
|
+
import os
|
|
26
|
+
import signal
|
|
27
|
+
import subprocess
|
|
28
|
+
import sys
|
|
29
|
+
import time
|
|
30
|
+
import traceback
|
|
31
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
from typing import Any
|
|
34
|
+
|
|
35
|
+
from cli_watch_kit.app import WatchApp
|
|
36
|
+
from cli_watch_kit.daemon.engine import Engine
|
|
37
|
+
from cli_watch_kit.daemon.server import handle
|
|
38
|
+
from cli_watch_kit.daemon.transport import (
|
|
39
|
+
DaemonFiles,
|
|
40
|
+
DaemonHung,
|
|
41
|
+
DaemonStartFailed,
|
|
42
|
+
DaemonTimeout,
|
|
43
|
+
DaemonUnavailable,
|
|
44
|
+
request,
|
|
45
|
+
start_server,
|
|
46
|
+
)
|
|
47
|
+
from cli_watch_kit.locks import FileLock
|
|
48
|
+
from cli_watch_kit.paths import ensure_private_dir
|
|
49
|
+
|
|
50
|
+
log = logging.getLogger("cli_watch_kit.daemon")
|
|
51
|
+
|
|
52
|
+
DEFAULT_IDLE_TIMEOUT = 30.0
|
|
53
|
+
DEFAULT_REQUEST_TIMEOUT = 10.0
|
|
54
|
+
"""Seconds a watcher waits for the daemon's answer; `<PREFIX>REQUEST_TIMEOUT` overrides it."""
|
|
55
|
+
HUNG_AFTER = 3
|
|
56
|
+
"""Polls in a row without an answer after which a watcher takes the daemon for hung and replaces it."""
|
|
57
|
+
SPAWN_TIMEOUT = 10.0
|
|
58
|
+
MAX_LOG_BYTES = 5 * 1024 * 1024
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
async def serve(
|
|
62
|
+
app: WatchApp,
|
|
63
|
+
files: DaemonFiles,
|
|
64
|
+
engine: Engine,
|
|
65
|
+
idle_timeout: float,
|
|
66
|
+
on_ready: Callable[[], None] | None = None,
|
|
67
|
+
) -> None:
|
|
68
|
+
stop = asyncio.Event()
|
|
69
|
+
loop = asyncio.get_running_loop()
|
|
70
|
+
if os.name != "nt":
|
|
71
|
+
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
72
|
+
loop.add_signal_handler(sig, stop.set)
|
|
73
|
+
identity = app.identity()
|
|
74
|
+
|
|
75
|
+
async def on_request(req: dict[str, Any]) -> dict[str, Any]:
|
|
76
|
+
return await handle(engine, req, stop, identity)
|
|
77
|
+
|
|
78
|
+
listener = await start_server(files, on_request)
|
|
79
|
+
log.info(
|
|
80
|
+
"daemon %s %s (cli-watch-kit %s, code %s) started, pid %d, endpoint %s, idle timeout %.0f s",
|
|
81
|
+
app.name,
|
|
82
|
+
app.version,
|
|
83
|
+
app.lib_version,
|
|
84
|
+
identity["fingerprint"],
|
|
85
|
+
os.getpid(),
|
|
86
|
+
files.describe_endpoint(),
|
|
87
|
+
idle_timeout,
|
|
88
|
+
)
|
|
89
|
+
if on_ready is not None:
|
|
90
|
+
on_ready()
|
|
91
|
+
engine_task = asyncio.create_task(engine.run(stop))
|
|
92
|
+
try:
|
|
93
|
+
while not stop.is_set():
|
|
94
|
+
with contextlib.suppress(TimeoutError):
|
|
95
|
+
await asyncio.wait_for(stop.wait(), 1.0)
|
|
96
|
+
if engine_task.done():
|
|
97
|
+
# The engine never stops on its own: without it the watchers would wait forever. Exit, so they
|
|
98
|
+
# start a new daemon and resubscribe.
|
|
99
|
+
reason = engine_task.exception() if not engine_task.cancelled() else "cancelled"
|
|
100
|
+
log.error("the engine stopped (%r), exiting", reason)
|
|
101
|
+
stop.set()
|
|
102
|
+
elif engine.idle_for(engine.clock()) >= idle_timeout:
|
|
103
|
+
log.info("no subscriptions for %.0f s, exiting", idle_timeout)
|
|
104
|
+
stop.set()
|
|
105
|
+
finally:
|
|
106
|
+
# Remove the endpoint first, so a watcher that connects from now on starts a new daemon.
|
|
107
|
+
listener.cleanup()
|
|
108
|
+
await listener.close()
|
|
109
|
+
stop.set()
|
|
110
|
+
with contextlib.suppress(Exception, asyncio.CancelledError):
|
|
111
|
+
await engine_task
|
|
112
|
+
log.info("daemon stopped")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def run_daemon(app: WatchApp, argv: Sequence[str] = ()) -> int:
|
|
116
|
+
"""The daemon process. On POSIX it detaches first, unless `--foreground` is given (for debugging)."""
|
|
117
|
+
if os.name != "nt" and "--foreground" not in argv:
|
|
118
|
+
return _detach(app)
|
|
119
|
+
return _serve_daemon(app, None)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _detach(app: WatchApp) -> int:
|
|
123
|
+
"""Fork the daemon off this launcher process, wait until it is ready or gave up, then let the launcher exit."""
|
|
124
|
+
read_end, write_end = os.pipe()
|
|
125
|
+
pid = os.fork()
|
|
126
|
+
if pid > 0:
|
|
127
|
+
os.close(write_end)
|
|
128
|
+
with os.fdopen(read_end, "rb") as ready:
|
|
129
|
+
# One byte once the daemon listens or knows another daemon holds the lock; EOF if it died first. The
|
|
130
|
+
# exit code tells the watcher that started it whether to wait for it or to give up.
|
|
131
|
+
started = ready.read(1)
|
|
132
|
+
return 0 if started else 1
|
|
133
|
+
os.close(read_end)
|
|
134
|
+
code = 1
|
|
135
|
+
try:
|
|
136
|
+
code = _serve_daemon(app, write_end)
|
|
137
|
+
except BaseException:
|
|
138
|
+
# Without this the traceback would die with os._exit below, and the log would say nothing.
|
|
139
|
+
traceback.print_exc()
|
|
140
|
+
sys.stderr.flush()
|
|
141
|
+
finally:
|
|
142
|
+
logging.shutdown()
|
|
143
|
+
# Never fall back into the launcher's code path, whatever happened above.
|
|
144
|
+
os._exit(code)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _serve_daemon(app: WatchApp, ready_fd: int | None) -> int:
|
|
148
|
+
"""Take the lock or exit, then serve until idle or stopped. `ready_fd` gets one byte when the launcher may go."""
|
|
149
|
+
|
|
150
|
+
def notify(ready: bool = True) -> None:
|
|
151
|
+
nonlocal ready_fd
|
|
152
|
+
if ready_fd is not None:
|
|
153
|
+
with contextlib.suppress(OSError):
|
|
154
|
+
if ready:
|
|
155
|
+
os.write(ready_fd, b"1")
|
|
156
|
+
os.close(ready_fd)
|
|
157
|
+
ready_fd = None
|
|
158
|
+
|
|
159
|
+
files = DaemonFiles.for_app(app)
|
|
160
|
+
ensure_private_dir(files.state_dir)
|
|
161
|
+
logging.basicConfig(
|
|
162
|
+
level=logging.DEBUG if os.environ.get(app.env("DEBUG")) else logging.INFO,
|
|
163
|
+
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
|
164
|
+
stream=sys.stderr,
|
|
165
|
+
)
|
|
166
|
+
lock = FileLock(files.lock)
|
|
167
|
+
try:
|
|
168
|
+
if not lock.try_acquire():
|
|
169
|
+
log.info("another daemon holds %s, exiting", files.lock)
|
|
170
|
+
notify()
|
|
171
|
+
return 0
|
|
172
|
+
lock.write_owner(str(os.getpid()))
|
|
173
|
+
engine = Engine.for_app(app)
|
|
174
|
+
idle_timeout = app.env_float("IDLE_TIMEOUT", DEFAULT_IDLE_TIMEOUT)
|
|
175
|
+
asyncio.run(serve(app, files, engine, idle_timeout, notify))
|
|
176
|
+
except Exception:
|
|
177
|
+
log.exception("the daemon failed")
|
|
178
|
+
return 1
|
|
179
|
+
finally:
|
|
180
|
+
# Still not ready here means the daemon failed before it listened: EOF tells the launcher so.
|
|
181
|
+
notify(ready=False)
|
|
182
|
+
lock.release()
|
|
183
|
+
return 0
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def ping(files: DaemonFiles, timeout: float = 2.0) -> dict[str, Any] | None:
|
|
187
|
+
"""The daemon's hello, or `None` when no daemon answers, a hung one included."""
|
|
188
|
+
return probe(files, timeout)[1]
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def probe(files: DaemonFiles, timeout: float = 2.0) -> tuple[str, dict[str, Any] | None]:
|
|
192
|
+
"""`("ok", hello)`, `("none", None)` when no daemon listens, or `("hung", None)` when one took the request but
|
|
193
|
+
did not answer within `timeout` seconds."""
|
|
194
|
+
try:
|
|
195
|
+
response = request(files, {"op": "hello"}, timeout=timeout)
|
|
196
|
+
except DaemonTimeout:
|
|
197
|
+
return "hung", None
|
|
198
|
+
except (DaemonUnavailable, ValueError):
|
|
199
|
+
return "none", None
|
|
200
|
+
return ("ok", response) if response.get("ok") else ("none", None)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def is_our_daemon(pid: int, daemon_module: str) -> bool:
|
|
204
|
+
"""Whether process `pid` runs `python -m <daemon_module>`. Checked before a hung daemon is killed, so a pid that
|
|
205
|
+
was reused by another program is never hit."""
|
|
206
|
+
if sys.platform.startswith("linux"):
|
|
207
|
+
try:
|
|
208
|
+
argv = Path(f"/proc/{pid}/cmdline").read_bytes().split(b"\0")
|
|
209
|
+
except OSError:
|
|
210
|
+
return False
|
|
211
|
+
module = daemon_module.encode()
|
|
212
|
+
return any(a == b"-m" and b == module for a, b in itertools.pairwise(argv))
|
|
213
|
+
if os.name == "nt":
|
|
214
|
+
return False
|
|
215
|
+
try:
|
|
216
|
+
command = subprocess.run(["ps", "-o", "command=", "-p", str(pid)], capture_output=True, text=True, timeout=5)
|
|
217
|
+
except (OSError, subprocess.SubprocessError):
|
|
218
|
+
return False
|
|
219
|
+
return f"-m {daemon_module}" in command.stdout
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def kill_hung_daemon(app: WatchApp, files: DaemonFiles, timeout: float = 5.0) -> int | None:
|
|
223
|
+
"""Kill the daemon that holds the lock, if its pid is verified: it is in the lock file, the lock is held, and the
|
|
224
|
+
process runs the app's daemon module. Return the pid once the lock is free, or `None` when it could not be done
|
|
225
|
+
(no pid, another program, Windows)."""
|
|
226
|
+
pid = files.owner_pid()
|
|
227
|
+
if pid is None or pid == os.getpid() or not files.lock_held() or not is_our_daemon(pid, app.daemon_module):
|
|
228
|
+
return None
|
|
229
|
+
log.warning("the daemon pid %d does not answer, killing it", pid)
|
|
230
|
+
# The daemon's own log is where someone looks for why it went away, so the killer notes it there too.
|
|
231
|
+
with contextlib.suppress(OSError), open(files.log, "a", encoding="utf-8") as daemon_log:
|
|
232
|
+
stamp = time.strftime("%Y-%m-%d %H:%M:%S")
|
|
233
|
+
daemon_log.write(
|
|
234
|
+
f"{stamp} WARNING cli_watch_kit.daemon: pid {os.getpid()}: the daemon pid {pid} does not answer,"
|
|
235
|
+
" killing it\n"
|
|
236
|
+
)
|
|
237
|
+
with contextlib.suppress(ProcessLookupError):
|
|
238
|
+
os.kill(pid, signal.SIGKILL)
|
|
239
|
+
deadline = time.monotonic() + timeout
|
|
240
|
+
while files.lock_held():
|
|
241
|
+
if time.monotonic() >= deadline:
|
|
242
|
+
return None
|
|
243
|
+
time.sleep(0.05)
|
|
244
|
+
return pid
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _replace_hung(app: WatchApp, files: DaemonFiles) -> None:
|
|
248
|
+
if kill_hung_daemon(app, files) is None:
|
|
249
|
+
pid = files.owner_pid()
|
|
250
|
+
raise DaemonHung(
|
|
251
|
+
f"the daemon (pid {pid if pid is not None else 'unknown'}) holds its lock but does not answer, and it"
|
|
252
|
+
f" could not be verified and killed; stop it with `{app.command} daemon stop` or kill it"
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def needs_drain(daemon: Mapping[str, Any], mine: Mapping[str, Any]) -> bool:
|
|
257
|
+
"""Whether a watcher of identity `mine` should replace a daemon that said `daemon` in its hello: its code differs
|
|
258
|
+
and is newer. An older watcher keeps using a newer daemon, so two installs never take turns replacing each other."""
|
|
259
|
+
if daemon.get("fingerprint") == mine.get("fingerprint"):
|
|
260
|
+
return False
|
|
261
|
+
try:
|
|
262
|
+
return float(mine.get("code_mtime") or 0) > float(daemon.get("code_mtime") or 0)
|
|
263
|
+
except (TypeError, ValueError):
|
|
264
|
+
return False
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _rotate_log(files: DaemonFiles) -> None:
|
|
268
|
+
with contextlib.suppress(OSError):
|
|
269
|
+
if files.log.stat().st_size > MAX_LOG_BYTES:
|
|
270
|
+
os.replace(files.log, files.log.with_suffix(".log.1"))
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def spawn_daemon(app: WatchApp, files: DaemonFiles) -> subprocess.Popen[bytes]:
|
|
274
|
+
"""Start a detached daemon process with its output in the log file."""
|
|
275
|
+
ensure_private_dir(files.state_dir)
|
|
276
|
+
_rotate_log(files)
|
|
277
|
+
fd = os.open(files.log, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
|
|
278
|
+
if os.name != "nt":
|
|
279
|
+
os.fchmod(fd, 0o600)
|
|
280
|
+
with os.fdopen(fd, "ab") as log_file:
|
|
281
|
+
kwargs: dict[str, Any] = {
|
|
282
|
+
"stdin": subprocess.DEVNULL,
|
|
283
|
+
"stdout": log_file,
|
|
284
|
+
"stderr": log_file,
|
|
285
|
+
"cwd": str(files.state_dir),
|
|
286
|
+
"close_fds": True,
|
|
287
|
+
}
|
|
288
|
+
command = [sys.executable, "-m", app.daemon_module]
|
|
289
|
+
if sys.platform == "win32":
|
|
290
|
+
flags = subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
|
|
291
|
+
try:
|
|
292
|
+
# Leave the caller's job object, so killing the caller's job does not kill the daemon. A job that
|
|
293
|
+
# forbids breakaway makes this fail; the daemon then shares the caller's fate.
|
|
294
|
+
return subprocess.Popen(command, creationflags=flags | subprocess.CREATE_BREAKAWAY_FROM_JOB, **kwargs)
|
|
295
|
+
except OSError:
|
|
296
|
+
return subprocess.Popen(command, creationflags=flags, **kwargs)
|
|
297
|
+
# The launcher forks the daemon off and exits once it is ready; see the module docstring.
|
|
298
|
+
return subprocess.Popen(command, start_new_session=True, **kwargs)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _wait_stopped(files: DaemonFiles, deadline: float) -> None:
|
|
302
|
+
while daemon_running(files):
|
|
303
|
+
if time.monotonic() >= deadline:
|
|
304
|
+
raise DaemonUnavailable(f"the old daemon did not stop within the start timeout; see {files.log}")
|
|
305
|
+
time.sleep(0.05)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def ensure_daemon(
|
|
309
|
+
app: WatchApp, files: DaemonFiles | None = None, timeout: float = SPAWN_TIMEOUT, *, drain: bool = True
|
|
310
|
+
) -> dict[str, Any]:
|
|
311
|
+
"""Return the hello response of a running daemon with code at least as new as ours, starting one first if needed.
|
|
312
|
+
|
|
313
|
+
With `drain`, a daemon running older code is asked to stop and is replaced.
|
|
314
|
+
"""
|
|
315
|
+
files = files or DaemonFiles.for_app(app)
|
|
316
|
+
mine = app.identity()
|
|
317
|
+
answer = app.env_float("REQUEST_TIMEOUT", DEFAULT_REQUEST_TIMEOUT)
|
|
318
|
+
hello = ping(files, min(2.0, answer))
|
|
319
|
+
if hello is not None and not (drain and needs_drain(hello, mine)):
|
|
320
|
+
return hello
|
|
321
|
+
files.check_socket_path()
|
|
322
|
+
with FileLock(files.spawn_lock) as spawn_lock:
|
|
323
|
+
spawn_lock.acquire(timeout)
|
|
324
|
+
deadline = time.monotonic() + timeout
|
|
325
|
+
state, hello = probe(files, min(2.0, answer))
|
|
326
|
+
if state == "hung" and probe(files, min(2.0, answer))[0] == "hung":
|
|
327
|
+
# Twice without an answer to a hello, which needs no work: the daemon is stuck, not busy.
|
|
328
|
+
_replace_hung(app, files)
|
|
329
|
+
elif hello is not None and drain and needs_drain(hello, mine):
|
|
330
|
+
try:
|
|
331
|
+
request(files, {"op": "drain", **mine}, timeout=2.0)
|
|
332
|
+
except DaemonTimeout:
|
|
333
|
+
_replace_hung(app, files)
|
|
334
|
+
except (DaemonUnavailable, ValueError):
|
|
335
|
+
pass
|
|
336
|
+
_wait_stopped(files, deadline)
|
|
337
|
+
spawned: subprocess.Popen[bytes] | None = None
|
|
338
|
+
while time.monotonic() < deadline:
|
|
339
|
+
hello = ping(files, timeout=min(1.0, answer))
|
|
340
|
+
if hello is not None:
|
|
341
|
+
return hello
|
|
342
|
+
if spawned is not None and spawned.poll() not in (None, 0) and not daemon_running(files):
|
|
343
|
+
# The daemon died before it listened. Starting it again would fail the same way.
|
|
344
|
+
raise DaemonStartFailed(f"the daemon failed to start: {_last_log_line(files)}; see {files.log}")
|
|
345
|
+
# A daemon that is exiting on idle releases its lock last; spawn again once it is gone.
|
|
346
|
+
if (spawned is None or spawned.poll() is not None) and not daemon_running(files):
|
|
347
|
+
# Nothing holds the daemon lock, so no daemon is alive: any endpoint file left is stale.
|
|
348
|
+
for stale in (files.socket, files.endpoint):
|
|
349
|
+
with contextlib.suppress(OSError):
|
|
350
|
+
stale.unlink()
|
|
351
|
+
spawned = spawn_daemon(app, files)
|
|
352
|
+
time.sleep(0.05)
|
|
353
|
+
raise DaemonUnavailable(f"the daemon did not start within {timeout:.0f} s; see {files.log}")
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _last_log_line(files: DaemonFiles) -> str:
|
|
357
|
+
try:
|
|
358
|
+
lines = files.log.read_text(encoding="utf-8", errors="replace").strip().splitlines()
|
|
359
|
+
except OSError:
|
|
360
|
+
return "no log"
|
|
361
|
+
return lines[-1] if lines else "an empty log"
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def daemon_running(files: DaemonFiles) -> bool:
|
|
365
|
+
"""Whether a daemon process holds the lock, answering or not."""
|
|
366
|
+
return files.lock_held()
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def stop_daemon(files: DaemonFiles, timeout: float = 10.0, *, kill: WatchApp | None = None) -> bool:
|
|
370
|
+
"""Ask the daemon to stop and wait until it released its lock. Return whether one was running.
|
|
371
|
+
|
|
372
|
+
A daemon that does not stop within `timeout` seconds (a hung one) is killed when `kill` names its app and its pid
|
|
373
|
+
is verified (see :func:`kill_hung_daemon`); otherwise this raises :class:`DaemonHung`.
|
|
374
|
+
"""
|
|
375
|
+
if not daemon_running(files):
|
|
376
|
+
return False
|
|
377
|
+
with contextlib.suppress(DaemonUnavailable, DaemonTimeout, ValueError):
|
|
378
|
+
request(files, {"op": "stop"}, timeout=2.0)
|
|
379
|
+
deadline = time.monotonic() + timeout
|
|
380
|
+
while time.monotonic() < deadline:
|
|
381
|
+
if not daemon_running(files):
|
|
382
|
+
return True
|
|
383
|
+
time.sleep(0.05)
|
|
384
|
+
if kill is not None and kill_hung_daemon(kill, files) is not None:
|
|
385
|
+
return True
|
|
386
|
+
pid = files.owner_pid()
|
|
387
|
+
raise DaemonHung(f"the daemon (pid {pid if pid is not None else 'unknown'}) did not stop within {timeout:.0f} s")
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Which resources poll next, and which of them poll together.
|
|
2
|
+
|
|
3
|
+
Resources wait in due-time order. Among those that are due, urgent ones go first (a new subscription, a filter that
|
|
4
|
+
grew), then the one served longest ago. That round-robin keeps one busy resource from starving a quiet one when the
|
|
5
|
+
budget is short. The scheduler holds no clock and does no I/O; the engine passes the time in.
|
|
6
|
+
|
|
7
|
+
Resources of one batch group (the same account and batch key) that are due go into one batch, together with those of
|
|
8
|
+
the group that become due within the coalescing window. A resource pulled in early is then rescheduled from the
|
|
9
|
+
batch's time, so the schedules of one group converge and stay aligned.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from collections.abc import Callable, Container, Hashable
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class _Entry:
|
|
20
|
+
due: float
|
|
21
|
+
urgent: bool
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Scheduler:
|
|
25
|
+
def __init__(self) -> None:
|
|
26
|
+
self._entries: dict[str, _Entry] = {}
|
|
27
|
+
self._last_served: dict[str, float] = {}
|
|
28
|
+
|
|
29
|
+
def schedule(self, key: str, due: float, *, urgent: bool = False) -> None:
|
|
30
|
+
self._entries[key] = _Entry(due, urgent)
|
|
31
|
+
|
|
32
|
+
def remove(self, key: str) -> None:
|
|
33
|
+
self._entries.pop(key, None)
|
|
34
|
+
|
|
35
|
+
def forget(self, key: str) -> None:
|
|
36
|
+
self._entries.pop(key, None)
|
|
37
|
+
self._last_served.pop(key, None)
|
|
38
|
+
|
|
39
|
+
def __contains__(self, key: str) -> bool:
|
|
40
|
+
return key in self._entries
|
|
41
|
+
|
|
42
|
+
def due_at(self, key: str) -> float | None:
|
|
43
|
+
entry = self._entries.get(key)
|
|
44
|
+
return entry.due if entry else None
|
|
45
|
+
|
|
46
|
+
def next_due(self, exclude: Container[str] = ()) -> float | None:
|
|
47
|
+
"""The earliest due time, leaving out `exclude` (keys that are due but wait for something else)."""
|
|
48
|
+
return min((e.due for k, e in self._entries.items() if k not in exclude), default=None)
|
|
49
|
+
|
|
50
|
+
def ready(self, now: float) -> list[str]:
|
|
51
|
+
"""Due keys in serving order: urgent first, then least recently served, then earliest due."""
|
|
52
|
+
due = [(key, entry) for key, entry in self._entries.items() if entry.due <= now]
|
|
53
|
+
due.sort(key=lambda item: (not item[1].urgent, self._last_served.get(item[0], float("-inf")), item[1].due))
|
|
54
|
+
return [key for key, _ in due]
|
|
55
|
+
|
|
56
|
+
def ready_batches(
|
|
57
|
+
self, now: float, group_of: Callable[[str], Hashable | None], window: float = 0.0
|
|
58
|
+
) -> list[list[str]]:
|
|
59
|
+
"""Due keys grouped into batches, in serving order of each batch's first key.
|
|
60
|
+
|
|
61
|
+
`group_of` returns a key's batch group, or `None` for a key that polls alone. Keys of a group that is due also
|
|
62
|
+
join when they are due within `window` seconds, in due order.
|
|
63
|
+
"""
|
|
64
|
+
batches: list[list[str]] = []
|
|
65
|
+
by_group: dict[Hashable, list[str]] = {}
|
|
66
|
+
for key in self.ready(now):
|
|
67
|
+
group = group_of(key)
|
|
68
|
+
if group is None:
|
|
69
|
+
batches.append([key])
|
|
70
|
+
elif group in by_group:
|
|
71
|
+
by_group[group].append(key)
|
|
72
|
+
else:
|
|
73
|
+
by_group[group] = [key]
|
|
74
|
+
batches.append(by_group[group])
|
|
75
|
+
if window > 0 and by_group:
|
|
76
|
+
early = sorted((entry.due, key) for key, entry in self._entries.items() if now < entry.due <= now + window)
|
|
77
|
+
for _, key in early:
|
|
78
|
+
group = group_of(key)
|
|
79
|
+
if group is not None and group in by_group:
|
|
80
|
+
by_group[group].append(key)
|
|
81
|
+
return batches
|
|
82
|
+
|
|
83
|
+
def mark_served(self, key: str, now: float) -> None:
|
|
84
|
+
self._entries.pop(key, None)
|
|
85
|
+
self._last_served[key] = now
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Dispatch of daemon requests to the engine and to the source's extra ops.
|
|
2
|
+
|
|
3
|
+
Built-in ops: `hello` (versions and code stamp), `subscribe`, `poll`, `unsubscribe`, `status`, `stop`, and `drain`,
|
|
4
|
+
which a watcher running newer code sends to stop an older daemon so that a current one replaces it.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import asyncio
|
|
10
|
+
import logging
|
|
11
|
+
import os
|
|
12
|
+
from collections.abc import Mapping
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from cli_watch_kit.daemon.engine import Engine, EngineError
|
|
16
|
+
from cli_watch_kit.source import OpContext, SourceError, Target
|
|
17
|
+
|
|
18
|
+
log = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
BUILTIN_OPS = frozenset({"hello", "subscribe", "poll", "unsubscribe", "status", "stop", "drain"})
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class RequestError(ValueError):
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _str(request: Mapping[str, Any], name: str) -> str:
|
|
28
|
+
value = request.get(name)
|
|
29
|
+
if not isinstance(value, str) or not value:
|
|
30
|
+
raise RequestError(f"{name} must be a non-empty string")
|
|
31
|
+
return value
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _object(request: Mapping[str, Any], name: str) -> dict[str, Any]:
|
|
35
|
+
value = request.get(name) or {}
|
|
36
|
+
if not isinstance(value, dict):
|
|
37
|
+
raise RequestError(f"{name} must be an object")
|
|
38
|
+
return value
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _params(request: Mapping[str, Any]) -> dict[str, Any]:
|
|
42
|
+
return _object(request, "params")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
async def handle(
|
|
46
|
+
engine: Engine, request: dict[str, Any], stop: asyncio.Event, identity: Mapping[str, Any]
|
|
47
|
+
) -> dict[str, Any]:
|
|
48
|
+
op = request.get("op")
|
|
49
|
+
try:
|
|
50
|
+
if op == "hello":
|
|
51
|
+
return {"ok": True, "pid": os.getpid(), **identity}
|
|
52
|
+
if op == "subscribe":
|
|
53
|
+
options = request.get("options") or {}
|
|
54
|
+
if not isinstance(options, dict):
|
|
55
|
+
raise RequestError("options must be an object")
|
|
56
|
+
raw_events = options.get("events")
|
|
57
|
+
if not isinstance(raw_events, list) or not all(isinstance(e, str) for e in raw_events):
|
|
58
|
+
raise RequestError("options.events must be a list of event kinds")
|
|
59
|
+
result = await engine.subscribe(
|
|
60
|
+
_str(request, "client_id"),
|
|
61
|
+
Target.from_wire(request.get("target")),
|
|
62
|
+
frozenset(raw_events),
|
|
63
|
+
options.get("baseline"),
|
|
64
|
+
_params(request),
|
|
65
|
+
_object(request, "source_options"),
|
|
66
|
+
)
|
|
67
|
+
return {"ok": True, **result}
|
|
68
|
+
if op == "poll":
|
|
69
|
+
cursors = request.get("cursors") or {}
|
|
70
|
+
if not isinstance(cursors, dict) or not all(isinstance(v, int) for v in cursors.values()):
|
|
71
|
+
raise RequestError("cursors must map subscription ids to integers")
|
|
72
|
+
acks = request.get("acks")
|
|
73
|
+
if acks is not None and (not isinstance(acks, dict) or not all(isinstance(v, int) for v in acks.values())):
|
|
74
|
+
raise RequestError("acks must map subscription ids to integers")
|
|
75
|
+
return {"ok": True, **engine.poll(_str(request, "client_id"), cursors, acks)}
|
|
76
|
+
if op == "unsubscribe":
|
|
77
|
+
removed = engine.unsubscribe(_str(request, "client_id"), _str(request, "subscription_id"))
|
|
78
|
+
return {"ok": True, "removed": removed}
|
|
79
|
+
if op == "status":
|
|
80
|
+
now = engine.clock()
|
|
81
|
+
return {
|
|
82
|
+
"ok": True,
|
|
83
|
+
"pid": os.getpid(),
|
|
84
|
+
**identity,
|
|
85
|
+
"uptime_s": round(now - engine.started_at, 1),
|
|
86
|
+
"idle_s": round(engine.idle_for(now), 1),
|
|
87
|
+
**engine.status(),
|
|
88
|
+
}
|
|
89
|
+
if op == "stop":
|
|
90
|
+
log.info("stop requested")
|
|
91
|
+
stop.set()
|
|
92
|
+
return {"ok": True, "pid": os.getpid()}
|
|
93
|
+
if op == "drain":
|
|
94
|
+
log.info(
|
|
95
|
+
"draining for a watcher with newer code (%s %s, cli-watch-kit %s, code %s)",
|
|
96
|
+
request.get("app_version"),
|
|
97
|
+
request.get("app"),
|
|
98
|
+
request.get("lib_version"),
|
|
99
|
+
request.get("fingerprint"),
|
|
100
|
+
)
|
|
101
|
+
stop.set()
|
|
102
|
+
return {"ok": True, "pid": os.getpid()}
|
|
103
|
+
handler = engine.source.ops.get(op) if isinstance(op, str) and op not in BUILTIN_OPS else None
|
|
104
|
+
if handler is None:
|
|
105
|
+
return {"ok": False, "error": f"unknown op {op!r}"}
|
|
106
|
+
body = {k: v for k, v in request.items() if k not in ("op", "params")}
|
|
107
|
+
return {"ok": True, **await handler(OpContext(engine, _params(request)), body)}
|
|
108
|
+
except (RequestError, EngineError, ValueError, TimeoutError) as exc:
|
|
109
|
+
return {"ok": False, "error": str(exc)}
|
|
110
|
+
except SourceError as exc:
|
|
111
|
+
return {"ok": False, "error": str(exc) or type(exc).__name__, "reason": exc.reason}
|
|
112
|
+
except Exception as exc:
|
|
113
|
+
# A bug in the daemon or in the source's code: answer, so the watcher does not take it for a dead daemon. The
|
|
114
|
+
# reason `error` lets a resubscribing watcher try again: the bug may sit in a path that fails only sometimes.
|
|
115
|
+
log.exception("the %r request failed", op)
|
|
116
|
+
return {"ok": False, "error": f"the daemon failed: {type(exc).__name__}: {exc}", "reason": "error"}
|