cli-watch-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cli_watch_kit/__init__.py +27 -0
- cli_watch_kit/app.py +175 -0
- cli_watch_kit/atomic.py +37 -0
- cli_watch_kit/budget.py +209 -0
- cli_watch_kit/changelog.py +87 -0
- cli_watch_kit/claude.py +195 -0
- cli_watch_kit/cli.py +410 -0
- cli_watch_kit/client.py +625 -0
- cli_watch_kit/daemon/__init__.py +25 -0
- cli_watch_kit/daemon/cache.py +48 -0
- cli_watch_kit/daemon/engine.py +874 -0
- cli_watch_kit/daemon/lifecycle.py +387 -0
- cli_watch_kit/daemon/scheduler.py +85 -0
- cli_watch_kit/daemon/server.py +116 -0
- cli_watch_kit/daemon/transport.py +324 -0
- cli_watch_kit/durations.py +28 -0
- cli_watch_kit/events.py +182 -0
- cli_watch_kit/locks.py +93 -0
- cli_watch_kit/paths.py +114 -0
- cli_watch_kit/py.typed +0 -0
- cli_watch_kit/sinks.py +106 -0
- cli_watch_kit/source.py +291 -0
- cli_watch_kit/timers.py +67 -0
- cli_watch_kit-0.1.0.dist-info/METADATA +207 -0
- cli_watch_kit-0.1.0.dist-info/RECORD +27 -0
- cli_watch_kit-0.1.0.dist-info/WHEEL +4 -0
- cli_watch_kit-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Build a `watch` command for a CLI: poll a source, report changes as JSON events, share one daemon between watchers.
|
|
2
|
+
|
|
3
|
+
The public API, by import path:
|
|
4
|
+
|
|
5
|
+
- :mod:`cli_watch_kit.source`: :class:`Source`, :class:`Target`, :class:`PollItem`, :class:`PollOutcome`,
|
|
6
|
+
:class:`PollContext`, the errors :class:`NotFound`, :class:`Unauthorized`, :class:`RateLimited`, :class:`Transient`.
|
|
7
|
+
- :mod:`cli_watch_kit.app`: :class:`WatchApp`.
|
|
8
|
+
- :mod:`cli_watch_kit.events`: :class:`Event`, :class:`RawEvent`, :class:`KindCatalog`, the lifecycle kinds.
|
|
9
|
+
- :mod:`cli_watch_kit.budget`: :class:`DimensionSpec`, :class:`Budget`.
|
|
10
|
+
- :mod:`cli_watch_kit.cli`: the click helpers :func:`watch_options`, :func:`run_watch`, :func:`daemon_group`.
|
|
11
|
+
- :mod:`cli_watch_kit.daemon`: :func:`main`, the body of an app's daemon module.
|
|
12
|
+
- :mod:`cli_watch_kit.client`: :class:`Watcher` and the backends, for a CLI that does not use click.
|
|
13
|
+
- :mod:`cli_watch_kit.claude`: the Claude Code stream, from the `claude-code` extra.
|
|
14
|
+
|
|
15
|
+
Importing this package imports nothing but the standard library.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
21
|
+
|
|
22
|
+
try:
|
|
23
|
+
__version__ = version("cli-watch-kit")
|
|
24
|
+
except PackageNotFoundError: # pragma: no cover - only when run from a source tree that was never installed
|
|
25
|
+
__version__ = "0.0.0"
|
|
26
|
+
|
|
27
|
+
__all__ = ["__version__"]
|
cli_watch_kit/app.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""`WatchApp`: everything the library needs to know about one CLI's watch command."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import importlib.util
|
|
7
|
+
import os
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
from dataclasses import dataclass, field, replace
|
|
10
|
+
from functools import cached_property
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from cli_watch_kit import __version__ as LIB_VERSION
|
|
15
|
+
from cli_watch_kit.paths import AppPaths
|
|
16
|
+
from cli_watch_kit.source import Source
|
|
17
|
+
|
|
18
|
+
LIB_PROTOCOL = 1
|
|
19
|
+
"""The wire protocol between the library's watchers and daemons. Bumped when a daemon of the previous version could
|
|
20
|
+
not serve a watcher of this one. It is part of the socket name, so daemons of two protocols never meet."""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class WatchApp:
|
|
25
|
+
"""One CLI's watch command.
|
|
26
|
+
|
|
27
|
+
- `name`: the app's name in paths, the daemon log and the Claude Code sender (`linear-watch`).
|
|
28
|
+
- `source`: the :class:`Source`, or a function that makes it, so a CLI does not build it before it is needed.
|
|
29
|
+
- `daemon_module`: the module that `python -m` runs as the daemon; it calls :func:`cli_watch_kit.daemon.main`.
|
|
30
|
+
- `app_protocol`: the app's own wire version, for what the source sends (snapshot fields, event kinds). Bump it
|
|
31
|
+
when an older daemon could not serve a newer watcher; it is part of the socket name.
|
|
32
|
+
- `version`: the app's version, shown in the status and compared in the handshake.
|
|
33
|
+
- `status_prefix`: the start of every status line on stderr (`linear watch: `).
|
|
34
|
+
- `envelope_tag`: the tag of a Claude Code message (`linear-watcher`).
|
|
35
|
+
- `command`: how a user runs the watch command, for hints (`linear watch`). Default: `name`.
|
|
36
|
+
- `env_prefix`: the prefix of the override variables. Default: `name` in upper case with `_`, plus `_`.
|
|
37
|
+
- `files_dir_name`: the name of the dir the watchers' files go under, in the session scratchpad or the temp
|
|
38
|
+
dir (`linear-cli`). Default: `name`.
|
|
39
|
+
- `client_poll_ms`, `lease_timeout`: how often watchers poll the engine and how long a subscription lives
|
|
40
|
+
without a poll. Default: 5000 ms and 20 s. `<PREFIX>CLIENT_POLL_MS` and `<PREFIX>LEASE_TIMEOUT` override both,
|
|
41
|
+
for the daemon and for `--no-daemon` alike.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
name: str
|
|
45
|
+
source: Source[Any] | Callable[[], Source[Any]]
|
|
46
|
+
daemon_module: str
|
|
47
|
+
app_protocol: int
|
|
48
|
+
version: str
|
|
49
|
+
status_prefix: str
|
|
50
|
+
envelope_tag: str
|
|
51
|
+
command: str = ""
|
|
52
|
+
env_prefix: str = ""
|
|
53
|
+
files_dir_name: str = ""
|
|
54
|
+
client_poll_ms: int | None = None
|
|
55
|
+
lease_timeout: float | None = None
|
|
56
|
+
_made: dict[str, Source[Any]] = field(default_factory=dict, init=False, compare=False, repr=False)
|
|
57
|
+
|
|
58
|
+
def __post_init__(self) -> None:
|
|
59
|
+
if not self.command:
|
|
60
|
+
object.__setattr__(self, "command", self.name)
|
|
61
|
+
if not self.env_prefix:
|
|
62
|
+
object.__setattr__(self, "env_prefix", self.name.upper().replace("-", "_") + "_")
|
|
63
|
+
|
|
64
|
+
def make_source(self) -> Source[Any]:
|
|
65
|
+
"""The source, built once."""
|
|
66
|
+
if isinstance(self.source, Source):
|
|
67
|
+
return self.source
|
|
68
|
+
made = self._made.get("source")
|
|
69
|
+
if made is None:
|
|
70
|
+
made = self.source()
|
|
71
|
+
self._made["source"] = made
|
|
72
|
+
return made
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def paths(self) -> AppPaths:
|
|
76
|
+
return AppPaths(self.name, self.env_prefix, self.files_dir_name or self.name)
|
|
77
|
+
|
|
78
|
+
def env(self, name: str) -> str:
|
|
79
|
+
"""The name of one of the app's environment variables."""
|
|
80
|
+
return f"{self.env_prefix}{name}"
|
|
81
|
+
|
|
82
|
+
def env_float(self, name: str, default: float) -> float:
|
|
83
|
+
"""The app's environment variable `name` as a number, or `default` when it is unset or not a number."""
|
|
84
|
+
try:
|
|
85
|
+
return float(os.environ.get(self.env(name), default))
|
|
86
|
+
except ValueError:
|
|
87
|
+
return default
|
|
88
|
+
|
|
89
|
+
def engine_settings(self) -> dict[str, Any]:
|
|
90
|
+
"""What an engine of this app runs with, the same in the daemon and in a `--no-daemon` watcher."""
|
|
91
|
+
from cli_watch_kit.daemon.engine import DEFAULT_CLIENT_POLL_MS, DEFAULT_LEASE_TIMEOUT
|
|
92
|
+
|
|
93
|
+
poll_ms = self.client_poll_ms if self.client_poll_ms is not None else DEFAULT_CLIENT_POLL_MS
|
|
94
|
+
lease = self.lease_timeout if self.lease_timeout is not None else DEFAULT_LEASE_TIMEOUT
|
|
95
|
+
return {
|
|
96
|
+
"client_poll_ms": int(self.env_float("CLIENT_POLL_MS", poll_ms)),
|
|
97
|
+
"lease_timeout": self.env_float("LEASE_TIMEOUT", lease),
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
def with_overrides(
|
|
101
|
+
self,
|
|
102
|
+
*,
|
|
103
|
+
source: Source[Any] | Callable[[], Source[Any]] | None = None,
|
|
104
|
+
client_poll_ms: int | None = None,
|
|
105
|
+
lease_timeout: float | None = None,
|
|
106
|
+
) -> WatchApp:
|
|
107
|
+
"""A copy of the app with another source or engine settings, for tests and for an in-process run:
|
|
108
|
+
`run_watch(APP.with_overrides(source=FakeSource()), target, WatchOptions(no_daemon=True))`."""
|
|
109
|
+
changes: dict[str, Any] = {}
|
|
110
|
+
if source is not None:
|
|
111
|
+
changes["source"] = source
|
|
112
|
+
if client_poll_ms is not None:
|
|
113
|
+
changes["client_poll_ms"] = client_poll_ms
|
|
114
|
+
if lease_timeout is not None:
|
|
115
|
+
changes["lease_timeout"] = lease_timeout
|
|
116
|
+
return replace(self, **changes)
|
|
117
|
+
|
|
118
|
+
@property
|
|
119
|
+
def lib_version(self) -> str:
|
|
120
|
+
return LIB_VERSION
|
|
121
|
+
|
|
122
|
+
@cached_property
|
|
123
|
+
def code(self) -> CodeStamp:
|
|
124
|
+
"""What code this process runs: the library's and the app's, for the version handshake."""
|
|
125
|
+
top = self.daemon_module.split(".", 1)[0]
|
|
126
|
+
return CodeStamp.of(["cli_watch_kit", top])
|
|
127
|
+
|
|
128
|
+
def identity(self) -> dict[str, Any]:
|
|
129
|
+
"""The versions and the code stamp, as the daemon's hello reports them and a watcher compares them."""
|
|
130
|
+
return {
|
|
131
|
+
"app": self.name,
|
|
132
|
+
"lib_protocol": LIB_PROTOCOL,
|
|
133
|
+
"app_protocol": self.app_protocol,
|
|
134
|
+
"lib_version": LIB_VERSION,
|
|
135
|
+
"app_version": self.version,
|
|
136
|
+
"fingerprint": self.code.fingerprint,
|
|
137
|
+
"code_mtime": self.code.mtime,
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@dataclass(frozen=True)
|
|
142
|
+
class CodeStamp:
|
|
143
|
+
"""A fingerprint of the installed code of some packages and the newest modification time among their files.
|
|
144
|
+
|
|
145
|
+
An editable install changes code without changing any version, so the version alone cannot tell a watcher that
|
|
146
|
+
the daemon runs older code. The fingerprint can; the modification time says which side is newer.
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
fingerprint: str
|
|
150
|
+
mtime: float
|
|
151
|
+
|
|
152
|
+
@classmethod
|
|
153
|
+
def of(cls, packages: list[str]) -> CodeStamp:
|
|
154
|
+
digest = hashlib.sha256()
|
|
155
|
+
newest = 0.0
|
|
156
|
+
for package in sorted(set(packages)):
|
|
157
|
+
spec = importlib.util.find_spec(package)
|
|
158
|
+
locations = list(spec.submodule_search_locations or []) if spec is not None else []
|
|
159
|
+
if spec is not None and not locations and spec.origin:
|
|
160
|
+
locations = [str(Path(spec.origin).parent)]
|
|
161
|
+
for location in locations:
|
|
162
|
+
for root, dirs, files in os.walk(location):
|
|
163
|
+
dirs[:] = sorted(d for d in dirs if d != "__pycache__")
|
|
164
|
+
for name in sorted(files):
|
|
165
|
+
if not name.endswith(".py"):
|
|
166
|
+
continue
|
|
167
|
+
path = Path(root, name)
|
|
168
|
+
try:
|
|
169
|
+
stat = path.stat()
|
|
170
|
+
except OSError:
|
|
171
|
+
continue
|
|
172
|
+
rel = path.relative_to(location).as_posix()
|
|
173
|
+
digest.update(f"{package}/{rel}:{stat.st_size}:{stat.st_mtime_ns}\n".encode())
|
|
174
|
+
newest = max(newest, stat.st_mtime)
|
|
175
|
+
return cls(digest.hexdigest()[:16], newest)
|
cli_watch_kit/atomic.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Atomic file writes.
|
|
2
|
+
|
|
3
|
+
Each write goes to a temp file with a unique name in the target's dir, then replaces the target with `os.replace`.
|
|
4
|
+
A reader therefore sees the old file or the new one, never a half-written one. The unique name matters: with a fixed
|
|
5
|
+
`<name>.tmp`, two processes writing the same file write into one temp file, and one of them then fails to rename it,
|
|
6
|
+
or renames the other's half-written bytes into place.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import contextlib
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import tempfile
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def atomic_write_bytes(path: Path, data: bytes) -> None:
|
|
20
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
21
|
+
fd, tmp = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
|
|
22
|
+
try:
|
|
23
|
+
with os.fdopen(fd, "wb") as fh:
|
|
24
|
+
fh.write(data)
|
|
25
|
+
os.replace(tmp, path)
|
|
26
|
+
except BaseException:
|
|
27
|
+
with contextlib.suppress(OSError):
|
|
28
|
+
os.unlink(tmp)
|
|
29
|
+
raise
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def atomic_write_text(path: Path, text: str) -> None:
|
|
33
|
+
atomic_write_bytes(path, text.encode("utf-8"))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def atomic_write_json(path: Path, value: Any) -> None:
|
|
37
|
+
atomic_write_text(path, json.dumps(value, ensure_ascii=False, indent=2) + "\n")
|
cli_watch_kit/budget.py
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""The rate budget of one account, with one token bucket per dimension (requests, complexity points, ...).
|
|
2
|
+
|
|
3
|
+
An API usually reports what is left of a limit and when the limit resets. A source passes those readings to
|
|
4
|
+
:meth:`Budget.observe`; the budget then paces on `remaining / (reset - now)` and takes `share` of it, leaving the rest
|
|
5
|
+
to the user's other tools. Before a reading, or for an API that sends none, each dimension has a fixed rate.
|
|
6
|
+
|
|
7
|
+
A poll may run only when every dimension has the tokens for its cost. A 429 blocks the whole account for its
|
|
8
|
+
`Retry-After`, and repeated 429s back off exponentially. When the polls the watchers want would exceed the rate of
|
|
9
|
+
any dimension, :meth:`Budget.stretch_factor` says how much every interval of the account must grow.
|
|
10
|
+
|
|
11
|
+
Time is passed in by the caller (wall-clock seconds, because reset times are usually Unix timestamps), which keeps the
|
|
12
|
+
class deterministic under test.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from collections.abc import Mapping
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
DEFAULT_SHARE = 0.5
|
|
22
|
+
MIN_BACKOFF = 5.0
|
|
23
|
+
MAX_BACKOFF = 300.0
|
|
24
|
+
MIN_RATE = 0.0001
|
|
25
|
+
"""Units per second a dimension keeps even with nothing left, so a wait time stays finite."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True, slots=True)
|
|
29
|
+
class DimensionSpec:
|
|
30
|
+
"""How one dimension is paced before the API reports anything about it.
|
|
31
|
+
|
|
32
|
+
`per_minute` is the fixed rate in units per minute, `burst` the bucket size. `share` is the part of a reported
|
|
33
|
+
allowance this tool may use.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
per_minute: float = 60.0
|
|
37
|
+
burst: float = 10.0
|
|
38
|
+
share: float = DEFAULT_SHARE
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
REQUESTS = "requests"
|
|
42
|
+
DEFAULT_DIMENSIONS: Mapping[str, DimensionSpec] = {REQUESTS: DimensionSpec()}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass
|
|
46
|
+
class Dimension:
|
|
47
|
+
name: str
|
|
48
|
+
spec: DimensionSpec
|
|
49
|
+
tokens: float
|
|
50
|
+
last_refill: float | None = None
|
|
51
|
+
remaining: float | None = None
|
|
52
|
+
reset_at: float | None = None
|
|
53
|
+
ceiling_per_second: float | None = None
|
|
54
|
+
"""An upper bound on the rate the API states separately, such as GitLab's per-minute limit."""
|
|
55
|
+
spent: float = 0.0
|
|
56
|
+
headers_seen: bool = False
|
|
57
|
+
|
|
58
|
+
@classmethod
|
|
59
|
+
def from_spec(cls, name: str, spec: DimensionSpec) -> Dimension:
|
|
60
|
+
return cls(name, spec, spec.burst)
|
|
61
|
+
|
|
62
|
+
def rate(self, now: float) -> float:
|
|
63
|
+
"""Units per second this tool may spend right now."""
|
|
64
|
+
share = self.spec.share
|
|
65
|
+
if self.remaining is not None and self.reset_at is not None and self.reset_at > now:
|
|
66
|
+
rate = self.remaining * share / max(1.0, self.reset_at - now)
|
|
67
|
+
if self.ceiling_per_second:
|
|
68
|
+
rate = min(rate, self.ceiling_per_second * share)
|
|
69
|
+
return max(rate, MIN_RATE)
|
|
70
|
+
if self.ceiling_per_second:
|
|
71
|
+
return self.ceiling_per_second * share
|
|
72
|
+
return max(self.spec.per_minute / 60.0, MIN_RATE)
|
|
73
|
+
|
|
74
|
+
def refill(self, now: float) -> None:
|
|
75
|
+
if self.last_refill is None:
|
|
76
|
+
self.last_refill = now
|
|
77
|
+
return
|
|
78
|
+
elapsed = max(0.0, now - self.last_refill)
|
|
79
|
+
self.tokens = min(self.spec.burst, self.tokens + elapsed * self.rate(now))
|
|
80
|
+
self.last_refill = now
|
|
81
|
+
|
|
82
|
+
def missing(self, cost: float) -> float:
|
|
83
|
+
"""Tokens short of `cost`. A cost above the bucket size needs a full bucket, else it could never run."""
|
|
84
|
+
return max(0.0, min(cost, self.spec.burst) - self.tokens)
|
|
85
|
+
|
|
86
|
+
def describe(self, now: float) -> dict[str, Any]:
|
|
87
|
+
return {
|
|
88
|
+
"rate_per_minute": round(self.rate(now) * 60, 2),
|
|
89
|
+
"tokens": round(self.tokens, 2),
|
|
90
|
+
"burst": self.spec.burst,
|
|
91
|
+
"remaining": self.remaining,
|
|
92
|
+
"reset_in_s": round(self.reset_at - now) if self.reset_at else None,
|
|
93
|
+
"spent": round(self.spent, 2),
|
|
94
|
+
"headers_seen": self.headers_seen,
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass
|
|
99
|
+
class Budget:
|
|
100
|
+
"""The budget of one account. Costs and readings for a dimension not declared up front use `DimensionSpec()`."""
|
|
101
|
+
|
|
102
|
+
specs: Mapping[str, DimensionSpec] = field(default_factory=lambda: dict(DEFAULT_DIMENSIONS))
|
|
103
|
+
dimensions: dict[str, Dimension] = field(default_factory=dict)
|
|
104
|
+
blocked_until: float = 0.0
|
|
105
|
+
backoff: float = MIN_BACKOFF
|
|
106
|
+
rate_limited_count: int = 0
|
|
107
|
+
|
|
108
|
+
def __post_init__(self) -> None:
|
|
109
|
+
for name, spec in self.specs.items():
|
|
110
|
+
self.dimensions.setdefault(name, Dimension.from_spec(name, spec))
|
|
111
|
+
|
|
112
|
+
def dimension(self, name: str) -> Dimension:
|
|
113
|
+
dim = self.dimensions.get(name)
|
|
114
|
+
if dim is None:
|
|
115
|
+
dim = Dimension.from_spec(name, self.specs.get(name, DimensionSpec()))
|
|
116
|
+
self.dimensions[name] = dim
|
|
117
|
+
return dim
|
|
118
|
+
|
|
119
|
+
def _refill(self, now: float) -> None:
|
|
120
|
+
for dim in self.dimensions.values():
|
|
121
|
+
dim.refill(now)
|
|
122
|
+
|
|
123
|
+
def can_spend(self, cost: Mapping[str, float], now: float) -> bool:
|
|
124
|
+
self._refill(now)
|
|
125
|
+
if now < self.blocked_until:
|
|
126
|
+
return False
|
|
127
|
+
return all(self.dimension(name).missing(amount) <= 0 for name, amount in cost.items())
|
|
128
|
+
|
|
129
|
+
def wait_time(self, cost: Mapping[str, float], now: float) -> float:
|
|
130
|
+
"""Seconds until `can_spend(cost)` could become true: the slowest dimension, or the 429 block."""
|
|
131
|
+
self._refill(now)
|
|
132
|
+
wait = max(0.0, self.blocked_until - now)
|
|
133
|
+
for name, amount in cost.items():
|
|
134
|
+
dim = self.dimension(name)
|
|
135
|
+
wait = max(wait, dim.missing(amount) / dim.rate(now))
|
|
136
|
+
return wait
|
|
137
|
+
|
|
138
|
+
def charge(self, cost: Mapping[str, float], now: float) -> None:
|
|
139
|
+
"""Take `cost` from the buckets. A negative amount gives back an estimate that was too high."""
|
|
140
|
+
self._refill(now)
|
|
141
|
+
for name, amount in cost.items():
|
|
142
|
+
dim = self.dimension(name)
|
|
143
|
+
dim.tokens = min(dim.spec.burst, dim.tokens - amount)
|
|
144
|
+
dim.spent += amount
|
|
145
|
+
|
|
146
|
+
def observe(
|
|
147
|
+
self,
|
|
148
|
+
name: str,
|
|
149
|
+
now: float,
|
|
150
|
+
*,
|
|
151
|
+
remaining: float | None = None,
|
|
152
|
+
reset_at: float | None = None,
|
|
153
|
+
ceiling_per_second: float | None = None,
|
|
154
|
+
) -> None:
|
|
155
|
+
"""Take in what the API reported about one dimension. `reset_at` is in the same clock as `now`."""
|
|
156
|
+
if remaining is None and reset_at is None and ceiling_per_second is None:
|
|
157
|
+
return
|
|
158
|
+
dim = self.dimension(name)
|
|
159
|
+
dim.headers_seen = True
|
|
160
|
+
if remaining is not None:
|
|
161
|
+
dim.remaining = remaining
|
|
162
|
+
if reset_at is not None:
|
|
163
|
+
dim.reset_at = reset_at
|
|
164
|
+
if ceiling_per_second is not None:
|
|
165
|
+
dim.ceiling_per_second = ceiling_per_second
|
|
166
|
+
|
|
167
|
+
def on_success(self) -> None:
|
|
168
|
+
self.backoff = MIN_BACKOFF
|
|
169
|
+
|
|
170
|
+
def on_rate_limited(self, retry_after: float | None, now: float) -> None:
|
|
171
|
+
"""Block the whole account: honor Retry-After, and back off exponentially on repeated 429s."""
|
|
172
|
+
wait = max(retry_after or 0.0, self.backoff)
|
|
173
|
+
self.blocked_until = max(self.blocked_until, now + wait)
|
|
174
|
+
self.backoff = min(self.backoff * 2, MAX_BACKOFF)
|
|
175
|
+
self.rate_limited_count += 1
|
|
176
|
+
for dim in self.dimensions.values():
|
|
177
|
+
dim.tokens = min(dim.tokens, 0.0)
|
|
178
|
+
|
|
179
|
+
def stretch_factor(self, demand_per_second: Mapping[str, float], now: float) -> float:
|
|
180
|
+
"""How much every poll interval of the account must grow so the demand of each dimension fits its rate."""
|
|
181
|
+
return self.stretch(demand_per_second, now)[0]
|
|
182
|
+
|
|
183
|
+
def stretch(self, demand_per_second: Mapping[str, float], now: float) -> tuple[float, str | None]:
|
|
184
|
+
""":meth:`stretch_factor`, and the dimension that drives it (`None` when nothing needs stretching)."""
|
|
185
|
+
factor, driver = 1.0, None
|
|
186
|
+
for name, demand in demand_per_second.items():
|
|
187
|
+
rate = self.dimension(name).rate(now)
|
|
188
|
+
if demand > rate and demand / rate > factor:
|
|
189
|
+
factor, driver = demand / rate, name
|
|
190
|
+
return factor, driver
|
|
191
|
+
|
|
192
|
+
def reset_in(self, now: float, dimension: str | None = None) -> float | None:
|
|
193
|
+
"""Seconds until the reported window of `dimension` resets (of any dimension, the latest, when `None`), or
|
|
194
|
+
`None` when that is not known."""
|
|
195
|
+
dims = [self.dimension(dimension)] if dimension is not None else list(self.dimensions.values())
|
|
196
|
+
resets = [d.reset_at - now for d in dims if d.reset_at is not None and d.reset_at > now]
|
|
197
|
+
return max(resets) if resets else None
|
|
198
|
+
|
|
199
|
+
def windows(self) -> tuple[tuple[str, float | None], ...]:
|
|
200
|
+
"""The reported reset times; a change means a new window began."""
|
|
201
|
+
return tuple((name, d.reset_at) for name, d in sorted(self.dimensions.items()))
|
|
202
|
+
|
|
203
|
+
def describe(self, now: float) -> dict[str, Any]:
|
|
204
|
+
self._refill(now)
|
|
205
|
+
return {
|
|
206
|
+
"dimensions": {name: dim.describe(now) for name, dim in sorted(self.dimensions.items())},
|
|
207
|
+
"blocked_for_s": round(max(0.0, self.blocked_until - now), 1),
|
|
208
|
+
"rate_limited_count": self.rate_limited_count,
|
|
209
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""A watcher's changelog in its resource dir: `events.jsonl` (append-only) and `baseline.json` (its last snapshot).
|
|
2
|
+
|
|
3
|
+
The baseline lets a watcher resubscribe after a daemon restart without a fake "something changed": the new daemon
|
|
4
|
+
diffs against it, and the watcher drops the new `watch_started` when the snapshot hash did not move. It is what this
|
|
5
|
+
watcher saw, which is why it should not be a state file other writers move forward: a watcher that resubscribed from
|
|
6
|
+
such a file would skip the events in between.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
from collections.abc import Iterable
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from cli_watch_kit.atomic import atomic_write_json, atomic_write_text
|
|
19
|
+
from cli_watch_kit.events import Event, utc_now_iso
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class StoredState:
|
|
24
|
+
snapshot_hash: str | None
|
|
25
|
+
snapshot: Any
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Changelog:
|
|
29
|
+
"""The changelog of the resource `resource` in `directory`. `readme`, when given, is written next to it once."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, directory: Path, resource: str, readme: str | None = None) -> None:
|
|
32
|
+
self.dir = directory
|
|
33
|
+
self.resource = resource
|
|
34
|
+
self.readme = readme
|
|
35
|
+
self.events_path = directory / "events.jsonl"
|
|
36
|
+
self.baseline_path = directory / "baseline.json"
|
|
37
|
+
|
|
38
|
+
def _ensure_dir(self) -> None:
|
|
39
|
+
self.dir.mkdir(parents=True, exist_ok=True)
|
|
40
|
+
if self.readme is not None:
|
|
41
|
+
readme = self.dir / "README.txt"
|
|
42
|
+
if not readme.exists():
|
|
43
|
+
atomic_write_text(readme, self.readme)
|
|
44
|
+
|
|
45
|
+
def append(self, events: Iterable[Event]) -> None:
|
|
46
|
+
lines = [json.dumps(e.to_json(), ensure_ascii=False, separators=(",", ":")) + "\n" for e in events]
|
|
47
|
+
if not lines:
|
|
48
|
+
return
|
|
49
|
+
self._ensure_dir()
|
|
50
|
+
with open(self.events_path, "a", encoding="utf-8") as fh:
|
|
51
|
+
fh.writelines(lines)
|
|
52
|
+
|
|
53
|
+
def last_seq(self) -> int:
|
|
54
|
+
"""The `seq` of the last event in events.jsonl, 0 when there is none."""
|
|
55
|
+
try:
|
|
56
|
+
with open(self.events_path, "rb") as fh:
|
|
57
|
+
fh.seek(0, os.SEEK_END)
|
|
58
|
+
size = fh.tell()
|
|
59
|
+
fh.seek(max(0, size - 256 * 1024))
|
|
60
|
+
tail = fh.read()
|
|
61
|
+
except OSError:
|
|
62
|
+
return 0
|
|
63
|
+
for line in reversed(tail.splitlines()):
|
|
64
|
+
try:
|
|
65
|
+
return int(json.loads(line)["seq"])
|
|
66
|
+
except (ValueError, KeyError, TypeError):
|
|
67
|
+
continue
|
|
68
|
+
return 0
|
|
69
|
+
|
|
70
|
+
def load_baseline(self) -> StoredState | None:
|
|
71
|
+
try:
|
|
72
|
+
raw = json.loads(self.baseline_path.read_text(encoding="utf-8"))
|
|
73
|
+
except (OSError, ValueError):
|
|
74
|
+
return None
|
|
75
|
+
if not isinstance(raw, dict) or raw.get("resource") != self.resource:
|
|
76
|
+
return None
|
|
77
|
+
return StoredState(raw.get("snapshot_hash"), raw.get("snapshot"))
|
|
78
|
+
|
|
79
|
+
def save_baseline(self, snapshot_hash: str, snapshot: Any) -> None:
|
|
80
|
+
self._ensure_dir()
|
|
81
|
+
payload = {
|
|
82
|
+
"resource": self.resource,
|
|
83
|
+
"snapshot_hash": snapshot_hash,
|
|
84
|
+
"snapshot": snapshot,
|
|
85
|
+
"written_at": utc_now_iso(),
|
|
86
|
+
}
|
|
87
|
+
atomic_write_json(self.baseline_path, payload)
|