cli-watch-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cli_watch_kit/__init__.py +27 -0
- cli_watch_kit/app.py +175 -0
- cli_watch_kit/atomic.py +37 -0
- cli_watch_kit/budget.py +209 -0
- cli_watch_kit/changelog.py +87 -0
- cli_watch_kit/claude.py +195 -0
- cli_watch_kit/cli.py +410 -0
- cli_watch_kit/client.py +625 -0
- cli_watch_kit/daemon/__init__.py +25 -0
- cli_watch_kit/daemon/cache.py +48 -0
- cli_watch_kit/daemon/engine.py +874 -0
- cli_watch_kit/daemon/lifecycle.py +387 -0
- cli_watch_kit/daemon/scheduler.py +85 -0
- cli_watch_kit/daemon/server.py +116 -0
- cli_watch_kit/daemon/transport.py +324 -0
- cli_watch_kit/durations.py +28 -0
- cli_watch_kit/events.py +182 -0
- cli_watch_kit/locks.py +93 -0
- cli_watch_kit/paths.py +114 -0
- cli_watch_kit/py.typed +0 -0
- cli_watch_kit/sinks.py +106 -0
- cli_watch_kit/source.py +291 -0
- cli_watch_kit/timers.py +67 -0
- cli_watch_kit-0.1.0.dist-info/METADATA +207 -0
- cli_watch_kit-0.1.0.dist-info/RECORD +27 -0
- cli_watch_kit-0.1.0.dist-info/WHEEL +4 -0
- cli_watch_kit-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,874 @@
|
|
|
1
|
+
"""The polling engine: subscriptions with leases, shared resources, per-resource event logs, and the loop that polls
|
|
2
|
+
the due resources in batches under each account's budget.
|
|
3
|
+
|
|
4
|
+
The daemon serves the engine over its socket. `--no-daemon` runs the same engine inside the watcher's process.
|
|
5
|
+
|
|
6
|
+
- **Resources** are shared: two subscriptions to the same target in the same account use one resource, polled for
|
|
7
|
+
the union of the event kinds their filters want. A resource can also be held by a parent resource (an MR holds its
|
|
8
|
+
head pipeline); the parent then receives the child's events in its own log.
|
|
9
|
+
- **Event log:** each resource keeps its last 500 events with sequence numbers. A new subscriber starts at the head of
|
|
10
|
+
the log and gets a private `watch_started` with the current summary, never the backlog.
|
|
11
|
+
- **Leases:** every watcher poll renews all of that watcher's subscriptions. A subscription not renewed within the
|
|
12
|
+
lease timeout is dropped, so a killed watcher never keeps a resource polled.
|
|
13
|
+
- **Batches:** due resources of one account with the same batch key are polled in one call of the source, split so
|
|
14
|
+
that no call costs more than the source's cap. Each call runs only when the account's budget has its cost.
|
|
15
|
+
- **Errors:** each kind of error is reported once as `watch_degraded` until a poll succeeds again, and the resource
|
|
16
|
+
backs off exponentially. Not finding the target before its first successful poll ends the watch.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import asyncio
|
|
22
|
+
import contextlib
|
|
23
|
+
import hashlib
|
|
24
|
+
import json
|
|
25
|
+
import logging
|
|
26
|
+
import time
|
|
27
|
+
import uuid
|
|
28
|
+
from collections import deque
|
|
29
|
+
from collections.abc import Callable, Hashable, Mapping, Sequence
|
|
30
|
+
from dataclasses import dataclass, field
|
|
31
|
+
from typing import Any
|
|
32
|
+
|
|
33
|
+
from cli_watch_kit.app import WatchApp
|
|
34
|
+
from cli_watch_kit.budget import Budget
|
|
35
|
+
from cli_watch_kit.daemon.cache import ByteBoundedCache
|
|
36
|
+
from cli_watch_kit.daemon.scheduler import Scheduler
|
|
37
|
+
from cli_watch_kit.events import (
|
|
38
|
+
LIFECYCLE_KINDS,
|
|
39
|
+
WATCH_DEGRADED,
|
|
40
|
+
WATCH_ENDED,
|
|
41
|
+
WATCH_STARTED,
|
|
42
|
+
Event,
|
|
43
|
+
RawEvent,
|
|
44
|
+
utc_now_iso,
|
|
45
|
+
)
|
|
46
|
+
from cli_watch_kit.source import (
|
|
47
|
+
_CURRENT_POLL,
|
|
48
|
+
NotFound,
|
|
49
|
+
Params,
|
|
50
|
+
PollContext,
|
|
51
|
+
PollItem,
|
|
52
|
+
PollOutcome,
|
|
53
|
+
RateLimited,
|
|
54
|
+
Source,
|
|
55
|
+
SourceError,
|
|
56
|
+
Target,
|
|
57
|
+
Transient,
|
|
58
|
+
Wants,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
log = logging.getLogger(__name__)
|
|
62
|
+
|
|
63
|
+
EVENT_LOG_SIZE = 500
|
|
64
|
+
DEFAULT_LEASE_TIMEOUT = 20.0
|
|
65
|
+
DEFAULT_CLIENT_POLL_MS = 5000
|
|
66
|
+
MAX_ERROR_BACKOFF = 300.0
|
|
67
|
+
MIN_INTERVAL = 0.5
|
|
68
|
+
RECENT_BATCHES = 20
|
|
69
|
+
MAX_STRETCHED_DELAY = 3600.0
|
|
70
|
+
"""The longest a stretched interval grows when the API reports no reset time."""
|
|
71
|
+
FALLBACK_INTERVAL = 60.0
|
|
72
|
+
"""The interval of a resource whose source failed to say one."""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class EngineError(Exception):
|
|
76
|
+
pass
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def stable_hash(value: Any) -> str:
|
|
80
|
+
"""A short hash of a JSON value that does not depend on key order."""
|
|
81
|
+
text = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str)
|
|
82
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass
|
|
86
|
+
class Subscription:
|
|
87
|
+
id: str
|
|
88
|
+
client_id: str
|
|
89
|
+
rid: str
|
|
90
|
+
events: frozenset[str]
|
|
91
|
+
renewed_at: float
|
|
92
|
+
options: Mapping[str, Any] = field(default_factory=dict)
|
|
93
|
+
started: bool = False
|
|
94
|
+
private: list[Event] = field(default_factory=list)
|
|
95
|
+
sent_snapshot_hash: str | None = None
|
|
96
|
+
unacked: list[Event] = field(default_factory=list)
|
|
97
|
+
"""Private events of the last answer, sent again until the watcher acknowledges that answer."""
|
|
98
|
+
unacked_snapshot_hash: str | None = None
|
|
99
|
+
answer: int = 0
|
|
100
|
+
own_baseline: Any = None
|
|
101
|
+
"""A baseline that differs from the one its resource was seeded with, to diff at the first poll."""
|
|
102
|
+
skip_until: int = 0
|
|
103
|
+
"""Log events up to this seq are covered by the subscription's own diff."""
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@dataclass
|
|
107
|
+
class Resource:
|
|
108
|
+
rid: str
|
|
109
|
+
account: str
|
|
110
|
+
target: Target
|
|
111
|
+
batch_key: Hashable | None
|
|
112
|
+
snapshot: Any = None
|
|
113
|
+
snapshot_json: Any = None
|
|
114
|
+
snapshot_hash: str | None = None
|
|
115
|
+
state: dict[str, Any] = field(default_factory=dict)
|
|
116
|
+
log: deque[Event] = field(default_factory=lambda: deque(maxlen=EVENT_LOG_SIZE))
|
|
117
|
+
seq: int = 0
|
|
118
|
+
events: frozenset[str] = frozenset()
|
|
119
|
+
options: tuple[Mapping[str, Any], ...] = ()
|
|
120
|
+
subscriptions: set[str] = field(default_factory=set)
|
|
121
|
+
parents: dict[str, Wants] = field(default_factory=dict)
|
|
122
|
+
"""Parent rid -> what that parent wants from this resource."""
|
|
123
|
+
children: set[str] = field(default_factory=set)
|
|
124
|
+
polled: bool = False
|
|
125
|
+
polling: bool = False
|
|
126
|
+
ended: bool = False
|
|
127
|
+
end_event: Event | None = None
|
|
128
|
+
degraded: str | None = None
|
|
129
|
+
failures: int = 0
|
|
130
|
+
polls: int = 0
|
|
131
|
+
interval: float | None = None
|
|
132
|
+
stretched: bool = False
|
|
133
|
+
"""Whether the budget stretched the delay of the next poll; only such a delay shrinks when a window resets."""
|
|
134
|
+
last_poll_at: float | None = None
|
|
135
|
+
last_error: str | None = None
|
|
136
|
+
|
|
137
|
+
@property
|
|
138
|
+
def key(self) -> str:
|
|
139
|
+
return self.target.key
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def held(self) -> bool:
|
|
143
|
+
return bool(self.subscriptions or self.parents)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@dataclass
|
|
147
|
+
class Account:
|
|
148
|
+
"""What the engine keeps per account: its budget, the params of its latest subscription, and counters."""
|
|
149
|
+
|
|
150
|
+
budget: Budget
|
|
151
|
+
params: Params
|
|
152
|
+
inflight: int = 0
|
|
153
|
+
calls: int = 0
|
|
154
|
+
items: int = 0
|
|
155
|
+
recent_batches: deque[list[str]] = field(default_factory=lambda: deque(maxlen=RECENT_BATCHES))
|
|
156
|
+
windows: tuple[tuple[str, float | None], ...] = ()
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class Engine:
|
|
160
|
+
def __init__(
|
|
161
|
+
self,
|
|
162
|
+
source: Source[Any],
|
|
163
|
+
*,
|
|
164
|
+
clock: Callable[[], float] = time.time,
|
|
165
|
+
lease_clock: Callable[[], float] | None = None,
|
|
166
|
+
lease_timeout: float = DEFAULT_LEASE_TIMEOUT,
|
|
167
|
+
client_poll_ms: int = DEFAULT_CLIENT_POLL_MS,
|
|
168
|
+
) -> None:
|
|
169
|
+
"""`clock` is wall time, because APIs report reset times as wall time. Leases run on `lease_clock`, by default
|
|
170
|
+
a monotonic clock (the given `clock` when it is not the wall clock, as in tests), so a jump of the wall
|
|
171
|
+
clock or a suspend does not expire every lease at once."""
|
|
172
|
+
self.source = source
|
|
173
|
+
self.clock = clock
|
|
174
|
+
self.lease_clock = lease_clock or (time.monotonic if clock is time.time else clock)
|
|
175
|
+
self.lease_timeout = lease_timeout
|
|
176
|
+
self.client_poll_ms = client_poll_ms
|
|
177
|
+
self.started_at = clock()
|
|
178
|
+
self.resources: dict[str, Resource] = {}
|
|
179
|
+
self.subscriptions: dict[str, Subscription] = {}
|
|
180
|
+
self.accounts: dict[str, Account] = {}
|
|
181
|
+
self.scheduler = Scheduler()
|
|
182
|
+
self.cache: ByteBoundedCache[Any] = ByteBoundedCache()
|
|
183
|
+
self._tasks: set[asyncio.Task[None]] = set()
|
|
184
|
+
self._wake: asyncio.Event | None = None
|
|
185
|
+
self._idle_since: float | None = clock()
|
|
186
|
+
|
|
187
|
+
@classmethod
|
|
188
|
+
def for_app(cls, app: WatchApp) -> Engine:
|
|
189
|
+
"""The engine of `app`, with its source and its settings (see :meth:`WatchApp.engine_settings`)."""
|
|
190
|
+
return cls(app.make_source(), **app.engine_settings())
|
|
191
|
+
|
|
192
|
+
# ----------------------------------------------------------------------------------------------------------
|
|
193
|
+
# Watcher-facing operations
|
|
194
|
+
|
|
195
|
+
async def subscribe(
|
|
196
|
+
self,
|
|
197
|
+
client_id: str,
|
|
198
|
+
target: Target,
|
|
199
|
+
events: frozenset[str],
|
|
200
|
+
baseline: Any = None,
|
|
201
|
+
params: Params | None = None,
|
|
202
|
+
options: Mapping[str, Any] | None = None,
|
|
203
|
+
) -> dict[str, Any]:
|
|
204
|
+
if target.kind not in self.source.catalog.kinds_by_resource:
|
|
205
|
+
raise EngineError(f"unknown resource kind {target.kind!r}")
|
|
206
|
+
unknown = set(events) - self.source.catalog.all_kinds
|
|
207
|
+
if unknown:
|
|
208
|
+
raise EngineError(f"unknown event kinds: {', '.join(sorted(unknown))}")
|
|
209
|
+
try:
|
|
210
|
+
checked = self.source.check_options(target, dict(options or {}))
|
|
211
|
+
except ValueError as exc:
|
|
212
|
+
raise EngineError(str(exc)) from exc
|
|
213
|
+
params = dict(params or {})
|
|
214
|
+
account_name = await self.source.account(target, params)
|
|
215
|
+
account = self.account(account_name)
|
|
216
|
+
if params:
|
|
217
|
+
# The latest key of an account wins: a rotated key replaces the old one for every watcher of the account.
|
|
218
|
+
account.params = params
|
|
219
|
+
now = self.lease_clock()
|
|
220
|
+
res = self._acquire(account_name, target)
|
|
221
|
+
previous = self._decode(res, baseline) if baseline is not None else None
|
|
222
|
+
own_baseline = None
|
|
223
|
+
seeded = False
|
|
224
|
+
if previous is not None and not res.polled:
|
|
225
|
+
if res.snapshot is None:
|
|
226
|
+
self._set_snapshot(res, previous)
|
|
227
|
+
seeded = True
|
|
228
|
+
elif stable_hash(self.source.encode(previous)) != res.snapshot_hash:
|
|
229
|
+
# Another resubscriber seeded the resource with another baseline: this one gets its own diff.
|
|
230
|
+
own_baseline = previous
|
|
231
|
+
sub = Subscription(uuid.uuid4().hex, client_id, res.rid, events | frozenset(LIFECYCLE_KINDS), now, checked)
|
|
232
|
+
sub.own_baseline = own_baseline
|
|
233
|
+
self.subscriptions[sub.id] = sub
|
|
234
|
+
res.subscriptions.add(sub.id)
|
|
235
|
+
self._idle_since = None
|
|
236
|
+
cursor = res.seq
|
|
237
|
+
if res.polled:
|
|
238
|
+
sub.private.append(self._started_event(res))
|
|
239
|
+
sub.started = True
|
|
240
|
+
if previous is not None:
|
|
241
|
+
sub.private.extend(self._gap_events(res, previous, sub.events))
|
|
242
|
+
if res.end_event is not None:
|
|
243
|
+
sub.private.append(res.end_event)
|
|
244
|
+
wants = (res.events, res.options)
|
|
245
|
+
try:
|
|
246
|
+
self._recompute_wants(res)
|
|
247
|
+
if seeded:
|
|
248
|
+
# The children start from the seeded snapshot too, so their gap is reported as well. This runs once
|
|
249
|
+
# the subscription counts, so the children know which kinds it wants from them.
|
|
250
|
+
self._reconcile_children(res)
|
|
251
|
+
except Exception:
|
|
252
|
+
# The source's children() failed: take the subscription back, so no orphan keeps the resource alive.
|
|
253
|
+
self.subscriptions.pop(sub.id, None)
|
|
254
|
+
res.subscriptions.discard(sub.id)
|
|
255
|
+
res.events, res.options = wants
|
|
256
|
+
if not self.subscriptions:
|
|
257
|
+
self._idle_since = self.clock()
|
|
258
|
+
self._release(res)
|
|
259
|
+
raise
|
|
260
|
+
log.info("subscribe %s account=%r client=%s sub=%s", target.key, account_name, client_id[:8], sub.id[:8])
|
|
261
|
+
self._kick()
|
|
262
|
+
return {"subscription_id": sub.id, "cursor": cursor}
|
|
263
|
+
|
|
264
|
+
def poll(self, client_id: str, cursors: Mapping[str, int], acks: Mapping[str, int] | None = None) -> dict[str, Any]:
|
|
265
|
+
"""New events and snapshots per subscription, after each cursor.
|
|
266
|
+
|
|
267
|
+
Events in the log are found again through the cursor when an answer is lost. Private events (a subscriber's
|
|
268
|
+
`watch_started`, a replayed `watch_ended`, the events from its baseline gap) and the snapshot are not, so an
|
|
269
|
+
answer numbers them in `acks`, and they are sent again until the watcher passes that number back in `acks`.
|
|
270
|
+
Without `acks` (a caller that never loses an answer) they count as delivered at once.
|
|
271
|
+
"""
|
|
272
|
+
now = self.lease_clock()
|
|
273
|
+
for owned in self.subscriptions.values():
|
|
274
|
+
if owned.client_id == client_id:
|
|
275
|
+
owned.renewed_at = now
|
|
276
|
+
events: dict[str, list[dict[str, Any]]] = {}
|
|
277
|
+
snapshots: dict[str, dict[str, Any]] = {}
|
|
278
|
+
unknown: list[str] = []
|
|
279
|
+
answers: dict[str, int] = {}
|
|
280
|
+
for sid, cursor in cursors.items():
|
|
281
|
+
sub = self.subscriptions.get(sid)
|
|
282
|
+
res = self.resources.get(sub.rid) if sub is not None and sub.client_id == client_id else None
|
|
283
|
+
if sub is None or res is None:
|
|
284
|
+
unknown.append(sid)
|
|
285
|
+
continue
|
|
286
|
+
if acks is None or (sub.answer and acks.get(sid) == sub.answer):
|
|
287
|
+
sub.unacked.clear()
|
|
288
|
+
if sub.unacked_snapshot_hash is not None:
|
|
289
|
+
sub.sent_snapshot_hash = sub.unacked_snapshot_hash
|
|
290
|
+
sub.unacked.extend(sub.private)
|
|
291
|
+
sub.private.clear()
|
|
292
|
+
batch = list(sub.unacked)
|
|
293
|
+
batch.extend(e for e in res.log if e.seq > max(cursor, sub.skip_until) and e.kind in sub.events)
|
|
294
|
+
if batch:
|
|
295
|
+
events[sid] = [e.to_json() for e in batch]
|
|
296
|
+
sub.unacked_snapshot_hash = None
|
|
297
|
+
if res.snapshot_hash is not None and res.snapshot_hash != sub.sent_snapshot_hash:
|
|
298
|
+
snapshots[sid] = {"hash": res.snapshot_hash, "data": res.snapshot_json}
|
|
299
|
+
sub.unacked_snapshot_hash = res.snapshot_hash
|
|
300
|
+
if acks is None:
|
|
301
|
+
sub.unacked.clear()
|
|
302
|
+
if sub.unacked_snapshot_hash is not None:
|
|
303
|
+
sub.sent_snapshot_hash = sub.unacked_snapshot_hash
|
|
304
|
+
sub.answer += 1
|
|
305
|
+
answers[sid] = sub.answer
|
|
306
|
+
return {
|
|
307
|
+
"events": events,
|
|
308
|
+
"snapshots": snapshots,
|
|
309
|
+
"unknown": unknown,
|
|
310
|
+
"acks": answers,
|
|
311
|
+
"next_poll_ms": self.client_poll_ms,
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
def unsubscribe(self, client_id: str, subscription_id: str) -> bool:
|
|
315
|
+
sub = self.subscriptions.get(subscription_id)
|
|
316
|
+
if sub is None or sub.client_id != client_id:
|
|
317
|
+
return False
|
|
318
|
+
self._drop_subscription(sub, "unsubscribed")
|
|
319
|
+
return True
|
|
320
|
+
|
|
321
|
+
def idle_for(self, now: float) -> float:
|
|
322
|
+
"""Seconds without any subscription, 0 while there is one."""
|
|
323
|
+
if self.subscriptions or self._idle_since is None:
|
|
324
|
+
return 0.0
|
|
325
|
+
return now - self._idle_since
|
|
326
|
+
|
|
327
|
+
def reap(self, now: float | None = None) -> None:
|
|
328
|
+
"""Drop the subscriptions whose lease ran out; `now` is on the lease clock."""
|
|
329
|
+
now = self.lease_clock() if now is None else now
|
|
330
|
+
expired = [s for s in self.subscriptions.values() if now - s.renewed_at > self.lease_timeout]
|
|
331
|
+
for sub in expired:
|
|
332
|
+
self._drop_subscription(sub, "lease expired")
|
|
333
|
+
|
|
334
|
+
def status(self) -> dict[str, Any]:
|
|
335
|
+
now = self.clock()
|
|
336
|
+
return {
|
|
337
|
+
"subscriptions": [
|
|
338
|
+
{
|
|
339
|
+
"id": s.id,
|
|
340
|
+
"client_id": s.client_id,
|
|
341
|
+
"resource": self.resources[s.rid].key if s.rid in self.resources else None,
|
|
342
|
+
"account": self.resources[s.rid].account if s.rid in self.resources else None,
|
|
343
|
+
"idle_s": round(self.lease_clock() - s.renewed_at, 1),
|
|
344
|
+
"events": sorted(s.events - frozenset(LIFECYCLE_KINDS)),
|
|
345
|
+
"options": dict(s.options),
|
|
346
|
+
}
|
|
347
|
+
for s in self.subscriptions.values()
|
|
348
|
+
],
|
|
349
|
+
"resources": [
|
|
350
|
+
{
|
|
351
|
+
"resource": r.key,
|
|
352
|
+
"account": r.account,
|
|
353
|
+
"batch_key": None if r.batch_key is None else str(r.batch_key),
|
|
354
|
+
"subscriptions": len(r.subscriptions),
|
|
355
|
+
"parents": sorted(self.resources[p].key for p in r.parents if p in self.resources),
|
|
356
|
+
"children": sorted(self.resources[c].key for c in r.children if c in self.resources),
|
|
357
|
+
"seq": r.seq,
|
|
358
|
+
"polls": r.polls,
|
|
359
|
+
"ended": r.ended,
|
|
360
|
+
"degraded": r.degraded,
|
|
361
|
+
"last_error": r.last_error,
|
|
362
|
+
"interval_s": r.interval,
|
|
363
|
+
"next_poll_in_s": round(d - now, 1) if (d := self.scheduler.due_at(r.rid)) is not None else None,
|
|
364
|
+
}
|
|
365
|
+
for r in self.resources.values()
|
|
366
|
+
],
|
|
367
|
+
"accounts": {
|
|
368
|
+
name: {
|
|
369
|
+
"budget": a.budget.describe(now),
|
|
370
|
+
"calls": a.calls,
|
|
371
|
+
"items": a.items,
|
|
372
|
+
"inflight": a.inflight,
|
|
373
|
+
"stretch": round(self._stretch(name, now), 2),
|
|
374
|
+
"recent_batches": list(a.recent_batches),
|
|
375
|
+
}
|
|
376
|
+
for name, a in sorted(self.accounts.items())
|
|
377
|
+
},
|
|
378
|
+
"cache": self.cache.describe(),
|
|
379
|
+
"source": self.source.status(),
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
# ----------------------------------------------------------------------------------------------------------
|
|
383
|
+
# Helpers for the source's extra ops
|
|
384
|
+
|
|
385
|
+
def account(self, name: str) -> Account:
|
|
386
|
+
account = self.accounts.get(name)
|
|
387
|
+
if account is None:
|
|
388
|
+
account = Account(Budget(self.source.dimensions), {})
|
|
389
|
+
self.accounts[name] = account
|
|
390
|
+
return account
|
|
391
|
+
|
|
392
|
+
def find(self, account: str, target: Target) -> Resource | None:
|
|
393
|
+
"""The resource of `target` in `account`, when some watcher holds it."""
|
|
394
|
+
return self.resources.get(self._rid(account, target))
|
|
395
|
+
|
|
396
|
+
async def await_budget(
|
|
397
|
+
self, account: str, cost: Mapping[str, float], timeout: float = 600.0, *, charge: bool = True
|
|
398
|
+
) -> None:
|
|
399
|
+
"""Wait until the account's budget has `cost`, then charge it, so an on-demand fetch stays as gentle as the
|
|
400
|
+
polls. With `charge=False` only wait: for an op whose client charges each request itself. Raises
|
|
401
|
+
`TimeoutError` after `timeout` seconds."""
|
|
402
|
+
budget = self.account(account).budget
|
|
403
|
+
deadline = time.monotonic() + timeout
|
|
404
|
+
while True:
|
|
405
|
+
now = self.clock()
|
|
406
|
+
if budget.can_spend(cost, now):
|
|
407
|
+
if charge:
|
|
408
|
+
budget.charge(cost, now)
|
|
409
|
+
return
|
|
410
|
+
if time.monotonic() >= deadline:
|
|
411
|
+
raise TimeoutError(f"the budget of account {account!r} had no room within {timeout:.0f} s")
|
|
412
|
+
await asyncio.sleep(min(5.0, max(0.05, budget.wait_time(cost, now))))
|
|
413
|
+
|
|
414
|
+
# ----------------------------------------------------------------------------------------------------------
|
|
415
|
+
# Resources
|
|
416
|
+
|
|
417
|
+
@staticmethod
|
|
418
|
+
def _rid(account: str, target: Target) -> str:
|
|
419
|
+
return f"{account}|{target.key}" if account else target.key
|
|
420
|
+
|
|
421
|
+
def _acquire(self, account: str, target: Target) -> Resource:
|
|
422
|
+
rid = self._rid(account, target)
|
|
423
|
+
res = self.resources.get(rid)
|
|
424
|
+
if res is None:
|
|
425
|
+
res = Resource(rid, account, target, self.source.batch_key(target))
|
|
426
|
+
self.resources[rid] = res
|
|
427
|
+
wait = 0.0
|
|
428
|
+
if res.batch_key is not None:
|
|
429
|
+
configured = self.source.baseline_wait
|
|
430
|
+
wait = self.source.coalesce_window if configured is None else configured
|
|
431
|
+
self.scheduler.schedule(rid, self.clock() + wait, urgent=True)
|
|
432
|
+
log.info("start polling %s", rid)
|
|
433
|
+
return res
|
|
434
|
+
|
|
435
|
+
def _release(self, res: Resource) -> None:
|
|
436
|
+
if res.held or self.resources.get(res.rid) is not res:
|
|
437
|
+
return
|
|
438
|
+
del self.resources[res.rid]
|
|
439
|
+
self.scheduler.forget(res.rid)
|
|
440
|
+
log.info("stop polling %s", res.rid)
|
|
441
|
+
for child_rid in list(res.children):
|
|
442
|
+
self._unlink_child(res, child_rid)
|
|
443
|
+
|
|
444
|
+
def _unlink_child(self, parent: Resource, child_rid: str) -> None:
|
|
445
|
+
parent.children.discard(child_rid)
|
|
446
|
+
child = self.resources.get(child_rid)
|
|
447
|
+
if child is None:
|
|
448
|
+
return
|
|
449
|
+
child.parents.pop(parent.rid, None)
|
|
450
|
+
self._recompute_wants(child)
|
|
451
|
+
self._release(child)
|
|
452
|
+
|
|
453
|
+
def _drop_subscription(self, sub: Subscription, why: str) -> None:
|
|
454
|
+
self.subscriptions.pop(sub.id, None)
|
|
455
|
+
log.info("drop subscription %s on %s: %s", sub.id[:8], sub.rid, why)
|
|
456
|
+
res = self.resources.get(sub.rid)
|
|
457
|
+
if res is not None:
|
|
458
|
+
res.subscriptions.discard(sub.id)
|
|
459
|
+
self._recompute_wants(res)
|
|
460
|
+
self._release(res)
|
|
461
|
+
if not self.subscriptions:
|
|
462
|
+
self._idle_since = self.clock()
|
|
463
|
+
|
|
464
|
+
def _recompute_wants(self, res: Resource) -> None:
|
|
465
|
+
"""The union of what the subscribers and parents of `res` want. Poll again at once when that grew."""
|
|
466
|
+
events: set[str] = set()
|
|
467
|
+
options: list[Mapping[str, Any]] = []
|
|
468
|
+
wants = [(s.events, s.options) for s in (self.subscriptions[sid] for sid in sorted(res.subscriptions))]
|
|
469
|
+
wants += [(w.events, w.options) for _, w in sorted(res.parents.items())]
|
|
470
|
+
for kinds, opts in wants:
|
|
471
|
+
events |= kinds
|
|
472
|
+
if opts and opts not in options:
|
|
473
|
+
options.append(opts)
|
|
474
|
+
new, new_options = frozenset(events), tuple(options)
|
|
475
|
+
if new == res.events and new_options == res.options:
|
|
476
|
+
return
|
|
477
|
+
grew = not new <= res.events or any(o not in res.options for o in new_options)
|
|
478
|
+
res.events, res.options = new, new_options
|
|
479
|
+
if grew and res.polled and not res.ended and not res.polling:
|
|
480
|
+
self.scheduler.schedule(res.rid, self.clock(), urgent=True)
|
|
481
|
+
self._kick()
|
|
482
|
+
if res.polled:
|
|
483
|
+
self._reconcile_children(res)
|
|
484
|
+
|
|
485
|
+
def _reconcile_children(self, res: Resource) -> None:
|
|
486
|
+
wanted: dict[str, tuple[Target, Wants]] = {}
|
|
487
|
+
if not res.ended and res.snapshot is not None:
|
|
488
|
+
for target, kinds in self.source.children(res.target, res.snapshot, self._item(res)).items():
|
|
489
|
+
rid = self._rid(res.account, target)
|
|
490
|
+
if rid != res.rid:
|
|
491
|
+
wanted[rid] = (target, kinds if isinstance(kinds, Wants) else Wants(frozenset(kinds)))
|
|
492
|
+
for child_rid in list(res.children):
|
|
493
|
+
if child_rid not in wanted:
|
|
494
|
+
self._unlink_child(res, child_rid)
|
|
495
|
+
for rid, (target, kinds) in wanted.items():
|
|
496
|
+
child = self._acquire(res.account, target)
|
|
497
|
+
res.children.add(rid)
|
|
498
|
+
if kinds.baseline is not None and not child.polled and child.snapshot is None:
|
|
499
|
+
self._seed(child, kinds.baseline)
|
|
500
|
+
if child.parents.get(res.rid) != kinds:
|
|
501
|
+
child.parents[res.rid] = kinds
|
|
502
|
+
self._recompute_wants(child)
|
|
503
|
+
|
|
504
|
+
def _gap_events(self, res: Resource, previous: Any, kinds: frozenset[str]) -> list[Event]:
|
|
505
|
+
"""What changed between a resubscribing watcher's baseline (decoded) and the snapshot the resource has now,
|
|
506
|
+
including what its children changed, as far as the baseline embeds their state (see `Wants.baseline`)."""
|
|
507
|
+
raws: list[RawEvent] = []
|
|
508
|
+
try:
|
|
509
|
+
if stable_hash(self.source.encode(previous)) == res.snapshot_hash:
|
|
510
|
+
return []
|
|
511
|
+
raws.extend(self.source.diff(res.target, previous, res.snapshot, kinds))
|
|
512
|
+
then = PollItem(res.target, previous, kinds, res.state, res.options)
|
|
513
|
+
for target, wants in self.source.children(res.target, previous, then).items():
|
|
514
|
+
child = self.resources.get(self._rid(res.account, target))
|
|
515
|
+
if not isinstance(wants, Wants) or wants.baseline is None or child is None or child.snapshot is None:
|
|
516
|
+
continue
|
|
517
|
+
before = self.source.decode(target, wants.baseline)
|
|
518
|
+
if stable_hash(self.source.encode(before)) != child.snapshot_hash:
|
|
519
|
+
raws.extend(self.source.diff(target, before, child.snapshot, wants.events & kinds))
|
|
520
|
+
except Exception:
|
|
521
|
+
log.exception("diffing the baseline of %s failed", res.rid)
|
|
522
|
+
observed = utc_now_iso()
|
|
523
|
+
return [Event(r.kind, res.key, res.seq, observed, dict(r.data), r.artifacts) for r in raws if r.kind in kinds]
|
|
524
|
+
|
|
525
|
+
def _decode(self, res: Resource, baseline: Any) -> Any:
|
|
526
|
+
"""A watcher's baseline as a snapshot, or `None` when the source cannot read it."""
|
|
527
|
+
try:
|
|
528
|
+
return self.source.decode(res.target, baseline)
|
|
529
|
+
except Exception as exc:
|
|
530
|
+
log.info("ignoring the baseline of %s: %s", res.rid, exc)
|
|
531
|
+
return None
|
|
532
|
+
|
|
533
|
+
def _seed(self, res: Resource, baseline: Any) -> None:
|
|
534
|
+
"""Start from a snapshot a watcher kept from an earlier daemon, so the first poll reports what changed."""
|
|
535
|
+
try:
|
|
536
|
+
snapshot = self.source.decode(res.target, baseline)
|
|
537
|
+
except (ValueError, TypeError, KeyError) as exc:
|
|
538
|
+
log.info("ignoring the baseline of %s: %s", res.rid, exc)
|
|
539
|
+
return
|
|
540
|
+
self._set_snapshot(res, snapshot)
|
|
541
|
+
|
|
542
|
+
def _set_snapshot(self, res: Resource, snapshot: Any) -> None:
|
|
543
|
+
res.snapshot = snapshot
|
|
544
|
+
res.snapshot_json = self.source.encode(snapshot)
|
|
545
|
+
res.snapshot_hash = stable_hash(res.snapshot_json)
|
|
546
|
+
|
|
547
|
+
# ----------------------------------------------------------------------------------------------------------
|
|
548
|
+
# Events
|
|
549
|
+
|
|
550
|
+
def _started_event(self, res: Resource) -> Event:
|
|
551
|
+
data = {"summary": self.source.summary(res.target, res.snapshot), "snapshot_hash": res.snapshot_hash}
|
|
552
|
+
return Event(WATCH_STARTED, res.key, res.seq, utc_now_iso(), data)
|
|
553
|
+
|
|
554
|
+
def _append(self, res: Resource, raw: RawEvent) -> None:
|
|
555
|
+
res.seq += 1
|
|
556
|
+
event = Event(raw.kind, res.key, res.seq, utc_now_iso(), dict(raw.data), raw.artifacts)
|
|
557
|
+
res.log.append(event)
|
|
558
|
+
if raw.kind in LIFECYCLE_KINDS:
|
|
559
|
+
return
|
|
560
|
+
for parent_rid in res.parents:
|
|
561
|
+
parent = self.resources.get(parent_rid)
|
|
562
|
+
if parent is not None and not parent.ended:
|
|
563
|
+
self._append(parent, raw)
|
|
564
|
+
|
|
565
|
+
def _end(self, res: Resource, reason: str, data: Mapping[str, Any]) -> None:
|
|
566
|
+
self._append(res, RawEvent(WATCH_ENDED, {"reason": reason, **data}))
|
|
567
|
+
res.ended = True
|
|
568
|
+
res.end_event = res.log[-1]
|
|
569
|
+
self.scheduler.remove(res.rid)
|
|
570
|
+
for child_rid in list(res.children):
|
|
571
|
+
self._unlink_child(res, child_rid)
|
|
572
|
+
log.info("ended %s: %s", res.rid, reason)
|
|
573
|
+
|
|
574
|
+
def _degrade(self, res: Resource, reason: str, message: str) -> None:
|
|
575
|
+
res.last_error = message
|
|
576
|
+
if res.degraded == reason:
|
|
577
|
+
return
|
|
578
|
+
res.degraded = reason
|
|
579
|
+
self._append(res, RawEvent(WATCH_DEGRADED, {"reason": reason, "message": message}))
|
|
580
|
+
log.warning("degraded %s: %s: %s", res.rid, reason, message)
|
|
581
|
+
|
|
582
|
+
# ----------------------------------------------------------------------------------------------------------
|
|
583
|
+
# Polling
|
|
584
|
+
|
|
585
|
+
def _kick(self) -> None:
|
|
586
|
+
if self._wake is not None:
|
|
587
|
+
self._wake.set()
|
|
588
|
+
|
|
589
|
+
def _item(self, res: Resource) -> PollItem[Any]:
|
|
590
|
+
return PollItem(res.target, res.snapshot, res.events, res.state, res.options)
|
|
591
|
+
|
|
592
|
+
def _interval(self, res: Resource) -> float:
|
|
593
|
+
if res.interval is None:
|
|
594
|
+
try:
|
|
595
|
+
res.interval = self.source.interval(res.target, res.snapshot)
|
|
596
|
+
except Exception:
|
|
597
|
+
log.exception("the interval of %s failed", res.rid)
|
|
598
|
+
return FALLBACK_INTERVAL
|
|
599
|
+
return max(res.interval, MIN_INTERVAL)
|
|
600
|
+
|
|
601
|
+
def _stretch(self, account: str, now: float) -> float:
|
|
602
|
+
return self._stretch_driver(account, now)[0]
|
|
603
|
+
|
|
604
|
+
def _stretch_driver(self, account: str, now: float) -> tuple[float, str | None]:
|
|
605
|
+
"""The factor all intervals of the account grow by so that the polls the watchers want fit the budget, and
|
|
606
|
+
the budget dimension that makes them grow.
|
|
607
|
+
|
|
608
|
+
The demand counts one call per batch group at the group's shortest interval, which is what batching makes of
|
|
609
|
+
it once the schedules align.
|
|
610
|
+
"""
|
|
611
|
+
groups: dict[Hashable, list[Resource]] = {}
|
|
612
|
+
for res in self.resources.values():
|
|
613
|
+
if res.account == account and not res.ended:
|
|
614
|
+
key = ("solo", res.rid) if res.batch_key is None else ("batch", res.batch_key)
|
|
615
|
+
groups.setdefault(key, []).append(res)
|
|
616
|
+
demand: dict[str, float] = {}
|
|
617
|
+
for members in groups.values():
|
|
618
|
+
interval = min(self._interval(r) for r in members)
|
|
619
|
+
for chunk in self._split(members):
|
|
620
|
+
for dim, amount in self.source.cost([self._item(r) for r in chunk]).items():
|
|
621
|
+
demand[dim] = demand.get(dim, 0.0) + amount / interval
|
|
622
|
+
return self.account(account).budget.stretch(demand, now)
|
|
623
|
+
|
|
624
|
+
def _stretched(self, account: str, base: float, now: float) -> float:
|
|
625
|
+
"""`base` stretched under budget pressure, but never past the end of the window of the dimension that drives
|
|
626
|
+
the stretch, plus `base`: with nothing left until that window resets, polling resumes when it does, not hours
|
|
627
|
+
later. A dimension with a longer window that is not short of anything plays no part."""
|
|
628
|
+
factor, driver = self._stretch_driver(account, now)
|
|
629
|
+
if driver is None:
|
|
630
|
+
return base
|
|
631
|
+
reset_in = self.account(account).budget.reset_in(now, driver)
|
|
632
|
+
cap = base + reset_in if reset_in is not None else max(base, MAX_STRETCHED_DELAY)
|
|
633
|
+
return min(base * factor, cap)
|
|
634
|
+
|
|
635
|
+
def _rescale(self, account_name: str, now: float) -> None:
|
|
636
|
+
"""A new window began: bring forward every resource of the account that waits longer than it now has to."""
|
|
637
|
+
for res in self.resources.values():
|
|
638
|
+
due = self.scheduler.due_at(res.rid)
|
|
639
|
+
if res.account != account_name or res.polling or res.ended or due is None or not res.stretched:
|
|
640
|
+
# Only a wait the budget caused is shortened; a source's next_delay or an error backoff stays.
|
|
641
|
+
continue
|
|
642
|
+
wanted = now + self._stretched(account_name, self._interval(res), now)
|
|
643
|
+
if due > wanted:
|
|
644
|
+
self.scheduler.schedule(res.rid, wanted)
|
|
645
|
+
res.stretched = wanted - now > self._interval(res)
|
|
646
|
+
|
|
647
|
+
def _exceeds(self, cost: Mapping[str, float]) -> bool:
|
|
648
|
+
cap = self.source.batch_cost_cap or {}
|
|
649
|
+
return any(cost.get(dim, 0.0) > limit for dim, limit in cap.items())
|
|
650
|
+
|
|
651
|
+
def _split(self, batch: Sequence[Resource]) -> list[list[Resource]]:
|
|
652
|
+
"""Split a batch greedily, in order, so that no call costs more than the source's cap."""
|
|
653
|
+
if not self.source.batch_cost_cap or len(batch) <= 1:
|
|
654
|
+
return [list(batch)]
|
|
655
|
+
chunks: list[list[Resource]] = []
|
|
656
|
+
current: list[Resource] = []
|
|
657
|
+
for res in batch:
|
|
658
|
+
candidate = [*current, res]
|
|
659
|
+
if current and self._exceeds(self.source.cost([self._item(r) for r in candidate])):
|
|
660
|
+
chunks.append(current)
|
|
661
|
+
current = [res]
|
|
662
|
+
else:
|
|
663
|
+
current = candidate
|
|
664
|
+
if current:
|
|
665
|
+
chunks.append(current)
|
|
666
|
+
return chunks
|
|
667
|
+
|
|
668
|
+
def _group_of(self, rid: str) -> Hashable | None:
|
|
669
|
+
res = self.resources.get(rid)
|
|
670
|
+
if res is None or res.batch_key is None or res.polling or res.ended:
|
|
671
|
+
return None
|
|
672
|
+
return (res.account, res.batch_key)
|
|
673
|
+
|
|
674
|
+
async def _poll(
|
|
675
|
+
self, account_name: str, chunk: list[Resource], items: list[PollItem[Any]], cost: Mapping[str, float]
|
|
676
|
+
) -> None:
|
|
677
|
+
account = self.account(account_name)
|
|
678
|
+
ctx = PollContext(account_name, account.params, account.budget, self.cache, self.clock, dict(cost))
|
|
679
|
+
outcomes: Mapping[str, PollOutcome[Any]] = {}
|
|
680
|
+
# Each poll runs in a task of its own, so this binding is private to the poll.
|
|
681
|
+
_CURRENT_POLL.set(ctx)
|
|
682
|
+
try:
|
|
683
|
+
outcomes = await self.source.poll(items, ctx)
|
|
684
|
+
except SourceError as exc:
|
|
685
|
+
outcomes = {item.key: PollOutcome(error=exc) for item in items}
|
|
686
|
+
except Exception as exc:
|
|
687
|
+
log.exception("poll of %s failed", ", ".join(item.key for item in items))
|
|
688
|
+
error = Transient(f"{type(exc).__name__}: {exc}")
|
|
689
|
+
outcomes = {item.key: PollOutcome(error=error) for item in items}
|
|
690
|
+
finally:
|
|
691
|
+
account.inflight -= 1
|
|
692
|
+
# The real cost replaces the estimate only in the dimensions the source reported; the others keep it.
|
|
693
|
+
refund = {dim: -amount for dim, amount in ctx.estimate.items() if dim in ctx.spent}
|
|
694
|
+
if refund:
|
|
695
|
+
account.budget.charge(refund, self.clock())
|
|
696
|
+
now = self.clock()
|
|
697
|
+
limited = next((o.error for o in outcomes.values() if isinstance(o.error, RateLimited)), None)
|
|
698
|
+
if limited is not None:
|
|
699
|
+
account.budget.on_rate_limited(limited.retry_after, now)
|
|
700
|
+
elif any(o.error is None for o in outcomes.values()):
|
|
701
|
+
account.budget.on_success()
|
|
702
|
+
windows = account.budget.windows()
|
|
703
|
+
if windows != account.windows:
|
|
704
|
+
previous, account.windows = account.windows, windows
|
|
705
|
+
if previous:
|
|
706
|
+
try:
|
|
707
|
+
self._rescale(account_name, now)
|
|
708
|
+
except Exception:
|
|
709
|
+
log.exception("rescheduling the account %r failed", account_name)
|
|
710
|
+
for res, item in zip(chunk, items, strict=True):
|
|
711
|
+
outcome = outcomes.get(item.key)
|
|
712
|
+
if outcome is None:
|
|
713
|
+
outcome = PollOutcome(error=Transient(f"the source returned no outcome for {item.key}"))
|
|
714
|
+
try:
|
|
715
|
+
self._apply(res, outcome, now)
|
|
716
|
+
except Exception as exc:
|
|
717
|
+
# A bug in the source's interval, summary, children or encode must cost this target one poll, not
|
|
718
|
+
# leave it marked as polling forever.
|
|
719
|
+
log.exception("handling the poll of %s failed", res.rid)
|
|
720
|
+
self._fail(res, f"{type(exc).__name__}: {exc}", now)
|
|
721
|
+
self._kick()
|
|
722
|
+
|
|
723
|
+
def _fail(self, res: Resource, message: str, now: float) -> None:
|
|
724
|
+
"""Report an error of the source's code for `res` as `watch_degraded` and try again after a backoff."""
|
|
725
|
+
res.polling = False
|
|
726
|
+
if self.resources.get(res.rid) is not res or res.ended:
|
|
727
|
+
return
|
|
728
|
+
self._degrade(res, "error", message)
|
|
729
|
+
self.scheduler.schedule(res.rid, now + self._error_delay(res))
|
|
730
|
+
|
|
731
|
+
def _apply(self, res: Resource, outcome: PollOutcome[Any], now: float) -> None:
|
|
732
|
+
res.polling = False
|
|
733
|
+
res.polls += 1
|
|
734
|
+
res.last_poll_at = now
|
|
735
|
+
if self.resources.get(res.rid) is not res:
|
|
736
|
+
return
|
|
737
|
+
delay: float | None = None
|
|
738
|
+
error = outcome.error
|
|
739
|
+
if error is not None:
|
|
740
|
+
if isinstance(error, NotFound) and not res.polled and res.snapshot is None:
|
|
741
|
+
# Not found before anything was ever seen: the watcher asked for something that does not exist.
|
|
742
|
+
# A resource seeded from a baseline existed before the restart, so a 404 now may be passing.
|
|
743
|
+
self._end(res, NotFound.reason, {"message": str(error)})
|
|
744
|
+
else:
|
|
745
|
+
self._degrade(res, error.reason, str(error) or type(error).__name__)
|
|
746
|
+
delay = self._error_delay(res)
|
|
747
|
+
if isinstance(error, RateLimited):
|
|
748
|
+
delay = max(delay, self.account(res.account).budget.blocked_until - now)
|
|
749
|
+
else:
|
|
750
|
+
res.failures = 0
|
|
751
|
+
res.degraded = None
|
|
752
|
+
res.last_error = None
|
|
753
|
+
first = not res.polled
|
|
754
|
+
baseline = res.snapshot is None
|
|
755
|
+
res.polled = True
|
|
756
|
+
if outcome.snapshot is not None:
|
|
757
|
+
self._set_snapshot(res, outcome.snapshot)
|
|
758
|
+
for raw in outcome.events:
|
|
759
|
+
self._append(res, raw)
|
|
760
|
+
if first:
|
|
761
|
+
for sid in res.subscriptions:
|
|
762
|
+
sub = self.subscriptions[sid]
|
|
763
|
+
if not sub.started:
|
|
764
|
+
sub.private.append(self._started_event(res))
|
|
765
|
+
sub.started = True
|
|
766
|
+
if sub.own_baseline is not None:
|
|
767
|
+
# Its baseline differs from the one the resource was seeded with: the events in the log
|
|
768
|
+
# are relative to the other one, so it gets its own diff instead.
|
|
769
|
+
sub.private.extend(self._gap_events(res, sub.own_baseline, sub.events))
|
|
770
|
+
sub.own_baseline = None
|
|
771
|
+
sub.skip_until = res.seq
|
|
772
|
+
if outcome.ended is not None:
|
|
773
|
+
data = dict(outcome.end_data)
|
|
774
|
+
if first and baseline:
|
|
775
|
+
data["already_ended"] = True
|
|
776
|
+
self._end(res, outcome.ended, data)
|
|
777
|
+
else:
|
|
778
|
+
self._reconcile_children(res)
|
|
779
|
+
res.interval = self.source.interval(res.target, res.snapshot)
|
|
780
|
+
base = outcome.next_delay if outcome.next_delay is not None else self._interval(res)
|
|
781
|
+
delay = self._stretched(res.account, base, now)
|
|
782
|
+
res.stretched = delay > base
|
|
783
|
+
if error is not None:
|
|
784
|
+
res.stretched = False
|
|
785
|
+
if delay is not None and not res.ended:
|
|
786
|
+
self.scheduler.schedule(res.rid, now + delay)
|
|
787
|
+
|
|
788
|
+
def _error_delay(self, res: Resource) -> float:
|
|
789
|
+
res.failures += 1
|
|
790
|
+
return float(min(self._interval(res) * (2 ** min(res.failures, 8)), MAX_ERROR_BACKOFF))
|
|
791
|
+
|
|
792
|
+
def _start_due(self, now: float) -> float:
|
|
793
|
+
"""Start every due batch the budgets allow. Return seconds until the loop should look again.
|
|
794
|
+
|
|
795
|
+
A due batch that waits for its account's budget sets the wait to when the budget covers it; one that waits
|
|
796
|
+
for a free slot is started when a running poll ends and wakes the loop. Neither makes the loop spin.
|
|
797
|
+
"""
|
|
798
|
+
wait = 1.0
|
|
799
|
+
waiting: set[str] = set()
|
|
800
|
+
for keys in self.scheduler.ready_batches(now, self._group_of, self.source.coalesce_window):
|
|
801
|
+
batch: list[Resource] = []
|
|
802
|
+
for key in keys:
|
|
803
|
+
res = self.resources.get(key)
|
|
804
|
+
if res is None:
|
|
805
|
+
self.scheduler.forget(key)
|
|
806
|
+
elif not res.polling and not res.ended:
|
|
807
|
+
batch.append(res)
|
|
808
|
+
if not batch:
|
|
809
|
+
continue
|
|
810
|
+
account_name = batch[0].account
|
|
811
|
+
account = self.account(account_name)
|
|
812
|
+
try:
|
|
813
|
+
chunks = self._split(batch)
|
|
814
|
+
except Exception as exc:
|
|
815
|
+
log.exception("splitting the batch of %s failed", ", ".join(r.rid for r in batch))
|
|
816
|
+
for res in batch:
|
|
817
|
+
self.scheduler.mark_served(res.rid, now)
|
|
818
|
+
self._fail(res, f"{type(exc).__name__}: {exc}", now)
|
|
819
|
+
continue
|
|
820
|
+
for chunk in chunks:
|
|
821
|
+
if account.inflight >= self.source.max_inflight:
|
|
822
|
+
waiting.update(r.rid for r in chunk)
|
|
823
|
+
continue
|
|
824
|
+
items = [self._item(r) for r in chunk]
|
|
825
|
+
try:
|
|
826
|
+
cost = self.source.cost(items)
|
|
827
|
+
except Exception as exc:
|
|
828
|
+
log.exception("the cost of %s failed", ", ".join(r.rid for r in chunk))
|
|
829
|
+
for res in chunk:
|
|
830
|
+
self.scheduler.mark_served(res.rid, now)
|
|
831
|
+
self._fail(res, f"{type(exc).__name__}: {exc}", now)
|
|
832
|
+
continue
|
|
833
|
+
if not account.budget.can_spend(cost, now):
|
|
834
|
+
wait = min(wait, max(0.05, account.budget.wait_time(cost, now)))
|
|
835
|
+
waiting.update(r.rid for r in chunk)
|
|
836
|
+
continue
|
|
837
|
+
account.budget.charge(cost, now)
|
|
838
|
+
account.inflight += 1
|
|
839
|
+
account.calls += 1
|
|
840
|
+
account.items += len(chunk)
|
|
841
|
+
account.recent_batches.append([r.key for r in chunk])
|
|
842
|
+
for res in chunk:
|
|
843
|
+
self.scheduler.mark_served(res.rid, now)
|
|
844
|
+
res.polling = True
|
|
845
|
+
task = asyncio.create_task(self._poll(account_name, chunk, items, cost))
|
|
846
|
+
self._tasks.add(task)
|
|
847
|
+
task.add_done_callback(self._tasks.discard)
|
|
848
|
+
next_due = self.scheduler.next_due(waiting)
|
|
849
|
+
if next_due is not None:
|
|
850
|
+
wait = min(wait, max(0.05, next_due - now))
|
|
851
|
+
return wait
|
|
852
|
+
|
|
853
|
+
async def poll_due_once(self) -> None:
|
|
854
|
+
"""Start every due batch and wait for all of them: one deterministic step, for tests and debugging."""
|
|
855
|
+
self.reap()
|
|
856
|
+
self._start_due(self.clock())
|
|
857
|
+
while self._tasks:
|
|
858
|
+
await asyncio.gather(*list(self._tasks))
|
|
859
|
+
|
|
860
|
+
async def run(self, stop: asyncio.Event) -> None:
|
|
861
|
+
self._wake = asyncio.Event()
|
|
862
|
+
try:
|
|
863
|
+
while not stop.is_set():
|
|
864
|
+
now = self.clock()
|
|
865
|
+
self.reap()
|
|
866
|
+
wait = self._start_due(now)
|
|
867
|
+
self._wake.clear()
|
|
868
|
+
with contextlib.suppress(TimeoutError):
|
|
869
|
+
await asyncio.wait_for(self._wake.wait(), wait)
|
|
870
|
+
finally:
|
|
871
|
+
for task in list(self._tasks):
|
|
872
|
+
task.cancel()
|
|
873
|
+
if self._tasks:
|
|
874
|
+
await asyncio.gather(*self._tasks, return_exceptions=True)
|