cli-watch-kit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,874 @@
1
+ """The polling engine: subscriptions with leases, shared resources, per-resource event logs, and the loop that polls
2
+ the due resources in batches under each account's budget.
3
+
4
+ The daemon serves the engine over its socket. `--no-daemon` runs the same engine inside the watcher's process.
5
+
6
+ - **Resources** are shared: two subscriptions to the same target in the same account use one resource, polled for
7
+ the union of the event kinds their filters want. A resource can also be held by a parent resource (an MR holds its
8
+ head pipeline); the parent then receives the child's events in its own log.
9
+ - **Event log:** each resource keeps its last 500 events with sequence numbers. A new subscriber starts at the head of
10
+ the log and gets a private `watch_started` with the current summary, never the backlog.
11
+ - **Leases:** every watcher poll renews all of that watcher's subscriptions. A subscription not renewed within the
12
+ lease timeout is dropped, so a killed watcher never keeps a resource polled.
13
+ - **Batches:** due resources of one account with the same batch key are polled in one call of the source, split so
14
+ that no call costs more than the source's cap. Each call runs only when the account's budget has its cost.
15
+ - **Errors:** each kind of error is reported once as `watch_degraded` until a poll succeeds again, and the resource
16
+ backs off exponentially. Not finding the target before its first successful poll ends the watch.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import asyncio
22
+ import contextlib
23
+ import hashlib
24
+ import json
25
+ import logging
26
+ import time
27
+ import uuid
28
+ from collections import deque
29
+ from collections.abc import Callable, Hashable, Mapping, Sequence
30
+ from dataclasses import dataclass, field
31
+ from typing import Any
32
+
33
+ from cli_watch_kit.app import WatchApp
34
+ from cli_watch_kit.budget import Budget
35
+ from cli_watch_kit.daemon.cache import ByteBoundedCache
36
+ from cli_watch_kit.daemon.scheduler import Scheduler
37
+ from cli_watch_kit.events import (
38
+ LIFECYCLE_KINDS,
39
+ WATCH_DEGRADED,
40
+ WATCH_ENDED,
41
+ WATCH_STARTED,
42
+ Event,
43
+ RawEvent,
44
+ utc_now_iso,
45
+ )
46
+ from cli_watch_kit.source import (
47
+ _CURRENT_POLL,
48
+ NotFound,
49
+ Params,
50
+ PollContext,
51
+ PollItem,
52
+ PollOutcome,
53
+ RateLimited,
54
+ Source,
55
+ SourceError,
56
+ Target,
57
+ Transient,
58
+ Wants,
59
+ )
60
+
61
+ log = logging.getLogger(__name__)
62
+
63
+ EVENT_LOG_SIZE = 500
64
+ DEFAULT_LEASE_TIMEOUT = 20.0
65
+ DEFAULT_CLIENT_POLL_MS = 5000
66
+ MAX_ERROR_BACKOFF = 300.0
67
+ MIN_INTERVAL = 0.5
68
+ RECENT_BATCHES = 20
69
+ MAX_STRETCHED_DELAY = 3600.0
70
+ """The longest a stretched interval grows when the API reports no reset time."""
71
+ FALLBACK_INTERVAL = 60.0
72
+ """The interval of a resource whose source failed to say one."""
73
+
74
+
75
+ class EngineError(Exception):
76
+ pass
77
+
78
+
79
+ def stable_hash(value: Any) -> str:
80
+ """A short hash of a JSON value that does not depend on key order."""
81
+ text = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str)
82
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
83
+
84
+
85
+ @dataclass
86
+ class Subscription:
87
+ id: str
88
+ client_id: str
89
+ rid: str
90
+ events: frozenset[str]
91
+ renewed_at: float
92
+ options: Mapping[str, Any] = field(default_factory=dict)
93
+ started: bool = False
94
+ private: list[Event] = field(default_factory=list)
95
+ sent_snapshot_hash: str | None = None
96
+ unacked: list[Event] = field(default_factory=list)
97
+ """Private events of the last answer, sent again until the watcher acknowledges that answer."""
98
+ unacked_snapshot_hash: str | None = None
99
+ answer: int = 0
100
+ own_baseline: Any = None
101
+ """A baseline that differs from the one its resource was seeded with, to diff at the first poll."""
102
+ skip_until: int = 0
103
+ """Log events up to this seq are covered by the subscription's own diff."""
104
+
105
+
106
+ @dataclass
107
+ class Resource:
108
+ rid: str
109
+ account: str
110
+ target: Target
111
+ batch_key: Hashable | None
112
+ snapshot: Any = None
113
+ snapshot_json: Any = None
114
+ snapshot_hash: str | None = None
115
+ state: dict[str, Any] = field(default_factory=dict)
116
+ log: deque[Event] = field(default_factory=lambda: deque(maxlen=EVENT_LOG_SIZE))
117
+ seq: int = 0
118
+ events: frozenset[str] = frozenset()
119
+ options: tuple[Mapping[str, Any], ...] = ()
120
+ subscriptions: set[str] = field(default_factory=set)
121
+ parents: dict[str, Wants] = field(default_factory=dict)
122
+ """Parent rid -> what that parent wants from this resource."""
123
+ children: set[str] = field(default_factory=set)
124
+ polled: bool = False
125
+ polling: bool = False
126
+ ended: bool = False
127
+ end_event: Event | None = None
128
+ degraded: str | None = None
129
+ failures: int = 0
130
+ polls: int = 0
131
+ interval: float | None = None
132
+ stretched: bool = False
133
+ """Whether the budget stretched the delay of the next poll; only such a delay shrinks when a window resets."""
134
+ last_poll_at: float | None = None
135
+ last_error: str | None = None
136
+
137
+ @property
138
+ def key(self) -> str:
139
+ return self.target.key
140
+
141
+ @property
142
+ def held(self) -> bool:
143
+ return bool(self.subscriptions or self.parents)
144
+
145
+
146
+ @dataclass
147
+ class Account:
148
+ """What the engine keeps per account: its budget, the params of its latest subscription, and counters."""
149
+
150
+ budget: Budget
151
+ params: Params
152
+ inflight: int = 0
153
+ calls: int = 0
154
+ items: int = 0
155
+ recent_batches: deque[list[str]] = field(default_factory=lambda: deque(maxlen=RECENT_BATCHES))
156
+ windows: tuple[tuple[str, float | None], ...] = ()
157
+
158
+
159
+ class Engine:
160
+ def __init__(
161
+ self,
162
+ source: Source[Any],
163
+ *,
164
+ clock: Callable[[], float] = time.time,
165
+ lease_clock: Callable[[], float] | None = None,
166
+ lease_timeout: float = DEFAULT_LEASE_TIMEOUT,
167
+ client_poll_ms: int = DEFAULT_CLIENT_POLL_MS,
168
+ ) -> None:
169
+ """`clock` is wall time, because APIs report reset times as wall time. Leases run on `lease_clock`, by default
170
+ a monotonic clock (the given `clock` when it is not the wall clock, as in tests), so a jump of the wall
171
+ clock or a suspend does not expire every lease at once."""
172
+ self.source = source
173
+ self.clock = clock
174
+ self.lease_clock = lease_clock or (time.monotonic if clock is time.time else clock)
175
+ self.lease_timeout = lease_timeout
176
+ self.client_poll_ms = client_poll_ms
177
+ self.started_at = clock()
178
+ self.resources: dict[str, Resource] = {}
179
+ self.subscriptions: dict[str, Subscription] = {}
180
+ self.accounts: dict[str, Account] = {}
181
+ self.scheduler = Scheduler()
182
+ self.cache: ByteBoundedCache[Any] = ByteBoundedCache()
183
+ self._tasks: set[asyncio.Task[None]] = set()
184
+ self._wake: asyncio.Event | None = None
185
+ self._idle_since: float | None = clock()
186
+
187
+ @classmethod
188
+ def for_app(cls, app: WatchApp) -> Engine:
189
+ """The engine of `app`, with its source and its settings (see :meth:`WatchApp.engine_settings`)."""
190
+ return cls(app.make_source(), **app.engine_settings())
191
+
192
+ # ----------------------------------------------------------------------------------------------------------
193
+ # Watcher-facing operations
194
+
195
+ async def subscribe(
196
+ self,
197
+ client_id: str,
198
+ target: Target,
199
+ events: frozenset[str],
200
+ baseline: Any = None,
201
+ params: Params | None = None,
202
+ options: Mapping[str, Any] | None = None,
203
+ ) -> dict[str, Any]:
204
+ if target.kind not in self.source.catalog.kinds_by_resource:
205
+ raise EngineError(f"unknown resource kind {target.kind!r}")
206
+ unknown = set(events) - self.source.catalog.all_kinds
207
+ if unknown:
208
+ raise EngineError(f"unknown event kinds: {', '.join(sorted(unknown))}")
209
+ try:
210
+ checked = self.source.check_options(target, dict(options or {}))
211
+ except ValueError as exc:
212
+ raise EngineError(str(exc)) from exc
213
+ params = dict(params or {})
214
+ account_name = await self.source.account(target, params)
215
+ account = self.account(account_name)
216
+ if params:
217
+ # The latest key of an account wins: a rotated key replaces the old one for every watcher of the account.
218
+ account.params = params
219
+ now = self.lease_clock()
220
+ res = self._acquire(account_name, target)
221
+ previous = self._decode(res, baseline) if baseline is not None else None
222
+ own_baseline = None
223
+ seeded = False
224
+ if previous is not None and not res.polled:
225
+ if res.snapshot is None:
226
+ self._set_snapshot(res, previous)
227
+ seeded = True
228
+ elif stable_hash(self.source.encode(previous)) != res.snapshot_hash:
229
+ # Another resubscriber seeded the resource with another baseline: this one gets its own diff.
230
+ own_baseline = previous
231
+ sub = Subscription(uuid.uuid4().hex, client_id, res.rid, events | frozenset(LIFECYCLE_KINDS), now, checked)
232
+ sub.own_baseline = own_baseline
233
+ self.subscriptions[sub.id] = sub
234
+ res.subscriptions.add(sub.id)
235
+ self._idle_since = None
236
+ cursor = res.seq
237
+ if res.polled:
238
+ sub.private.append(self._started_event(res))
239
+ sub.started = True
240
+ if previous is not None:
241
+ sub.private.extend(self._gap_events(res, previous, sub.events))
242
+ if res.end_event is not None:
243
+ sub.private.append(res.end_event)
244
+ wants = (res.events, res.options)
245
+ try:
246
+ self._recompute_wants(res)
247
+ if seeded:
248
+ # The children start from the seeded snapshot too, so their gap is reported as well. This runs once
249
+ # the subscription counts, so the children know which kinds it wants from them.
250
+ self._reconcile_children(res)
251
+ except Exception:
252
+ # The source's children() failed: take the subscription back, so no orphan keeps the resource alive.
253
+ self.subscriptions.pop(sub.id, None)
254
+ res.subscriptions.discard(sub.id)
255
+ res.events, res.options = wants
256
+ if not self.subscriptions:
257
+ self._idle_since = self.clock()
258
+ self._release(res)
259
+ raise
260
+ log.info("subscribe %s account=%r client=%s sub=%s", target.key, account_name, client_id[:8], sub.id[:8])
261
+ self._kick()
262
+ return {"subscription_id": sub.id, "cursor": cursor}
263
+
264
+ def poll(self, client_id: str, cursors: Mapping[str, int], acks: Mapping[str, int] | None = None) -> dict[str, Any]:
265
+ """New events and snapshots per subscription, after each cursor.
266
+
267
+ Events in the log are found again through the cursor when an answer is lost. Private events (a subscriber's
268
+ `watch_started`, a replayed `watch_ended`, the events from its baseline gap) and the snapshot are not, so an
269
+ answer numbers them in `acks`, and they are sent again until the watcher passes that number back in `acks`.
270
+ Without `acks` (a caller that never loses an answer) they count as delivered at once.
271
+ """
272
+ now = self.lease_clock()
273
+ for owned in self.subscriptions.values():
274
+ if owned.client_id == client_id:
275
+ owned.renewed_at = now
276
+ events: dict[str, list[dict[str, Any]]] = {}
277
+ snapshots: dict[str, dict[str, Any]] = {}
278
+ unknown: list[str] = []
279
+ answers: dict[str, int] = {}
280
+ for sid, cursor in cursors.items():
281
+ sub = self.subscriptions.get(sid)
282
+ res = self.resources.get(sub.rid) if sub is not None and sub.client_id == client_id else None
283
+ if sub is None or res is None:
284
+ unknown.append(sid)
285
+ continue
286
+ if acks is None or (sub.answer and acks.get(sid) == sub.answer):
287
+ sub.unacked.clear()
288
+ if sub.unacked_snapshot_hash is not None:
289
+ sub.sent_snapshot_hash = sub.unacked_snapshot_hash
290
+ sub.unacked.extend(sub.private)
291
+ sub.private.clear()
292
+ batch = list(sub.unacked)
293
+ batch.extend(e for e in res.log if e.seq > max(cursor, sub.skip_until) and e.kind in sub.events)
294
+ if batch:
295
+ events[sid] = [e.to_json() for e in batch]
296
+ sub.unacked_snapshot_hash = None
297
+ if res.snapshot_hash is not None and res.snapshot_hash != sub.sent_snapshot_hash:
298
+ snapshots[sid] = {"hash": res.snapshot_hash, "data": res.snapshot_json}
299
+ sub.unacked_snapshot_hash = res.snapshot_hash
300
+ if acks is None:
301
+ sub.unacked.clear()
302
+ if sub.unacked_snapshot_hash is not None:
303
+ sub.sent_snapshot_hash = sub.unacked_snapshot_hash
304
+ sub.answer += 1
305
+ answers[sid] = sub.answer
306
+ return {
307
+ "events": events,
308
+ "snapshots": snapshots,
309
+ "unknown": unknown,
310
+ "acks": answers,
311
+ "next_poll_ms": self.client_poll_ms,
312
+ }
313
+
314
+ def unsubscribe(self, client_id: str, subscription_id: str) -> bool:
315
+ sub = self.subscriptions.get(subscription_id)
316
+ if sub is None or sub.client_id != client_id:
317
+ return False
318
+ self._drop_subscription(sub, "unsubscribed")
319
+ return True
320
+
321
+ def idle_for(self, now: float) -> float:
322
+ """Seconds without any subscription, 0 while there is one."""
323
+ if self.subscriptions or self._idle_since is None:
324
+ return 0.0
325
+ return now - self._idle_since
326
+
327
+ def reap(self, now: float | None = None) -> None:
328
+ """Drop the subscriptions whose lease ran out; `now` is on the lease clock."""
329
+ now = self.lease_clock() if now is None else now
330
+ expired = [s for s in self.subscriptions.values() if now - s.renewed_at > self.lease_timeout]
331
+ for sub in expired:
332
+ self._drop_subscription(sub, "lease expired")
333
+
334
+ def status(self) -> dict[str, Any]:
335
+ now = self.clock()
336
+ return {
337
+ "subscriptions": [
338
+ {
339
+ "id": s.id,
340
+ "client_id": s.client_id,
341
+ "resource": self.resources[s.rid].key if s.rid in self.resources else None,
342
+ "account": self.resources[s.rid].account if s.rid in self.resources else None,
343
+ "idle_s": round(self.lease_clock() - s.renewed_at, 1),
344
+ "events": sorted(s.events - frozenset(LIFECYCLE_KINDS)),
345
+ "options": dict(s.options),
346
+ }
347
+ for s in self.subscriptions.values()
348
+ ],
349
+ "resources": [
350
+ {
351
+ "resource": r.key,
352
+ "account": r.account,
353
+ "batch_key": None if r.batch_key is None else str(r.batch_key),
354
+ "subscriptions": len(r.subscriptions),
355
+ "parents": sorted(self.resources[p].key for p in r.parents if p in self.resources),
356
+ "children": sorted(self.resources[c].key for c in r.children if c in self.resources),
357
+ "seq": r.seq,
358
+ "polls": r.polls,
359
+ "ended": r.ended,
360
+ "degraded": r.degraded,
361
+ "last_error": r.last_error,
362
+ "interval_s": r.interval,
363
+ "next_poll_in_s": round(d - now, 1) if (d := self.scheduler.due_at(r.rid)) is not None else None,
364
+ }
365
+ for r in self.resources.values()
366
+ ],
367
+ "accounts": {
368
+ name: {
369
+ "budget": a.budget.describe(now),
370
+ "calls": a.calls,
371
+ "items": a.items,
372
+ "inflight": a.inflight,
373
+ "stretch": round(self._stretch(name, now), 2),
374
+ "recent_batches": list(a.recent_batches),
375
+ }
376
+ for name, a in sorted(self.accounts.items())
377
+ },
378
+ "cache": self.cache.describe(),
379
+ "source": self.source.status(),
380
+ }
381
+
382
+ # ----------------------------------------------------------------------------------------------------------
383
+ # Helpers for the source's extra ops
384
+
385
+ def account(self, name: str) -> Account:
386
+ account = self.accounts.get(name)
387
+ if account is None:
388
+ account = Account(Budget(self.source.dimensions), {})
389
+ self.accounts[name] = account
390
+ return account
391
+
392
+ def find(self, account: str, target: Target) -> Resource | None:
393
+ """The resource of `target` in `account`, when some watcher holds it."""
394
+ return self.resources.get(self._rid(account, target))
395
+
396
+ async def await_budget(
397
+ self, account: str, cost: Mapping[str, float], timeout: float = 600.0, *, charge: bool = True
398
+ ) -> None:
399
+ """Wait until the account's budget has `cost`, then charge it, so an on-demand fetch stays as gentle as the
400
+ polls. With `charge=False` only wait: for an op whose client charges each request itself. Raises
401
+ `TimeoutError` after `timeout` seconds."""
402
+ budget = self.account(account).budget
403
+ deadline = time.monotonic() + timeout
404
+ while True:
405
+ now = self.clock()
406
+ if budget.can_spend(cost, now):
407
+ if charge:
408
+ budget.charge(cost, now)
409
+ return
410
+ if time.monotonic() >= deadline:
411
+ raise TimeoutError(f"the budget of account {account!r} had no room within {timeout:.0f} s")
412
+ await asyncio.sleep(min(5.0, max(0.05, budget.wait_time(cost, now))))
413
+
414
+ # ----------------------------------------------------------------------------------------------------------
415
+ # Resources
416
+
417
+ @staticmethod
418
+ def _rid(account: str, target: Target) -> str:
419
+ return f"{account}|{target.key}" if account else target.key
420
+
421
+ def _acquire(self, account: str, target: Target) -> Resource:
422
+ rid = self._rid(account, target)
423
+ res = self.resources.get(rid)
424
+ if res is None:
425
+ res = Resource(rid, account, target, self.source.batch_key(target))
426
+ self.resources[rid] = res
427
+ wait = 0.0
428
+ if res.batch_key is not None:
429
+ configured = self.source.baseline_wait
430
+ wait = self.source.coalesce_window if configured is None else configured
431
+ self.scheduler.schedule(rid, self.clock() + wait, urgent=True)
432
+ log.info("start polling %s", rid)
433
+ return res
434
+
435
+ def _release(self, res: Resource) -> None:
436
+ if res.held or self.resources.get(res.rid) is not res:
437
+ return
438
+ del self.resources[res.rid]
439
+ self.scheduler.forget(res.rid)
440
+ log.info("stop polling %s", res.rid)
441
+ for child_rid in list(res.children):
442
+ self._unlink_child(res, child_rid)
443
+
444
+ def _unlink_child(self, parent: Resource, child_rid: str) -> None:
445
+ parent.children.discard(child_rid)
446
+ child = self.resources.get(child_rid)
447
+ if child is None:
448
+ return
449
+ child.parents.pop(parent.rid, None)
450
+ self._recompute_wants(child)
451
+ self._release(child)
452
+
453
+ def _drop_subscription(self, sub: Subscription, why: str) -> None:
454
+ self.subscriptions.pop(sub.id, None)
455
+ log.info("drop subscription %s on %s: %s", sub.id[:8], sub.rid, why)
456
+ res = self.resources.get(sub.rid)
457
+ if res is not None:
458
+ res.subscriptions.discard(sub.id)
459
+ self._recompute_wants(res)
460
+ self._release(res)
461
+ if not self.subscriptions:
462
+ self._idle_since = self.clock()
463
+
464
+ def _recompute_wants(self, res: Resource) -> None:
465
+ """The union of what the subscribers and parents of `res` want. Poll again at once when that grew."""
466
+ events: set[str] = set()
467
+ options: list[Mapping[str, Any]] = []
468
+ wants = [(s.events, s.options) for s in (self.subscriptions[sid] for sid in sorted(res.subscriptions))]
469
+ wants += [(w.events, w.options) for _, w in sorted(res.parents.items())]
470
+ for kinds, opts in wants:
471
+ events |= kinds
472
+ if opts and opts not in options:
473
+ options.append(opts)
474
+ new, new_options = frozenset(events), tuple(options)
475
+ if new == res.events and new_options == res.options:
476
+ return
477
+ grew = not new <= res.events or any(o not in res.options for o in new_options)
478
+ res.events, res.options = new, new_options
479
+ if grew and res.polled and not res.ended and not res.polling:
480
+ self.scheduler.schedule(res.rid, self.clock(), urgent=True)
481
+ self._kick()
482
+ if res.polled:
483
+ self._reconcile_children(res)
484
+
485
+ def _reconcile_children(self, res: Resource) -> None:
486
+ wanted: dict[str, tuple[Target, Wants]] = {}
487
+ if not res.ended and res.snapshot is not None:
488
+ for target, kinds in self.source.children(res.target, res.snapshot, self._item(res)).items():
489
+ rid = self._rid(res.account, target)
490
+ if rid != res.rid:
491
+ wanted[rid] = (target, kinds if isinstance(kinds, Wants) else Wants(frozenset(kinds)))
492
+ for child_rid in list(res.children):
493
+ if child_rid not in wanted:
494
+ self._unlink_child(res, child_rid)
495
+ for rid, (target, kinds) in wanted.items():
496
+ child = self._acquire(res.account, target)
497
+ res.children.add(rid)
498
+ if kinds.baseline is not None and not child.polled and child.snapshot is None:
499
+ self._seed(child, kinds.baseline)
500
+ if child.parents.get(res.rid) != kinds:
501
+ child.parents[res.rid] = kinds
502
+ self._recompute_wants(child)
503
+
504
+ def _gap_events(self, res: Resource, previous: Any, kinds: frozenset[str]) -> list[Event]:
505
+ """What changed between a resubscribing watcher's baseline (decoded) and the snapshot the resource has now,
506
+ including what its children changed, as far as the baseline embeds their state (see `Wants.baseline`)."""
507
+ raws: list[RawEvent] = []
508
+ try:
509
+ if stable_hash(self.source.encode(previous)) == res.snapshot_hash:
510
+ return []
511
+ raws.extend(self.source.diff(res.target, previous, res.snapshot, kinds))
512
+ then = PollItem(res.target, previous, kinds, res.state, res.options)
513
+ for target, wants in self.source.children(res.target, previous, then).items():
514
+ child = self.resources.get(self._rid(res.account, target))
515
+ if not isinstance(wants, Wants) or wants.baseline is None or child is None or child.snapshot is None:
516
+ continue
517
+ before = self.source.decode(target, wants.baseline)
518
+ if stable_hash(self.source.encode(before)) != child.snapshot_hash:
519
+ raws.extend(self.source.diff(target, before, child.snapshot, wants.events & kinds))
520
+ except Exception:
521
+ log.exception("diffing the baseline of %s failed", res.rid)
522
+ observed = utc_now_iso()
523
+ return [Event(r.kind, res.key, res.seq, observed, dict(r.data), r.artifacts) for r in raws if r.kind in kinds]
524
+
525
+ def _decode(self, res: Resource, baseline: Any) -> Any:
526
+ """A watcher's baseline as a snapshot, or `None` when the source cannot read it."""
527
+ try:
528
+ return self.source.decode(res.target, baseline)
529
+ except Exception as exc:
530
+ log.info("ignoring the baseline of %s: %s", res.rid, exc)
531
+ return None
532
+
533
+ def _seed(self, res: Resource, baseline: Any) -> None:
534
+ """Start from a snapshot a watcher kept from an earlier daemon, so the first poll reports what changed."""
535
+ try:
536
+ snapshot = self.source.decode(res.target, baseline)
537
+ except (ValueError, TypeError, KeyError) as exc:
538
+ log.info("ignoring the baseline of %s: %s", res.rid, exc)
539
+ return
540
+ self._set_snapshot(res, snapshot)
541
+
542
+ def _set_snapshot(self, res: Resource, snapshot: Any) -> None:
543
+ res.snapshot = snapshot
544
+ res.snapshot_json = self.source.encode(snapshot)
545
+ res.snapshot_hash = stable_hash(res.snapshot_json)
546
+
547
+ # ----------------------------------------------------------------------------------------------------------
548
+ # Events
549
+
550
+ def _started_event(self, res: Resource) -> Event:
551
+ data = {"summary": self.source.summary(res.target, res.snapshot), "snapshot_hash": res.snapshot_hash}
552
+ return Event(WATCH_STARTED, res.key, res.seq, utc_now_iso(), data)
553
+
554
+ def _append(self, res: Resource, raw: RawEvent) -> None:
555
+ res.seq += 1
556
+ event = Event(raw.kind, res.key, res.seq, utc_now_iso(), dict(raw.data), raw.artifacts)
557
+ res.log.append(event)
558
+ if raw.kind in LIFECYCLE_KINDS:
559
+ return
560
+ for parent_rid in res.parents:
561
+ parent = self.resources.get(parent_rid)
562
+ if parent is not None and not parent.ended:
563
+ self._append(parent, raw)
564
+
565
+ def _end(self, res: Resource, reason: str, data: Mapping[str, Any]) -> None:
566
+ self._append(res, RawEvent(WATCH_ENDED, {"reason": reason, **data}))
567
+ res.ended = True
568
+ res.end_event = res.log[-1]
569
+ self.scheduler.remove(res.rid)
570
+ for child_rid in list(res.children):
571
+ self._unlink_child(res, child_rid)
572
+ log.info("ended %s: %s", res.rid, reason)
573
+
574
+ def _degrade(self, res: Resource, reason: str, message: str) -> None:
575
+ res.last_error = message
576
+ if res.degraded == reason:
577
+ return
578
+ res.degraded = reason
579
+ self._append(res, RawEvent(WATCH_DEGRADED, {"reason": reason, "message": message}))
580
+ log.warning("degraded %s: %s: %s", res.rid, reason, message)
581
+
582
+ # ----------------------------------------------------------------------------------------------------------
583
+ # Polling
584
+
585
+ def _kick(self) -> None:
586
+ if self._wake is not None:
587
+ self._wake.set()
588
+
589
+ def _item(self, res: Resource) -> PollItem[Any]:
590
+ return PollItem(res.target, res.snapshot, res.events, res.state, res.options)
591
+
592
+ def _interval(self, res: Resource) -> float:
593
+ if res.interval is None:
594
+ try:
595
+ res.interval = self.source.interval(res.target, res.snapshot)
596
+ except Exception:
597
+ log.exception("the interval of %s failed", res.rid)
598
+ return FALLBACK_INTERVAL
599
+ return max(res.interval, MIN_INTERVAL)
600
+
601
+ def _stretch(self, account: str, now: float) -> float:
602
+ return self._stretch_driver(account, now)[0]
603
+
604
+ def _stretch_driver(self, account: str, now: float) -> tuple[float, str | None]:
605
+ """The factor all intervals of the account grow by so that the polls the watchers want fit the budget, and
606
+ the budget dimension that makes them grow.
607
+
608
+ The demand counts one call per batch group at the group's shortest interval, which is what batching makes of
609
+ it once the schedules align.
610
+ """
611
+ groups: dict[Hashable, list[Resource]] = {}
612
+ for res in self.resources.values():
613
+ if res.account == account and not res.ended:
614
+ key = ("solo", res.rid) if res.batch_key is None else ("batch", res.batch_key)
615
+ groups.setdefault(key, []).append(res)
616
+ demand: dict[str, float] = {}
617
+ for members in groups.values():
618
+ interval = min(self._interval(r) for r in members)
619
+ for chunk in self._split(members):
620
+ for dim, amount in self.source.cost([self._item(r) for r in chunk]).items():
621
+ demand[dim] = demand.get(dim, 0.0) + amount / interval
622
+ return self.account(account).budget.stretch(demand, now)
623
+
624
+ def _stretched(self, account: str, base: float, now: float) -> float:
625
+ """`base` stretched under budget pressure, but never past the end of the window of the dimension that drives
626
+ the stretch, plus `base`: with nothing left until that window resets, polling resumes when it does, not hours
627
+ later. A dimension with a longer window that is not short of anything plays no part."""
628
+ factor, driver = self._stretch_driver(account, now)
629
+ if driver is None:
630
+ return base
631
+ reset_in = self.account(account).budget.reset_in(now, driver)
632
+ cap = base + reset_in if reset_in is not None else max(base, MAX_STRETCHED_DELAY)
633
+ return min(base * factor, cap)
634
+
635
+ def _rescale(self, account_name: str, now: float) -> None:
636
+ """A new window began: bring forward every resource of the account that waits longer than it now has to."""
637
+ for res in self.resources.values():
638
+ due = self.scheduler.due_at(res.rid)
639
+ if res.account != account_name or res.polling or res.ended or due is None or not res.stretched:
640
+ # Only a wait the budget caused is shortened; a source's next_delay or an error backoff stays.
641
+ continue
642
+ wanted = now + self._stretched(account_name, self._interval(res), now)
643
+ if due > wanted:
644
+ self.scheduler.schedule(res.rid, wanted)
645
+ res.stretched = wanted - now > self._interval(res)
646
+
647
+ def _exceeds(self, cost: Mapping[str, float]) -> bool:
648
+ cap = self.source.batch_cost_cap or {}
649
+ return any(cost.get(dim, 0.0) > limit for dim, limit in cap.items())
650
+
651
+ def _split(self, batch: Sequence[Resource]) -> list[list[Resource]]:
652
+ """Split a batch greedily, in order, so that no call costs more than the source's cap."""
653
+ if not self.source.batch_cost_cap or len(batch) <= 1:
654
+ return [list(batch)]
655
+ chunks: list[list[Resource]] = []
656
+ current: list[Resource] = []
657
+ for res in batch:
658
+ candidate = [*current, res]
659
+ if current and self._exceeds(self.source.cost([self._item(r) for r in candidate])):
660
+ chunks.append(current)
661
+ current = [res]
662
+ else:
663
+ current = candidate
664
+ if current:
665
+ chunks.append(current)
666
+ return chunks
667
+
668
+ def _group_of(self, rid: str) -> Hashable | None:
669
+ res = self.resources.get(rid)
670
+ if res is None or res.batch_key is None or res.polling or res.ended:
671
+ return None
672
+ return (res.account, res.batch_key)
673
+
674
+ async def _poll(
675
+ self, account_name: str, chunk: list[Resource], items: list[PollItem[Any]], cost: Mapping[str, float]
676
+ ) -> None:
677
+ account = self.account(account_name)
678
+ ctx = PollContext(account_name, account.params, account.budget, self.cache, self.clock, dict(cost))
679
+ outcomes: Mapping[str, PollOutcome[Any]] = {}
680
+ # Each poll runs in a task of its own, so this binding is private to the poll.
681
+ _CURRENT_POLL.set(ctx)
682
+ try:
683
+ outcomes = await self.source.poll(items, ctx)
684
+ except SourceError as exc:
685
+ outcomes = {item.key: PollOutcome(error=exc) for item in items}
686
+ except Exception as exc:
687
+ log.exception("poll of %s failed", ", ".join(item.key for item in items))
688
+ error = Transient(f"{type(exc).__name__}: {exc}")
689
+ outcomes = {item.key: PollOutcome(error=error) for item in items}
690
+ finally:
691
+ account.inflight -= 1
692
+ # The real cost replaces the estimate only in the dimensions the source reported; the others keep it.
693
+ refund = {dim: -amount for dim, amount in ctx.estimate.items() if dim in ctx.spent}
694
+ if refund:
695
+ account.budget.charge(refund, self.clock())
696
+ now = self.clock()
697
+ limited = next((o.error for o in outcomes.values() if isinstance(o.error, RateLimited)), None)
698
+ if limited is not None:
699
+ account.budget.on_rate_limited(limited.retry_after, now)
700
+ elif any(o.error is None for o in outcomes.values()):
701
+ account.budget.on_success()
702
+ windows = account.budget.windows()
703
+ if windows != account.windows:
704
+ previous, account.windows = account.windows, windows
705
+ if previous:
706
+ try:
707
+ self._rescale(account_name, now)
708
+ except Exception:
709
+ log.exception("rescheduling the account %r failed", account_name)
710
+ for res, item in zip(chunk, items, strict=True):
711
+ outcome = outcomes.get(item.key)
712
+ if outcome is None:
713
+ outcome = PollOutcome(error=Transient(f"the source returned no outcome for {item.key}"))
714
+ try:
715
+ self._apply(res, outcome, now)
716
+ except Exception as exc:
717
+ # A bug in the source's interval, summary, children or encode must cost this target one poll, not
718
+ # leave it marked as polling forever.
719
+ log.exception("handling the poll of %s failed", res.rid)
720
+ self._fail(res, f"{type(exc).__name__}: {exc}", now)
721
+ self._kick()
722
+
723
+ def _fail(self, res: Resource, message: str, now: float) -> None:
724
+ """Report an error of the source's code for `res` as `watch_degraded` and try again after a backoff."""
725
+ res.polling = False
726
+ if self.resources.get(res.rid) is not res or res.ended:
727
+ return
728
+ self._degrade(res, "error", message)
729
+ self.scheduler.schedule(res.rid, now + self._error_delay(res))
730
+
731
+ def _apply(self, res: Resource, outcome: PollOutcome[Any], now: float) -> None:
732
+ res.polling = False
733
+ res.polls += 1
734
+ res.last_poll_at = now
735
+ if self.resources.get(res.rid) is not res:
736
+ return
737
+ delay: float | None = None
738
+ error = outcome.error
739
+ if error is not None:
740
+ if isinstance(error, NotFound) and not res.polled and res.snapshot is None:
741
+ # Not found before anything was ever seen: the watcher asked for something that does not exist.
742
+ # A resource seeded from a baseline existed before the restart, so a 404 now may be passing.
743
+ self._end(res, NotFound.reason, {"message": str(error)})
744
+ else:
745
+ self._degrade(res, error.reason, str(error) or type(error).__name__)
746
+ delay = self._error_delay(res)
747
+ if isinstance(error, RateLimited):
748
+ delay = max(delay, self.account(res.account).budget.blocked_until - now)
749
+ else:
750
+ res.failures = 0
751
+ res.degraded = None
752
+ res.last_error = None
753
+ first = not res.polled
754
+ baseline = res.snapshot is None
755
+ res.polled = True
756
+ if outcome.snapshot is not None:
757
+ self._set_snapshot(res, outcome.snapshot)
758
+ for raw in outcome.events:
759
+ self._append(res, raw)
760
+ if first:
761
+ for sid in res.subscriptions:
762
+ sub = self.subscriptions[sid]
763
+ if not sub.started:
764
+ sub.private.append(self._started_event(res))
765
+ sub.started = True
766
+ if sub.own_baseline is not None:
767
+ # Its baseline differs from the one the resource was seeded with: the events in the log
768
+ # are relative to the other one, so it gets its own diff instead.
769
+ sub.private.extend(self._gap_events(res, sub.own_baseline, sub.events))
770
+ sub.own_baseline = None
771
+ sub.skip_until = res.seq
772
+ if outcome.ended is not None:
773
+ data = dict(outcome.end_data)
774
+ if first and baseline:
775
+ data["already_ended"] = True
776
+ self._end(res, outcome.ended, data)
777
+ else:
778
+ self._reconcile_children(res)
779
+ res.interval = self.source.interval(res.target, res.snapshot)
780
+ base = outcome.next_delay if outcome.next_delay is not None else self._interval(res)
781
+ delay = self._stretched(res.account, base, now)
782
+ res.stretched = delay > base
783
+ if error is not None:
784
+ res.stretched = False
785
+ if delay is not None and not res.ended:
786
+ self.scheduler.schedule(res.rid, now + delay)
787
+
788
+ def _error_delay(self, res: Resource) -> float:
789
+ res.failures += 1
790
+ return float(min(self._interval(res) * (2 ** min(res.failures, 8)), MAX_ERROR_BACKOFF))
791
+
792
+ def _start_due(self, now: float) -> float:
793
+ """Start every due batch the budgets allow. Return seconds until the loop should look again.
794
+
795
+ A due batch that waits for its account's budget sets the wait to when the budget covers it; one that waits
796
+ for a free slot is started when a running poll ends and wakes the loop. Neither makes the loop spin.
797
+ """
798
+ wait = 1.0
799
+ waiting: set[str] = set()
800
+ for keys in self.scheduler.ready_batches(now, self._group_of, self.source.coalesce_window):
801
+ batch: list[Resource] = []
802
+ for key in keys:
803
+ res = self.resources.get(key)
804
+ if res is None:
805
+ self.scheduler.forget(key)
806
+ elif not res.polling and not res.ended:
807
+ batch.append(res)
808
+ if not batch:
809
+ continue
810
+ account_name = batch[0].account
811
+ account = self.account(account_name)
812
+ try:
813
+ chunks = self._split(batch)
814
+ except Exception as exc:
815
+ log.exception("splitting the batch of %s failed", ", ".join(r.rid for r in batch))
816
+ for res in batch:
817
+ self.scheduler.mark_served(res.rid, now)
818
+ self._fail(res, f"{type(exc).__name__}: {exc}", now)
819
+ continue
820
+ for chunk in chunks:
821
+ if account.inflight >= self.source.max_inflight:
822
+ waiting.update(r.rid for r in chunk)
823
+ continue
824
+ items = [self._item(r) for r in chunk]
825
+ try:
826
+ cost = self.source.cost(items)
827
+ except Exception as exc:
828
+ log.exception("the cost of %s failed", ", ".join(r.rid for r in chunk))
829
+ for res in chunk:
830
+ self.scheduler.mark_served(res.rid, now)
831
+ self._fail(res, f"{type(exc).__name__}: {exc}", now)
832
+ continue
833
+ if not account.budget.can_spend(cost, now):
834
+ wait = min(wait, max(0.05, account.budget.wait_time(cost, now)))
835
+ waiting.update(r.rid for r in chunk)
836
+ continue
837
+ account.budget.charge(cost, now)
838
+ account.inflight += 1
839
+ account.calls += 1
840
+ account.items += len(chunk)
841
+ account.recent_batches.append([r.key for r in chunk])
842
+ for res in chunk:
843
+ self.scheduler.mark_served(res.rid, now)
844
+ res.polling = True
845
+ task = asyncio.create_task(self._poll(account_name, chunk, items, cost))
846
+ self._tasks.add(task)
847
+ task.add_done_callback(self._tasks.discard)
848
+ next_due = self.scheduler.next_due(waiting)
849
+ if next_due is not None:
850
+ wait = min(wait, max(0.05, next_due - now))
851
+ return wait
852
+
853
+ async def poll_due_once(self) -> None:
854
+ """Start every due batch and wait for all of them: one deterministic step, for tests and debugging."""
855
+ self.reap()
856
+ self._start_due(self.clock())
857
+ while self._tasks:
858
+ await asyncio.gather(*list(self._tasks))
859
+
860
+ async def run(self, stop: asyncio.Event) -> None:
861
+ self._wake = asyncio.Event()
862
+ try:
863
+ while not stop.is_set():
864
+ now = self.clock()
865
+ self.reap()
866
+ wait = self._start_due(now)
867
+ self._wake.clear()
868
+ with contextlib.suppress(TimeoutError):
869
+ await asyncio.wait_for(self._wake.wait(), wait)
870
+ finally:
871
+ for task in list(self._tasks):
872
+ task.cancel()
873
+ if self._tasks:
874
+ await asyncio.gather(*self._tasks, return_exceptions=True)