stacktrace-cli 0.19.0__py3-none-any.whl → 0.20.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stacktrace/__init__.py +16 -2
- stacktrace/__main__.py +19 -3
- stacktrace/cloud/boot.py +160 -21
- stacktrace/correlate/observed.py +1 -1
- stacktrace/daemon/activity_log.py +2 -1
- stacktrace/daemon/costs.py +22 -6
- stacktrace/daemon/frame_limits.py +11 -0
- stacktrace/daemon/inline_hook.py +65 -6
- stacktrace/daemon/paths.py +7 -2
- stacktrace/daemon/policy_collect.py +10 -2
- stacktrace/daemon/policy_result.py +38 -0
- stacktrace/daemon/policy_runtime.py +63 -33
- stacktrace/daemon/protocol.py +2 -5
- stacktrace/daemon/runtime.py +3 -2
- stacktrace/daemon/unreadable.py +101 -0
- stacktrace/daemon/usage.py +10 -1
- stacktrace/detector/deterministic.py +1870 -178
- stacktrace/detector/guard_files.py +115 -0
- stacktrace/detector/inline.py +20 -1
- stacktrace/detector/rules.py +70 -4
- stacktrace/detector/run.py +2 -2
- stacktrace/detector/secret_paths.py +206 -0
- stacktrace/detector/secret_refs.py +231 -0
- stacktrace/detector/wipers.py +494 -0
- stacktrace/finding_titles.py +8 -1
- stacktrace/hooks/admit.py +102 -0
- stacktrace/hooks/claude.py +63 -10
- stacktrace/hooks/codex.py +10 -13
- stacktrace/hooks/cursor.py +10 -13
- stacktrace/hooks/devin.py +10 -13
- stacktrace/hooks/session.py +60 -9
- stacktrace/monitor/site/app.js +2 -0
- stacktrace/policy_display.py +5 -1
- stacktrace/remote/cli.py +4 -0
- stacktrace/remote/hook_policy.py +8 -12
- stacktrace/remote/policy_service.py +27 -0
- stacktrace/sessions/outcome.py +19 -0
- stacktrace/sessions/toolclass.py +13 -0
- {stacktrace_cli-0.19.0.dist-info → stacktrace_cli-0.20.0.dist-info}/METADATA +4 -4
- {stacktrace_cli-0.19.0.dist-info → stacktrace_cli-0.20.0.dist-info}/RECORD +42 -33
- {stacktrace_cli-0.19.0.dist-info → stacktrace_cli-0.20.0.dist-info}/WHEEL +0 -0
- {stacktrace_cli-0.19.0.dist-info → stacktrace_cli-0.20.0.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""An admission answer, apart from the runtime that computes it.
|
|
2
|
+
|
|
3
|
+
The hook builds these from the daemon's reply on every covered call, so this
|
|
4
|
+
module imports only the standard library (ADR-0194). `policy_runtime` imports
|
|
5
|
+
them from here.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class DecidedBy:
|
|
16
|
+
"""Which policy entry, rule and revision decided an answer (ADR-0159)."""
|
|
17
|
+
|
|
18
|
+
policy_entry_id: str
|
|
19
|
+
rule_id: str
|
|
20
|
+
policy_revision: int
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class Result:
|
|
25
|
+
outcome: str
|
|
26
|
+
reason: str
|
|
27
|
+
revision: int | None = None
|
|
28
|
+
rule: str | None = None
|
|
29
|
+
detail: str | None = None
|
|
30
|
+
decided_by: DecidedBy | None = None
|
|
31
|
+
#: Run only once the caller has confirmed this decision reached the hook.
|
|
32
|
+
#: A detection block's `on_blocked` commit and a component deny's
|
|
33
|
+
#: `on_component_denied` commit are deferred to here: committing either
|
|
34
|
+
#: eagerly, inside `authorize()`, would keep a block's marker for a prompt
|
|
35
|
+
#: or call whose deny never reached the agent, because the hook's own
|
|
36
|
+
#: timeout (`CALL_TIMEOUT`) expired first and it already let the prompt or
|
|
37
|
+
#: call through. The scan would then report a call that ran as blocked.
|
|
38
|
+
commit: tuple[Callable[[], None], ...] = ()
|
|
@@ -17,7 +17,8 @@ from typing import TYPE_CHECKING, Literal
|
|
|
17
17
|
|
|
18
18
|
from openaca.core import Policy, PolicyValidationError, parse_policy
|
|
19
19
|
|
|
20
|
-
from stacktrace.
|
|
20
|
+
from stacktrace.correlate.observed import declares_remote_transport
|
|
21
|
+
from stacktrace.daemon.jobs import Backoff, Job, JobThread
|
|
21
22
|
from stacktrace.daemon.policy_inputs import (
|
|
22
23
|
DetectionEntry,
|
|
23
24
|
PolicyEntry,
|
|
@@ -28,6 +29,7 @@ from stacktrace.daemon.policy_inputs import (
|
|
|
28
29
|
Tenant,
|
|
29
30
|
off_rules,
|
|
30
31
|
)
|
|
32
|
+
from stacktrace.daemon.policy_result import DecidedBy, Result
|
|
31
33
|
from stacktrace.detector.rules import ADMISSION_RULES, BLOCKABLE, POSTURE_RULE, VULNERABILITY_RULE
|
|
32
34
|
from stacktrace.policy_display import default_description
|
|
33
35
|
from stacktrace.remote.config import ConfigError, RemoteConfig, load_remote_config
|
|
@@ -203,33 +205,9 @@ class Entry:
|
|
|
203
205
|
version: str | None = None
|
|
204
206
|
identity: str | None = None
|
|
205
207
|
evidence: tuple[str, ...] = ()
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
class DecidedBy:
|
|
210
|
-
"""Which policy entry, rule and revision decided an answer (ADR-0159)."""
|
|
211
|
-
|
|
212
|
-
policy_entry_id: str
|
|
213
|
-
rule_id: str
|
|
214
|
-
policy_revision: int
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
@dataclass(frozen=True)
|
|
218
|
-
class Result:
|
|
219
|
-
outcome: str
|
|
220
|
-
reason: str
|
|
221
|
-
revision: int | None = None
|
|
222
|
-
rule: str | None = None
|
|
223
|
-
detail: str | None = None
|
|
224
|
-
decided_by: DecidedBy | None = None
|
|
225
|
-
#: Run only once the caller has confirmed this decision reached the hook.
|
|
226
|
-
#: A detection block's `on_blocked` commit and a component deny's
|
|
227
|
-
#: `on_component_denied` commit are deferred to here: committing either
|
|
228
|
-
#: eagerly, inside `authorize()`, would keep a block's marker for a prompt
|
|
229
|
-
#: or call whose deny never reached the agent, because the hook's own
|
|
230
|
-
#: timeout (`CALL_TIMEOUT`) expired first and it already let the prompt or
|
|
231
|
-
#: call through. The scan would then report a call that ran as blocked.
|
|
232
|
-
commit: tuple[Callable[[], None], ...] = ()
|
|
208
|
+
#: An MCP server's configured transport. The egress rules judge a call's
|
|
209
|
+
#: input only for a server whose configuration names a remote one (ADR-0195).
|
|
210
|
+
transport: str | None = None
|
|
233
211
|
|
|
234
212
|
|
|
235
213
|
@dataclass(frozen=True)
|
|
@@ -533,6 +511,18 @@ def configuration_digest(scope: Scope, watch_paths: tuple[str, ...]) -> str:
|
|
|
533
511
|
return hashlib.sha256(json.dumps(records).encode()).hexdigest()
|
|
534
512
|
|
|
535
513
|
|
|
514
|
+
def _occurrence_of(call: dict[str, object]) -> dict[str, object]:
|
|
515
|
+
"""An MCP action's occurrence keys, with the call id its `call` carries as
|
|
516
|
+
`tool_use_id`, the key a component deny's marker names the call by
|
|
517
|
+
(ADR-0195). The call itself is kept for the egress judge."""
|
|
518
|
+
inner = call.get("call")
|
|
519
|
+
tool_use_id = inner.get("tool_use_id") if isinstance(inner, dict) else None # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
|
|
520
|
+
result = dict(call)
|
|
521
|
+
if isinstance(tool_use_id, str):
|
|
522
|
+
result["tool_use_id"] = tool_use_id
|
|
523
|
+
return result
|
|
524
|
+
|
|
525
|
+
|
|
536
526
|
class PolicyRuntime:
|
|
537
527
|
"""One coalescing worker; hooks only register/read, never collect or fetch."""
|
|
538
528
|
|
|
@@ -563,6 +553,14 @@ class PolicyRuntime:
|
|
|
563
553
|
self._attempt_interval = min(30, poll_interval)
|
|
564
554
|
self._fetch_at = float("-inf")
|
|
565
555
|
self._attempts: dict[Scope, float] = {}
|
|
556
|
+
#: How long a scope whose last collection failed waits before the
|
|
557
|
+
#: next: 30 s, doubling, never past an hour. A collection that fails
|
|
558
|
+
#: every time, such as a project too large to walk within the
|
|
559
|
+
#: deadline, otherwise ran again every 30 s for as long as the daemon
|
|
560
|
+
#: did. A published snapshot resets it; a configuration change does
|
|
561
|
+
#: not, because `~/.claude.json` changes about once a minute.
|
|
562
|
+
self._retries: dict[Scope, Backoff] = {}
|
|
563
|
+
self._waits: dict[Scope, float] = {}
|
|
566
564
|
self._digests: dict[Scope, str] = {}
|
|
567
565
|
self._paths: dict[Scope, tuple[str, ...]] = {}
|
|
568
566
|
self._error: str | None = None
|
|
@@ -660,7 +658,14 @@ class PolicyRuntime:
|
|
|
660
658
|
# Freshness begins before collection; slow work does not extend it.
|
|
661
659
|
started = self.clock()
|
|
662
660
|
wall_started = self.wall_clock()
|
|
663
|
-
|
|
661
|
+
try:
|
|
662
|
+
collected = self.collect(document, scope)
|
|
663
|
+
except Exception:
|
|
664
|
+
retries = self._retries.setdefault(
|
|
665
|
+
scope, Backoff(ceiling=REFRESH_INTERVAL, first=self._attempt_interval)
|
|
666
|
+
)
|
|
667
|
+
self._waits[scope] = retries.next()
|
|
668
|
+
raise
|
|
664
669
|
# Recheck the inputs observed before collection. New discovered paths
|
|
665
670
|
# join the next pass's watch set only after successful publication.
|
|
666
671
|
digest = configuration_digest(scope, self._paths.get(scope, ()))
|
|
@@ -682,6 +687,8 @@ class PolicyRuntime:
|
|
|
682
687
|
collected.builtin_skills,
|
|
683
688
|
):
|
|
684
689
|
self._paths[scope] = collected.watch_paths
|
|
690
|
+
self._retries.pop(scope, None)
|
|
691
|
+
self._waits.pop(scope, None)
|
|
685
692
|
self._digests[scope] = configuration_digest(scope, collected.watch_paths)
|
|
686
693
|
self._error = None
|
|
687
694
|
if len(ready) > 1:
|
|
@@ -698,7 +705,8 @@ class PolicyRuntime:
|
|
|
698
705
|
def _due(self, scope: Scope) -> bool:
|
|
699
706
|
if scope.agent_kind in UNCOLLECTED:
|
|
700
707
|
return False
|
|
701
|
-
|
|
708
|
+
wait = self._waits.get(scope, self._attempt_interval)
|
|
709
|
+
if self.clock() - self._attempts.get(scope, float("-inf")) < wait:
|
|
702
710
|
return False
|
|
703
711
|
snapshot = self.cache.snapshots.get(scope)
|
|
704
712
|
return (
|
|
@@ -787,6 +795,10 @@ class PolicyRuntime:
|
|
|
787
795
|
kind, name, source = action["action_kind"], action["name"], action["source"]
|
|
788
796
|
builtin = kind == "tool" and source == "builtin" and "call" in call
|
|
789
797
|
prompt = kind == "prompt" and "call" in call
|
|
798
|
+
# An MCP call's input, judged once its server is decided, whether
|
|
799
|
+
# or not the snapshot places the server (ADR-0202). A server
|
|
800
|
+
# configured as remote adds egress (ADR-0195).
|
|
801
|
+
mcp = kind == "mcp_tool" and "call" in call
|
|
790
802
|
if "prompt_id" in call and not prompt:
|
|
791
803
|
return Result("defer", "invalid_action")
|
|
792
804
|
if (
|
|
@@ -795,9 +807,7 @@ class PolicyRuntime:
|
|
|
795
807
|
and not prompt
|
|
796
808
|
):
|
|
797
809
|
return Result("defer", "uncovered_action")
|
|
798
|
-
if "call" in call and not builtin and not prompt:
|
|
799
|
-
# This step judges built-in calls only; an MCP call's egress
|
|
800
|
-
# needs its server's declared transport first.
|
|
810
|
+
if "call" in call and not builtin and not prompt and not mcp:
|
|
801
811
|
return Result("defer", "invalid_action")
|
|
802
812
|
scope = Scope(
|
|
803
813
|
str(action["config_root"]),
|
|
@@ -840,10 +850,30 @@ class PolicyRuntime:
|
|
|
840
850
|
if e.action == "block"
|
|
841
851
|
and e.rule_id in BLOCKABLE
|
|
842
852
|
and event in BLOCKABLE[e.rule_id].events
|
|
853
|
+
and (not mcp or BLOCKABLE[e.rule_id].mcp)
|
|
854
|
+
)
|
|
855
|
+
# The one collected server the selector names: a selector
|
|
856
|
+
# two components share has none, and no transport.
|
|
857
|
+
server = (
|
|
858
|
+
snapshot.resolved.get((str(kind), str(name), str(source)))
|
|
859
|
+
if mcp and snapshot is not None
|
|
860
|
+
else None
|
|
843
861
|
)
|
|
844
862
|
self.worker.wake()
|
|
845
863
|
if builtin or prompt:
|
|
846
864
|
return self._detection({**action, **call}, blocking, revision) or result
|
|
865
|
+
if mcp:
|
|
866
|
+
call = _occurrence_of(call)
|
|
867
|
+
transport = server.transport if server is not None else None
|
|
868
|
+
if result.outcome != "deny":
|
|
869
|
+
# Admission is decided; the arguments need no server.
|
|
870
|
+
remote = (
|
|
871
|
+
{"transport": transport} if declares_remote_transport(transport) else {}
|
|
872
|
+
)
|
|
873
|
+
judged = self._detection({**action, **call, **remote}, blocking, revision)
|
|
874
|
+
if judged is not None and judged.outcome == "deny":
|
|
875
|
+
return judged
|
|
876
|
+
call.pop("call", None)
|
|
847
877
|
if result.outcome == "deny":
|
|
848
878
|
# A decision from the snapshot's own table names one component;
|
|
849
879
|
# a category default names none.
|
stacktrace/daemon/protocol.py
CHANGED
|
@@ -6,16 +6,13 @@ import json
|
|
|
6
6
|
from dataclasses import asdict
|
|
7
7
|
from typing import Any
|
|
8
8
|
|
|
9
|
+
# The client, the server and the tests read all three from here.
|
|
10
|
+
from .frame_limits import MAX_CALL_FRAME_BYTES, MAX_FRAME_BYTES, VERSION # noqa: F401
|
|
9
11
|
from .store import Notification, SessionKey
|
|
10
12
|
|
|
11
|
-
VERSION = 1
|
|
12
|
-
MAX_FRAME_BYTES = 64 * 1024
|
|
13
13
|
# Status includes bounded project paths whose JSON Unicode escapes can exceed the usual frame limit.
|
|
14
14
|
MAX_POLICY_STATUS_FRAME_BYTES = 128 * 1024
|
|
15
15
|
MAX_POLICY_VIEW_FRAME_BYTES = 4 * 1024 * 1024
|
|
16
|
-
#: An `authorize` request that carries a built-in call's input (ADR-0158). Only
|
|
17
|
-
#: that request may be this large; every other incoming frame keeps the usual limit.
|
|
18
|
-
MAX_CALL_FRAME_BYTES = 2 * 1024 * 1024
|
|
19
16
|
|
|
20
17
|
#: How many entries any one list-shaped field puts on the wire -- BOM progress
|
|
21
18
|
#: lines carried back to an operator who asked for the upload, the agent kinds a
|
stacktrace/daemon/runtime.py
CHANGED
|
@@ -29,7 +29,7 @@ from stacktrace.telemetry import emit_error
|
|
|
29
29
|
from stacktrace.telemetry.posthog import flush as telemetry_flush
|
|
30
30
|
from stacktrace.webhook.config import get_config_path
|
|
31
31
|
|
|
32
|
-
from . import activity_log
|
|
32
|
+
from . import activity_log, unreadable
|
|
33
33
|
from .accounts import FleetOrg, SignedInAccounts
|
|
34
34
|
from .bom import BomJobReport, run_bom_job
|
|
35
35
|
from .client import MonitorOutputError, ProtocolError, relay_notifications
|
|
@@ -1051,7 +1051,8 @@ def _backfill(baseline: Baseline, root: Path, kinds: tuple[str, ...]) -> None:
|
|
|
1051
1051
|
return
|
|
1052
1052
|
logger.warning(
|
|
1053
1053
|
"cost baseline: %d transcript sources could not be read; the rules that need "
|
|
1054
|
-
"the baseline stay silent, and it is read again in an hour",
|
|
1054
|
+
"the baseline stay silent, and it is read again in an hour; %s",
|
|
1055
1055
|
len(baseline.unreadable),
|
|
1056
|
+
unreadable.describe(baseline.unreadable),
|
|
1056
1057
|
)
|
|
1057
1058
|
time.sleep(_BACKFILL_RETRY)
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Why a read could not reach every transcript source, for the daemon log.
|
|
2
|
+
|
|
3
|
+
`SessionView.unavailable` holds one `<kind>: <message>` entry for each
|
|
4
|
+
OpenAIDR `ReaderFailure`. The message is the reader's own diagnostic, almost
|
|
5
|
+
always `<path>: <why>`: a local file, then a schema version, a record type
|
|
6
|
+
it does not read, or an OS error. It is never a prompt, an argument or a
|
|
7
|
+
response. `stacktrace sessions` prints the same entries whole.
|
|
8
|
+
|
|
9
|
+
Every log line that says sources could not be read names them through
|
|
10
|
+
`describe`: one entry for each reason with its path removed, the most
|
|
11
|
+
common first, with up to three example paths. Strings are written with
|
|
12
|
+
`%r`, so a newline in a file name cannot forge a line, and the line is
|
|
13
|
+
bounded however many sources fail. These strings stay in the local log.
|
|
14
|
+
A detector run, and so Fleet, gets a kind and a count only
|
|
15
|
+
(`detector.run._collection_failures`).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import re
|
|
21
|
+
import time
|
|
22
|
+
from collections.abc import Callable, Iterable
|
|
23
|
+
|
|
24
|
+
#: The reasons one line names; the rest are counted.
|
|
25
|
+
MAX_REASONS = 10
|
|
26
|
+
#: The example paths named for each reason.
|
|
27
|
+
MAX_EXAMPLES = 3
|
|
28
|
+
#: The characters kept of one reason or one path.
|
|
29
|
+
MAX_TEXT = 200
|
|
30
|
+
#: How long the same set of failures stays out of a line logged on a timer.
|
|
31
|
+
REPEAT_AFTER = 3600.0
|
|
32
|
+
|
|
33
|
+
_WINDOWS_PATH = re.compile(r"[A-Za-z]:[\\/]")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def describe(unavailable: Iterable[str]) -> str:
|
|
37
|
+
"""The failures grouped by reason, e.g. `reasons: 622 x 'codex:
|
|
38
|
+
compressed rollout is not read' (e.g. '/a.jsonl.zst', ...)`."""
|
|
39
|
+
groups: dict[str, tuple[int, list[str]]] = {}
|
|
40
|
+
for entry in unavailable:
|
|
41
|
+
reason, path = _split(entry)
|
|
42
|
+
count, examples = groups.get(reason, (0, []))
|
|
43
|
+
if path is not None and path not in examples and len(examples) < MAX_EXAMPLES:
|
|
44
|
+
examples.append(path)
|
|
45
|
+
groups[reason] = (count + 1, examples)
|
|
46
|
+
if not groups:
|
|
47
|
+
return "reasons: none named"
|
|
48
|
+
ranked = sorted(groups.items(), key=lambda item: (-item[1][0], item[0]))
|
|
49
|
+
parts = []
|
|
50
|
+
for reason, (count, examples) in ranked[:MAX_REASONS]:
|
|
51
|
+
part = f"{count} x {_quote(reason)}"
|
|
52
|
+
if examples:
|
|
53
|
+
part += f" (e.g. {', '.join(_quote(path) for path in examples)})"
|
|
54
|
+
parts.append(part)
|
|
55
|
+
rest = ranked[MAX_REASONS:]
|
|
56
|
+
if rest:
|
|
57
|
+
parts.append(
|
|
58
|
+
f"and {len(rest)} more reasons, {sum(count for _, (count, _) in rest)} sources"
|
|
59
|
+
)
|
|
60
|
+
return "reasons: " + "; ".join(parts)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _split(entry: str) -> tuple[str, str | None]:
|
|
64
|
+
"""`<kind>: <path>: <why>` as `(<kind>: <why>, <path>)`. An entry whose
|
|
65
|
+
message does not start with a path is its own reason."""
|
|
66
|
+
kind, sep, message = entry.partition(": ")
|
|
67
|
+
head, mark, why = message.partition(": ")
|
|
68
|
+
if sep and mark and _is_path(head):
|
|
69
|
+
return f"{kind}: {why}", head
|
|
70
|
+
return entry, None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _is_path(text: str) -> bool:
|
|
74
|
+
return text.startswith(("/", "~", "\\\\")) or _WINDOWS_PATH.match(text) is not None
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _quote(text: str) -> str:
|
|
78
|
+
if len(text) > MAX_TEXT:
|
|
79
|
+
text = text[:MAX_TEXT] + "..."
|
|
80
|
+
return repr(text)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class OncePerChange:
|
|
84
|
+
"""Whether a line logged on every tick is due: when the set of failures
|
|
85
|
+
differs from the one last logged, or an hour after it was."""
|
|
86
|
+
|
|
87
|
+
def __init__(
|
|
88
|
+
self, every: float = REPEAT_AFTER, clock: Callable[[], float] = time.monotonic
|
|
89
|
+
) -> None:
|
|
90
|
+
self._every = every
|
|
91
|
+
self._clock = clock
|
|
92
|
+
self._last: frozenset[str] | None = None
|
|
93
|
+
self._at = 0.0
|
|
94
|
+
|
|
95
|
+
def due(self, unavailable: Iterable[str]) -> bool:
|
|
96
|
+
current = frozenset(unavailable)
|
|
97
|
+
now = self._clock()
|
|
98
|
+
if current == self._last and now - self._at < self._every:
|
|
99
|
+
return False
|
|
100
|
+
self._last, self._at = current, now
|
|
101
|
+
return True
|
stacktrace/daemon/usage.py
CHANGED
|
@@ -89,6 +89,7 @@ from stacktrace.sessions.access import collect_sessions
|
|
|
89
89
|
from stacktrace.sessions.protocols import SessionLike
|
|
90
90
|
|
|
91
91
|
from .store import FindingStore, PendingComponent, PendingFigures, UsagePage
|
|
92
|
+
from .unreadable import describe
|
|
92
93
|
|
|
93
94
|
logger = logging.getLogger(__name__)
|
|
94
95
|
|
|
@@ -445,6 +446,8 @@ class UsageBackfill:
|
|
|
445
446
|
#: Where an incomplete read reads from again, and not before when.
|
|
446
447
|
self._again: tuple[datetime, float] | None = None
|
|
447
448
|
self._reads = 0
|
|
449
|
+
#: What the last read could not read, named when the reads give up.
|
|
450
|
+
self._unreadable: tuple[str, ...] = ()
|
|
448
451
|
|
|
449
452
|
@property
|
|
450
453
|
def busy(self) -> bool:
|
|
@@ -516,6 +519,7 @@ class UsageBackfill:
|
|
|
516
519
|
def _run(self, since: datetime, now: datetime) -> None:
|
|
517
520
|
# A read that raises names no reader, so every kind it took missed.
|
|
518
521
|
missed = frozenset(self._to_read()[1])
|
|
522
|
+
self._unreadable = ()
|
|
519
523
|
try:
|
|
520
524
|
missed = self.run(since, now)
|
|
521
525
|
except Exception as error: # noqa: BLE001 - the backfill is never a reason to stop
|
|
@@ -526,7 +530,11 @@ class UsageBackfill:
|
|
|
526
530
|
if missed and self._reads < BACKFILL_READS:
|
|
527
531
|
self._again = (since, monotonic() + BACKFILL_RETRY)
|
|
528
532
|
elif missed:
|
|
529
|
-
logger.warning(
|
|
533
|
+
logger.warning(
|
|
534
|
+
"usage backfill: incomplete after %d reads; %s",
|
|
535
|
+
self._reads,
|
|
536
|
+
describe(self._unreadable),
|
|
537
|
+
)
|
|
530
538
|
# Given up: nothing before the end of the span it read can
|
|
531
539
|
# be vouched for in the kinds whose readers missed, so their
|
|
532
540
|
# coverage starts there (cost-notifications §2, "Like for
|
|
@@ -554,6 +562,7 @@ class UsageBackfill:
|
|
|
554
562
|
first upload's backfill done."""
|
|
555
563
|
uncovered, kinds = self._to_read()
|
|
556
564
|
view = self._collect(kinds, since, self.root)
|
|
565
|
+
self._unreadable = tuple(view.unavailable)
|
|
557
566
|
# A branch's copies name no run and count no component (`cost.copies`).
|
|
558
567
|
sessions = distinct(view.sessions)
|
|
559
568
|
names: dict[tuple[str, str], RunNames] = {}
|