agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Bounded, content-free facts from the collector currently being observed."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import Counter
|
|
6
|
+
from contextlib import contextmanager
|
|
7
|
+
from contextvars import ContextVar
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
import hashlib
|
|
10
|
+
from typing import Iterator
|
|
11
|
+
|
|
12
|
+
_KNOWN_CLIENTS = frozenset({
|
|
13
|
+
"cli", "sdk-py", "sdk-cli", "t3-code", "t3 code", "claude code", "codex",
|
|
14
|
+
"codex_cli_rs", "codex_vscode", "codex_exec", "codex-tui", "vscode", "exec", "mcp",
|
|
15
|
+
"automation", "rex", "super.engineering", "zeron", "paseo", "diri", "emdash",
|
|
16
|
+
"monocode", "synara", "bb", "orca", "traycer", "claude desktop", "capy",
|
|
17
|
+
"cloudroom", "superset", "paradev", "conductor", "codex desktop", "sdk-ts",
|
|
18
|
+
})
|
|
19
|
+
_MAX_CLIENTS = 32
|
|
20
|
+
_MAX_COUNT = 2**63 - 1
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class Diagnostics:
|
|
25
|
+
skipped: Counter[str] = field(default_factory=Counter)
|
|
26
|
+
clients: set[str] = field(default_factory=set)
|
|
27
|
+
client_overflow: bool = False
|
|
28
|
+
|
|
29
|
+
def snapshot(self) -> dict[str, object]:
|
|
30
|
+
return {"skipped": dict(self.skipped), "clients": sorted(self.clients),
|
|
31
|
+
"client_overflow": self.client_overflow}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
_current: ContextVar[Diagnostics | None] = ContextVar("aut_collector_diagnostics", default=None)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@contextmanager
|
|
38
|
+
def capture(observed: Diagnostics | None = None) -> Iterator[Diagnostics]:
|
|
39
|
+
observed = observed if observed is not None else Diagnostics()
|
|
40
|
+
token = _current.set(observed)
|
|
41
|
+
try:
|
|
42
|
+
yield observed
|
|
43
|
+
finally:
|
|
44
|
+
_current.reset(token)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def observe_client(value: object) -> None:
|
|
48
|
+
observed = _current.get()
|
|
49
|
+
if observed is None or not isinstance(value, str) or not value:
|
|
50
|
+
return
|
|
51
|
+
normalized = value.strip().casefold() if len(value) <= 1024 else ""
|
|
52
|
+
label = normalized if normalized in _KNOWN_CLIENTS else (
|
|
53
|
+
"unknown:" + hashlib.sha256(value.encode("utf-8", errors="replace")).hexdigest()[:16])
|
|
54
|
+
if label in observed.clients:
|
|
55
|
+
return
|
|
56
|
+
if len(observed.clients) >= _MAX_CLIENTS:
|
|
57
|
+
observed.client_overflow = True
|
|
58
|
+
return
|
|
59
|
+
observed.clients.add(label)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def note_skips(skipped: Counter[str]) -> None:
|
|
63
|
+
observed = _current.get()
|
|
64
|
+
if observed is None:
|
|
65
|
+
return
|
|
66
|
+
for key, count in skipped.items():
|
|
67
|
+
if type(count) is not int or count <= 0:
|
|
68
|
+
continue
|
|
69
|
+
category = ("malformed_records" if key == "malformed" else
|
|
70
|
+
"unreadable_files" if key == "unreadable" else
|
|
71
|
+
"missing_payloads" if key == "missing_payloads" else
|
|
72
|
+
"activity_errors" if key.startswith("activity:") else None)
|
|
73
|
+
if category:
|
|
74
|
+
observed.skipped[category] = min(_MAX_COUNT, observed.skipped[category] + count)
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""The episode rows the local pages read and the corrections they write."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from typing import Any, TypedDict
|
|
8
|
+
|
|
9
|
+
from .sink import connect, write_many
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class EpisodeMember(TypedDict):
|
|
13
|
+
session_id: str
|
|
14
|
+
role: str # seed, or a non-seed member
|
|
15
|
+
repo_keys: list[str]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Episode(TypedDict):
|
|
19
|
+
"""One agents.episode row and its members, timestamps in local time."""
|
|
20
|
+
id: str
|
|
21
|
+
first_ts: datetime
|
|
22
|
+
last_ts: datetime
|
|
23
|
+
n_sessions: int
|
|
24
|
+
n_seed_sessions: int
|
|
25
|
+
n_commits: int
|
|
26
|
+
n_prs: int
|
|
27
|
+
outcome: str
|
|
28
|
+
n_steers: int
|
|
29
|
+
outcome_source: str
|
|
30
|
+
confirmed: bool
|
|
31
|
+
members: list[EpisodeMember]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def load_month(dsn: str, start: datetime, end: datetime) -> list[Episode]:
|
|
35
|
+
with connect(dsn) as conn:
|
|
36
|
+
return load_episodes(conn, "e.first_ts >= %s AND e.first_ts < %s", (start, end))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def load_since(dsn: str, since: datetime) -> list[Episode]:
|
|
40
|
+
"""The episodes whose last activity is at or after `since`."""
|
|
41
|
+
with connect(dsn) as conn:
|
|
42
|
+
return load_episodes(conn, "e.last_ts >= %s", (since,))
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def load_episodes(conn: Any, where: str, params: tuple[Any, ...]) -> list[Episode]:
|
|
46
|
+
"""Episodes matching `where` (over agents.episode e), each with its members."""
|
|
47
|
+
rows = conn.execute(
|
|
48
|
+
"SELECT e.episode_id::text, e.first_ts, e.last_ts, e.n_sessions, e.n_seed_sessions, "
|
|
49
|
+
"e.n_commits, e.n_prs, e.outcome, m.session_id, m.role, m.repo_keys, e.n_steers, "
|
|
50
|
+
"e.outcome_source, e.confirmed "
|
|
51
|
+
"FROM agents.episode e JOIN agents.episode_member m USING (episode_id) "
|
|
52
|
+
f"WHERE {where} " # nosec B608 # every caller passes constant SQL with %s params
|
|
53
|
+
"ORDER BY e.first_ts, e.episode_id, m.role DESC, m.session_id", params).fetchall()
|
|
54
|
+
eps: dict[str, Episode] = {}
|
|
55
|
+
for eid, f, last, ns, nseed, nc, npr, out, sid, role, keys, nst, src, conf in rows:
|
|
56
|
+
ep = eps.setdefault(eid, {"id": eid, "first_ts": f.astimezone(), "last_ts":
|
|
57
|
+
last.astimezone(), "n_sessions": ns, "n_seed_sessions": nseed,
|
|
58
|
+
"n_commits": nc, "n_prs": npr, "outcome": out, "n_steers": nst,
|
|
59
|
+
"outcome_source": src, "confirmed": conf, "members": []})
|
|
60
|
+
ep["members"].append({"session_id": sid, "role": role, "repo_keys": list(keys or [])})
|
|
61
|
+
return list(eps.values())
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def answered_sessions(dsn: str, sessions: list[str]) -> set[str]:
|
|
65
|
+
"""Seed sessions that already have a confirm or reject correction (the episodes to hide)."""
|
|
66
|
+
with connect(dsn) as conn:
|
|
67
|
+
rows = conn.execute(
|
|
68
|
+
"SELECT DISTINCT a_session FROM agents.episode_correction "
|
|
69
|
+
"WHERE action IN ('confirm', 'reject') AND a_session = ANY(%s)",
|
|
70
|
+
(sessions,)).fetchall()
|
|
71
|
+
return {r[0] for r in rows}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def insert_corrections(dsn: str, rows: list[dict[str, Any]]) -> None:
|
|
75
|
+
write_many(dsn, "INSERT INTO agents.episode_correction (id, ts, action, a_session, b_session, "
|
|
76
|
+
"attrs) VALUES (%s, %s, %s, %s, %s, %s::jsonb)",
|
|
77
|
+
[(r["id"], r["ts"], r["action"], r["a_session"], r["b_session"],
|
|
78
|
+
json.dumps(r["attrs"], sort_keys=True)) for r in rows])
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""The rows collectors emit and the sink stores: usage, activity and active time.
|
|
2
|
+
|
|
3
|
+
The usage event is deliberately close to what the agents already log, but normalized so counts add up
|
|
4
|
+
uniformly across tools. Key normalization rule for cached input:
|
|
5
|
+
|
|
6
|
+
input_tokens = FRESH (non-cached) prompt tokens
|
|
7
|
+
cache_read_tokens = tokens served from prompt cache
|
|
8
|
+
cache_write_tokens= tokens written into the prompt cache
|
|
9
|
+
|
|
10
|
+
Claude Code and Pi already separate these. Codex reports input_tokens INCLUSIVE of
|
|
11
|
+
cached, so its collector subtracts cached to keep the same meaning everywhere.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import hashlib
|
|
17
|
+
from dataclasses import asdict, dataclass, field
|
|
18
|
+
from datetime import datetime
|
|
19
|
+
from typing import Any, Callable, Literal, Sequence
|
|
20
|
+
|
|
21
|
+
RunOrigin = Literal["interactive", "automated", "subagent", "unknown"]
|
|
22
|
+
UsageKind = Literal["agent", "human"] # agent: an assistant turn with tokens; human: a user prompt
|
|
23
|
+
TimeSource = Literal["aw", "manual"]
|
|
24
|
+
ActivityKind = Literal["steer", "interrupt", "rejection", "commit", "pr", "turn", "compaction",
|
|
25
|
+
"invoke", "link", "label", "eval", "steer_estimate"]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def parse_ts(ts: object) -> datetime | None:
|
|
29
|
+
"""The datetime of a non-empty ISO-8601 string the sink can cast to timestamptz, else None:
|
|
30
|
+
None, "", numbers and malformed strings, so a bad ts can't wedge the batch."""
|
|
31
|
+
if not isinstance(ts, str) or not ts.strip():
|
|
32
|
+
return None
|
|
33
|
+
try:
|
|
34
|
+
return datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
|
35
|
+
except ValueError:
|
|
36
|
+
return None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def valid_ts(ts: object) -> bool:
|
|
40
|
+
return parse_ts(ts) is not None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def make_event_id(agent: str, *parts: object) -> str:
|
|
44
|
+
"""Deterministic idempotency key. Same source turn -> same id, so re-scans dedup
|
|
45
|
+
at the sink via ON CONFLICT. Never include a wall-clock 'now' here."""
|
|
46
|
+
raw = "|".join([agent, *(str(p) for p in parts)])
|
|
47
|
+
return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:32]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(slots=True)
|
|
51
|
+
class EventSession:
|
|
52
|
+
"""Which session an event belongs to (one agents.event_session row). event_id, machine and
|
|
53
|
+
agent come from the owning UsageEvent, because the runner stamps machine after collect()."""
|
|
54
|
+
session_id: str
|
|
55
|
+
parent_session_id: str | None # the spawning session, subagents only
|
|
56
|
+
run_origin: RunOrigin
|
|
57
|
+
turn_id: str | None # the user turn (promptId) this event answers
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(slots=True)
|
|
61
|
+
class UsageEvent:
|
|
62
|
+
event_id: str
|
|
63
|
+
ts: str | None # ISO-8601 from the source log line
|
|
64
|
+
agent: str # claude_code | codex | cursor | cursor_cli | pi | ...
|
|
65
|
+
model: str | None
|
|
66
|
+
provider: str | None
|
|
67
|
+
kind: UsageKind = "agent"
|
|
68
|
+
input_tokens: int = 0
|
|
69
|
+
output_tokens: int = 0
|
|
70
|
+
cache_read_tokens: int = 0
|
|
71
|
+
cache_write_tokens: int = 0
|
|
72
|
+
reasoning_tokens: int | None = None # codex and pi (a subset of output); None elsewhere
|
|
73
|
+
total_tokens: int = 0
|
|
74
|
+
est_cost_usd: float | None = None # NOTIONAL / API-equivalent; subs are flat-rate
|
|
75
|
+
project: str | None = None # working-dir basename, never a full path
|
|
76
|
+
machine: str | None = field(default=None) # stamped by the runner, not the collector
|
|
77
|
+
session: EventSession | None = None # claude_code, codex, pi, cursor_cli; never in to_dict()
|
|
78
|
+
|
|
79
|
+
def to_dict(self) -> dict[str, Any]:
|
|
80
|
+
d = asdict(self)
|
|
81
|
+
del d["session"] # session rows go out separately, never in the usage payload
|
|
82
|
+
return d
|
|
83
|
+
|
|
84
|
+
def session_row(self) -> dict[str, Any] | None:
|
|
85
|
+
if self.session is None:
|
|
86
|
+
return None
|
|
87
|
+
return {"event_id": self.event_id, "machine": self.machine, "agent": self.agent,
|
|
88
|
+
**asdict(self.session)}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclass(slots=True)
|
|
92
|
+
class ActivityEvent:
|
|
93
|
+
"""One agents.activity_event row: a non-token thing that happened (a steer, a commit, a turn).
|
|
94
|
+
attrs holds counts, flags, ids and hashes only: never prompt text, output, paths or titles."""
|
|
95
|
+
event_id: str
|
|
96
|
+
ts: str | None
|
|
97
|
+
agent: str
|
|
98
|
+
kind: ActivityKind
|
|
99
|
+
session_id: str | None
|
|
100
|
+
turn_id: str | None
|
|
101
|
+
attrs: dict[str, Any]
|
|
102
|
+
machine: str | None = None # stamped by the runner, like UsageEvent.machine
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@dataclass(slots=True)
|
|
106
|
+
class TimeInterval:
|
|
107
|
+
"""One agents.time_interval row: a span a person was at the keyboard (source 'aw') or said
|
|
108
|
+
they worked (source 'manual'). title is private, already redacted and cut on this machine."""
|
|
109
|
+
event_id: str
|
|
110
|
+
ts: str # start, ISO-8601
|
|
111
|
+
end_ts: str
|
|
112
|
+
source: TimeSource
|
|
113
|
+
app: str | None
|
|
114
|
+
title: str | None
|
|
115
|
+
category: str | None
|
|
116
|
+
work_relevant: bool
|
|
117
|
+
attrs: dict[str, Any]
|
|
118
|
+
machine: str | None = None # stamped by the runner (aw) or `aut time add` (manual)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# What the sender takes and collectors yield: one source unit's rows, mixed, and the callback that
|
|
122
|
+
# advances its watermark once they are delivered. Sequence, so a collector that emits one row type
|
|
123
|
+
# can say which.
|
|
124
|
+
Row = UsageEvent | ActivityEvent | TimeInterval
|
|
125
|
+
Batch = tuple[Sequence[Row], Callable[[], None]]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class CollectorUnavailable(Exception):
|
|
129
|
+
"""A collector could not read a source that is there (an expired token, a stopped server, a
|
|
130
|
+
locked store). Raised after the batches it could read, with a one-line message for local stderr;
|
|
131
|
+
the scan records status 'degraded' and carries on. Chain the cause: its type is what the
|
|
132
|
+
heartbeat row stores."""
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""Keep verified commits when a machine is offline, using private salted identities."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from datetime import date, datetime, timedelta
|
|
8
|
+
import json
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
import re
|
|
11
|
+
from typing import Literal, TypedDict, cast
|
|
12
|
+
|
|
13
|
+
Status = Literal['complete', 'partial', 'failed', 'deferred']
|
|
14
|
+
STATUSES = {'complete', 'partial', 'failed', 'deferred'}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class SourceState(TypedDict):
|
|
18
|
+
complete_from: str | None
|
|
19
|
+
complete_until: str | None
|
|
20
|
+
stores: list[int]
|
|
21
|
+
status: str
|
|
22
|
+
attempted_at: str
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Cache(TypedDict):
|
|
26
|
+
version: int
|
|
27
|
+
key_id: str
|
|
28
|
+
commits: dict[str, str]
|
|
29
|
+
sources: dict[str, SourceState]
|
|
30
|
+
baseline_sources: list[str]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class Snapshot:
|
|
35
|
+
source_id: str
|
|
36
|
+
status: Status
|
|
37
|
+
commits: dict[str, date]
|
|
38
|
+
stores: int
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _key(value: object) -> bool:
|
|
42
|
+
return isinstance(value, str) and re.fullmatch(r'[0-9a-f]{64}', value) is not None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def empty_cache(key_id: str) -> Cache:
|
|
46
|
+
if not _key(key_id):
|
|
47
|
+
raise ValueError('Invalid Git cache identity')
|
|
48
|
+
return {'version': 1, 'key_id': key_id, 'commits': {}, 'sources': {}, 'baseline_sources': []}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _require(valid: bool) -> None:
|
|
52
|
+
if not valid:
|
|
53
|
+
raise ValueError('Invalid Git cache structure')
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def load_cache(path: Path, key_id: str, *, reset: bool = False) -> Cache:
|
|
57
|
+
if reset or not path.exists():
|
|
58
|
+
return empty_cache(key_id)
|
|
59
|
+
try:
|
|
60
|
+
raw = json.loads(path.read_text())
|
|
61
|
+
_require(isinstance(raw, dict) and type(raw['version']) is int
|
|
62
|
+
and raw['version'] == 1 and raw['key_id'] == key_id)
|
|
63
|
+
_require(isinstance(raw['commits'], dict) and isinstance(raw['sources'], dict))
|
|
64
|
+
_require(isinstance(raw['baseline_sources'], list)
|
|
65
|
+
and all(_key(value) for value in raw['baseline_sources'])
|
|
66
|
+
and len(set(raw['baseline_sources'])) == len(raw['baseline_sources']))
|
|
67
|
+
for key, value in raw['commits'].items():
|
|
68
|
+
_require(_key(key) and isinstance(value, str))
|
|
69
|
+
date.fromisoformat(value)
|
|
70
|
+
for key, source in raw['sources'].items():
|
|
71
|
+
_require(_key(key) and isinstance(source, dict) and source['status'] in STATUSES)
|
|
72
|
+
_require(isinstance(source['stores'], list) and len(source['stores']) <= 5)
|
|
73
|
+
_require(all(type(n) is int and n > 0 for n in source['stores']))
|
|
74
|
+
first, last = source['complete_from'], source['complete_until']
|
|
75
|
+
_require((first is None) == (last is None))
|
|
76
|
+
if first is not None:
|
|
77
|
+
_require(date.fromisoformat(first) <= date.fromisoformat(last))
|
|
78
|
+
_require(datetime.fromisoformat(source['attempted_at']).utcoffset() is not None)
|
|
79
|
+
except (OSError, ValueError, KeyError, TypeError) as exc:
|
|
80
|
+
raise ValueError('Git identity cache is unreadable, corrupt, or uses another salt/policy') from exc
|
|
81
|
+
return cast(Cache, raw)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def merge_sources(cache: Cache, snapshots: list[Snapshot], since: date, until: date,
|
|
85
|
+
now: datetime, *, reset: bool = False) -> tuple[Cache, dict[date, bool]]:
|
|
86
|
+
if not snapshots or len({s.source_id for s in snapshots}) != len(snapshots):
|
|
87
|
+
raise ValueError('Git sources must be nonempty and unique')
|
|
88
|
+
if now.utcoffset() is None:
|
|
89
|
+
raise ValueError('Git source observations need an aware timestamp')
|
|
90
|
+
if reset and any(s.status != 'complete' for s in snapshots):
|
|
91
|
+
raise ValueError('A Git identity reset needs every source complete')
|
|
92
|
+
updated = empty_cache(cache['key_id']) if reset else copy.deepcopy(cache)
|
|
93
|
+
for snapshot in snapshots:
|
|
94
|
+
if (not _key(snapshot.source_id) or snapshot.status not in STATUSES
|
|
95
|
+
or type(snapshot.stores) is not int or snapshot.stores < 0
|
|
96
|
+
or (snapshot.status == 'complete' and not snapshot.stores)
|
|
97
|
+
or (snapshot.status in ('failed', 'deferred') and snapshot.commits)):
|
|
98
|
+
raise ValueError('Invalid Git source observation')
|
|
99
|
+
source = updated['sources'].setdefault(snapshot.source_id, {
|
|
100
|
+
'complete_from': None, 'complete_until': None, 'stores': [],
|
|
101
|
+
'status': snapshot.status, 'attempted_at': now.isoformat()})
|
|
102
|
+
source['status'], source['attempted_at'] = snapshot.status, now.isoformat()
|
|
103
|
+
for key, day in snapshot.commits.items():
|
|
104
|
+
if not _key(key) or type(day) is not date:
|
|
105
|
+
raise ValueError('Invalid salted Git commit observation')
|
|
106
|
+
if not since <= day <= until:
|
|
107
|
+
continue
|
|
108
|
+
previous = updated['commits'].get(key)
|
|
109
|
+
if previous is not None and previous != day.isoformat():
|
|
110
|
+
raise ValueError('One Git commit has conflicting author days')
|
|
111
|
+
updated['commits'][key] = day.isoformat()
|
|
112
|
+
if snapshot.status == 'complete':
|
|
113
|
+
source['stores'] = source['stores'][-4:] + [snapshot.stores]
|
|
114
|
+
if since <= until:
|
|
115
|
+
first, last = source['complete_from'], source['complete_until']
|
|
116
|
+
if (first is None or last is None
|
|
117
|
+
or since > date.fromisoformat(last) + timedelta(days=1)
|
|
118
|
+
or until < date.fromisoformat(first) - timedelta(days=1)):
|
|
119
|
+
source['complete_from'], source['complete_until'] = since.isoformat(), until.isoformat()
|
|
120
|
+
else:
|
|
121
|
+
source['complete_from'] = min(first, since.isoformat())
|
|
122
|
+
source['complete_until'] = max(last, until.isoformat())
|
|
123
|
+
days = [since + timedelta(days=i) for i in range(max(0, (until - since).days + 1))]
|
|
124
|
+
complete = {day: all(
|
|
125
|
+
(source := updated['sources'][snapshot.source_id])['complete_from'] is not None
|
|
126
|
+
and str(source['complete_from']) <= day.isoformat() <= str(source['complete_until'])
|
|
127
|
+
for snapshot in snapshots) for day in days}
|
|
128
|
+
return updated, complete
|