agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
"""Private, content-free external observations. Unknown counts remain None."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import re
|
|
10
|
+
import sqlite3
|
|
11
|
+
from datetime import datetime
|
|
12
|
+
from typing import Any, Iterable
|
|
13
|
+
|
|
14
|
+
MAX_FILE_BYTES = 8 * 1024 * 1024
|
|
15
|
+
MAX_RECORD_BYTES = 4096
|
|
16
|
+
MAX_BATCH_RECORDS = 10000
|
|
17
|
+
MAX_STORE_RECORDS = 1000000
|
|
18
|
+
COUNTS = (
|
|
19
|
+
"input_tokens",
|
|
20
|
+
"input_tokens_total",
|
|
21
|
+
"output_tokens",
|
|
22
|
+
"reasoning_tokens",
|
|
23
|
+
"cache_read_tokens",
|
|
24
|
+
"cache_write_tokens",
|
|
25
|
+
)
|
|
26
|
+
COMMON = {"version", "id", "kind", "timestamp"}
|
|
27
|
+
FIELDS = {
|
|
28
|
+
"usage": set(COUNTS) | {"start", "temporality", "counter"},
|
|
29
|
+
"quota": {"remaining", "limit", "unit", "period", "reset", "fresh_until", "scope"},
|
|
30
|
+
"tool": {"status", "duration_ms"},
|
|
31
|
+
}
|
|
32
|
+
IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:/-]{0,127}\Z")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def timestamp(value: Any) -> datetime:
|
|
36
|
+
if not isinstance(value, str) or "T" not in value:
|
|
37
|
+
raise ValueError("timestamp must be an ISO timezone timestamp")
|
|
38
|
+
try:
|
|
39
|
+
result = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
40
|
+
except ValueError:
|
|
41
|
+
raise ValueError("invalid timestamp") from None
|
|
42
|
+
if result.tzinfo is None:
|
|
43
|
+
raise ValueError("timestamp requires timezone")
|
|
44
|
+
return result
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def identifier(value: Any) -> str:
|
|
48
|
+
if not isinstance(value, str) or not IDENTIFIER.fullmatch(value):
|
|
49
|
+
raise ValueError("invalid identifier")
|
|
50
|
+
return value
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _components_valid(r: dict[str, Any]) -> bool:
|
|
54
|
+
total = r["input_tokens_total"]
|
|
55
|
+
known_input = sum(r[key] for key in ("input_tokens", "cache_read_tokens") if r[key] is not None)
|
|
56
|
+
if r["input_tokens"] is not None and r["cache_write_tokens"] is not None:
|
|
57
|
+
known_input += r["cache_write_tokens"]
|
|
58
|
+
if total is not None and known_input > total:
|
|
59
|
+
return False
|
|
60
|
+
if total is not None and all(
|
|
61
|
+
r[key] is not None for key in ("input_tokens", "cache_read_tokens", "cache_write_tokens")
|
|
62
|
+
) and known_input != total:
|
|
63
|
+
return False
|
|
64
|
+
return not (
|
|
65
|
+
r["reasoning_tokens"] is not None
|
|
66
|
+
and r["output_tokens"] is not None
|
|
67
|
+
and r["reasoning_tokens"] > r["output_tokens"]
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def validate_record(record: Any) -> dict[str, Any]:
|
|
72
|
+
"""Validate schema v1. Reject text and unknown fields at this trust boundary."""
|
|
73
|
+
if not isinstance(record, dict):
|
|
74
|
+
raise ValueError("record must be object")
|
|
75
|
+
r = dict(record)
|
|
76
|
+
kind = r.get("kind")
|
|
77
|
+
if (
|
|
78
|
+
not isinstance(kind, str)
|
|
79
|
+
or kind not in FIELDS
|
|
80
|
+
or type(r.get("version")) is not int
|
|
81
|
+
or r["version"] != 1
|
|
82
|
+
):
|
|
83
|
+
raise ValueError("unknown schema version or kind")
|
|
84
|
+
if set(r) - COMMON - FIELDS[kind] or not COMMON <= set(r):
|
|
85
|
+
raise ValueError("missing or unknown fields")
|
|
86
|
+
identifier(r["id"])
|
|
87
|
+
timestamp(r["timestamp"])
|
|
88
|
+
for key in ("start", "reset", "fresh_until"):
|
|
89
|
+
if r.get(key) is not None:
|
|
90
|
+
timestamp(r[key])
|
|
91
|
+
for key in (*COUNTS, "remaining", "limit", "duration_ms"):
|
|
92
|
+
if key in r and r[key] is not None:
|
|
93
|
+
if type(r[key]) is not int or not 0 <= r[key] <= 2**63 - 1:
|
|
94
|
+
raise ValueError("counts must be nonnegative int64 or null")
|
|
95
|
+
if kind == "usage":
|
|
96
|
+
for key in COUNTS:
|
|
97
|
+
r.setdefault(key, None)
|
|
98
|
+
if not any(r[k] is not None for k in COUNTS):
|
|
99
|
+
raise ValueError("usage needs a known count")
|
|
100
|
+
if r.get("temporality", "delta") not in ("delta", "cumulative"):
|
|
101
|
+
raise ValueError("invalid temporality")
|
|
102
|
+
if ("start" in r or "temporality" in r) and "counter" not in r:
|
|
103
|
+
raise ValueError("counter windows require a series identifier")
|
|
104
|
+
if "counter" in r:
|
|
105
|
+
identifier(r["counter"])
|
|
106
|
+
if not _components_valid(r):
|
|
107
|
+
raise ValueError("input components disagree with total or reasoning subset exceeds total")
|
|
108
|
+
if r.get("start") is not None and timestamp(r["start"]) >= timestamp(r["timestamp"]):
|
|
109
|
+
raise ValueError("invalid usage window")
|
|
110
|
+
if kind == "quota":
|
|
111
|
+
required = {"remaining", "limit", "unit", "period", "reset", "fresh_until", "scope"}
|
|
112
|
+
if not required <= set(r):
|
|
113
|
+
raise ValueError("quota needs explicit values, including null unknowns")
|
|
114
|
+
if r["unit"] not in ("tokens", "requests", "percent", "credits", "unknown"):
|
|
115
|
+
raise ValueError("invalid quota unit")
|
|
116
|
+
if r["period"] not in ("minute", "hour", "day", "week", "month", "session", "unknown"):
|
|
117
|
+
raise ValueError("invalid quota period")
|
|
118
|
+
identifier(r["scope"])
|
|
119
|
+
if kind == "tool" and r.get("status") not in ("ok", "error", "unknown"):
|
|
120
|
+
raise ValueError("invalid tool status")
|
|
121
|
+
if len(json.dumps(r).encode()) > MAX_RECORD_BYTES:
|
|
122
|
+
raise ValueError("record too large")
|
|
123
|
+
return r
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _connect(store: str | Path) -> sqlite3.Connection:
|
|
127
|
+
path = Path(store)
|
|
128
|
+
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
129
|
+
fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600)
|
|
130
|
+
os.close(fd)
|
|
131
|
+
os.chmod(path, 0o600)
|
|
132
|
+
db = sqlite3.connect(path)
|
|
133
|
+
db.execute(
|
|
134
|
+
"CREATE TABLE IF NOT EXISTS observations "
|
|
135
|
+
"(source TEXT, stream TEXT, role TEXT, id TEXT, body TEXT, "
|
|
136
|
+
"PRIMARY KEY(source, stream, id))"
|
|
137
|
+
)
|
|
138
|
+
db.execute(
|
|
139
|
+
"CREATE TABLE IF NOT EXISTS counters "
|
|
140
|
+
"(source TEXT, stream TEXT, counter TEXT, timestamp TEXT, temporality TEXT, "
|
|
141
|
+
"PRIMARY KEY(source, stream, counter))"
|
|
142
|
+
)
|
|
143
|
+
db.execute(
|
|
144
|
+
"CREATE TABLE IF NOT EXISTS streams "
|
|
145
|
+
"(source TEXT, stream TEXT, role TEXT, representation TEXT, "
|
|
146
|
+
"PRIMARY KEY(source, stream))"
|
|
147
|
+
)
|
|
148
|
+
db.execute(
|
|
149
|
+
"CREATE TABLE IF NOT EXISTS tool_disputes "
|
|
150
|
+
"(source TEXT, stream TEXT, id TEXT, body TEXT, "
|
|
151
|
+
"PRIMARY KEY(source, stream, id))"
|
|
152
|
+
)
|
|
153
|
+
return db
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def ingest(
|
|
157
|
+
store: str | Path, records: Iterable[dict[str, Any]], *, source: str, stream: str, role: str
|
|
158
|
+
) -> dict[str, Any]:
|
|
159
|
+
identifier(source)
|
|
160
|
+
identifier(stream)
|
|
161
|
+
if role not in ("primary", "crosscheck"):
|
|
162
|
+
raise ValueError("invalid role")
|
|
163
|
+
batch: list[dict[str, Any]] = []
|
|
164
|
+
for raw in records:
|
|
165
|
+
if len(batch) >= MAX_BATCH_RECORDS:
|
|
166
|
+
raise ValueError("batch too large")
|
|
167
|
+
batch.append(validate_record(raw))
|
|
168
|
+
result = dict(accepted=0, duplicates=0, conflicts=0, excluded=0)
|
|
169
|
+
db = _connect(store)
|
|
170
|
+
try:
|
|
171
|
+
with db:
|
|
172
|
+
db.execute("BEGIN IMMEDIATE")
|
|
173
|
+
size = db.execute("SELECT count(*) FROM observations").fetchone()[0]
|
|
174
|
+
for r in batch:
|
|
175
|
+
mode = ("counter" if "counter" in r else "event") if r["kind"] == "usage" else None
|
|
176
|
+
ownership = db.execute(
|
|
177
|
+
"SELECT role, representation FROM streams WHERE source=? AND stream=?",
|
|
178
|
+
(source, stream),
|
|
179
|
+
).fetchone()
|
|
180
|
+
if ownership:
|
|
181
|
+
if ownership[0] != role:
|
|
182
|
+
raise ValueError("stream role changed; use a separate crosscheck stream")
|
|
183
|
+
body = json.dumps(r, sort_keys=True, separators=(",", ":"))
|
|
184
|
+
old = db.execute(
|
|
185
|
+
"SELECT body, role FROM observations WHERE source=? AND stream=? AND id=?",
|
|
186
|
+
(source, stream, r["id"]),
|
|
187
|
+
).fetchone()
|
|
188
|
+
if old:
|
|
189
|
+
disputed = db.execute(
|
|
190
|
+
"SELECT 1 FROM tool_disputes WHERE source=? AND stream=? AND id=?",
|
|
191
|
+
(source, stream, r["id"]),
|
|
192
|
+
).fetchone()
|
|
193
|
+
conflict = old != (body, role) or bool(disputed)
|
|
194
|
+
if conflict and (r["kind"] == "tool" or json.loads(old[0])["kind"] == "tool"):
|
|
195
|
+
# Retain conflicting evidence without rewriting the original observation.
|
|
196
|
+
db.execute(
|
|
197
|
+
"INSERT OR IGNORE INTO tool_disputes VALUES (?,?,?,?)",
|
|
198
|
+
(source, stream, r["id"], body),
|
|
199
|
+
)
|
|
200
|
+
result["conflicts" if conflict else "duplicates"] += 1
|
|
201
|
+
continue
|
|
202
|
+
if ownership:
|
|
203
|
+
if mode is not None and ownership[1] is not None and mode != ownership[1]:
|
|
204
|
+
raise ValueError("incompatible usage representation")
|
|
205
|
+
mode = mode or ownership[1]
|
|
206
|
+
db.execute(
|
|
207
|
+
"INSERT OR REPLACE INTO streams VALUES (?,?,?,?)", (source, stream, role, mode)
|
|
208
|
+
)
|
|
209
|
+
if size >= MAX_STORE_RECORDS:
|
|
210
|
+
raise ValueError("store limit reached")
|
|
211
|
+
# Primary ownership is source-wide: alternate streams must be crosschecks.
|
|
212
|
+
owners = db.execute(
|
|
213
|
+
"SELECT DISTINCT stream FROM observations WHERE source=? AND role='primary'",
|
|
214
|
+
(source,),
|
|
215
|
+
).fetchall()
|
|
216
|
+
if role == "primary" and owners and (stream,) not in owners:
|
|
217
|
+
raise ValueError("overlapping primary streams; use crosscheck")
|
|
218
|
+
if r["kind"] == "usage" and "counter" in r:
|
|
219
|
+
identity = (source, stream, r["counter"])
|
|
220
|
+
last = db.execute(
|
|
221
|
+
"SELECT timestamp, temporality FROM counters WHERE source=? AND stream=? AND counter=?",
|
|
222
|
+
identity,
|
|
223
|
+
).fetchone()
|
|
224
|
+
if last and last[1] != r.get("temporality", "delta"):
|
|
225
|
+
raise ValueError("counter temporality changed")
|
|
226
|
+
if last and timestamp(r["timestamp"]) <= timestamp(last[0]):
|
|
227
|
+
raise ValueError("out-of-order counter observation")
|
|
228
|
+
db.execute(
|
|
229
|
+
"INSERT OR REPLACE INTO counters VALUES (?,?,?,?,?)",
|
|
230
|
+
(*identity, r["timestamp"], r.get("temporality", "delta")),
|
|
231
|
+
)
|
|
232
|
+
db.execute(
|
|
233
|
+
"INSERT INTO observations VALUES (?,?,?,?,?)",
|
|
234
|
+
(source, stream, role, r["id"], body),
|
|
235
|
+
)
|
|
236
|
+
size += 1
|
|
237
|
+
result["accepted"] += 1
|
|
238
|
+
result["excluded"] += role == "crosscheck"
|
|
239
|
+
finally:
|
|
240
|
+
db.close()
|
|
241
|
+
return result
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def disputed_tool_ids(store: str | Path, *, source: str, stream: str) -> set[str]:
|
|
245
|
+
"""Return quarantined identities without loading unrelated observations."""
|
|
246
|
+
db = _connect(store)
|
|
247
|
+
try:
|
|
248
|
+
return {row[0] for row in db.execute(
|
|
249
|
+
"SELECT DISTINCT id FROM tool_disputes WHERE source=? AND stream=?", (source, stream)
|
|
250
|
+
)}
|
|
251
|
+
finally:
|
|
252
|
+
db.close()
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def records(store: str | Path) -> list[dict[str, Any]]:
|
|
256
|
+
"""Return normalized evidence; only additive=True observations can feed totals."""
|
|
257
|
+
db = _connect(store)
|
|
258
|
+
try:
|
|
259
|
+
rows = db.execute("SELECT source, stream, role, body FROM observations").fetchall()
|
|
260
|
+
disputed = set(db.execute("SELECT source, stream, id FROM tool_disputes").fetchall())
|
|
261
|
+
finally:
|
|
262
|
+
db.close()
|
|
263
|
+
result = []
|
|
264
|
+
for source, stream, role, body in rows:
|
|
265
|
+
r = json.loads(body)
|
|
266
|
+
r.update(source=source, stream=stream, role=role)
|
|
267
|
+
if r["kind"] == "tool" and (source, stream, r["id"]) in disputed:
|
|
268
|
+
r["status"] = "unknown"
|
|
269
|
+
r["disputed"] = True
|
|
270
|
+
r["additive"] = (
|
|
271
|
+
role == "primary"
|
|
272
|
+
and r["kind"] == "usage"
|
|
273
|
+
and (r.get("temporality", "delta") == "delta")
|
|
274
|
+
)
|
|
275
|
+
result.append(r)
|
|
276
|
+
# Windows are checked across all stored evidence, including late arrivals.
|
|
277
|
+
windows: dict[tuple[str, str, str], list[dict[str, Any]]] = {}
|
|
278
|
+
for r in result:
|
|
279
|
+
if r["kind"] == "usage" and "counter" in r:
|
|
280
|
+
windows.setdefault((r["source"], r["stream"], r["counter"]), []).append(r)
|
|
281
|
+
for group in windows.values():
|
|
282
|
+
group.sort(key=lambda r: timestamp(r["timestamp"]))
|
|
283
|
+
previous = None
|
|
284
|
+
for r in group:
|
|
285
|
+
start = r.get("start")
|
|
286
|
+
if r.get("temporality") == "cumulative":
|
|
287
|
+
r["additive"] = False
|
|
288
|
+
r["coverage"] = "unknown_first_point"
|
|
289
|
+
r["delta"] = None
|
|
290
|
+
if start is not None:
|
|
291
|
+
same_start = previous is not None and previous.get("start") == start
|
|
292
|
+
baseline: dict[str, Any] = (
|
|
293
|
+
previous
|
|
294
|
+
if same_start and previous is not None
|
|
295
|
+
else dict.fromkeys(COUNTS, 0)
|
|
296
|
+
)
|
|
297
|
+
delta = {
|
|
298
|
+
key: (
|
|
299
|
+
r[key] - baseline[key]
|
|
300
|
+
if r[key] is not None and baseline[key] is not None
|
|
301
|
+
else None
|
|
302
|
+
)
|
|
303
|
+
for key in COUNTS
|
|
304
|
+
}
|
|
305
|
+
if any(value is not None and value < 0 for value in delta.values()):
|
|
306
|
+
r["coverage"] = "counter_decreased"
|
|
307
|
+
elif (
|
|
308
|
+
previous is not None
|
|
309
|
+
and not same_start
|
|
310
|
+
and (timestamp(start) < timestamp(previous["timestamp"]))
|
|
311
|
+
):
|
|
312
|
+
r["coverage"] = "overlapping_reset"
|
|
313
|
+
elif not _components_valid(delta):
|
|
314
|
+
r["coverage"] = "invalid_components"
|
|
315
|
+
else:
|
|
316
|
+
r["delta"] = delta
|
|
317
|
+
r["additive"] = r["role"] == "primary"
|
|
318
|
+
r["coverage"] = "delta" if same_start else "known_start_reset"
|
|
319
|
+
elif previous is not None:
|
|
320
|
+
r["coverage"] = "unknown_start"
|
|
321
|
+
elif start is None or (
|
|
322
|
+
previous is not None and timestamp(start) < timestamp(previous["timestamp"])
|
|
323
|
+
):
|
|
324
|
+
r["additive"] = False
|
|
325
|
+
r["coverage"] = "unknown_or_overlapping_window"
|
|
326
|
+
if previous is not None:
|
|
327
|
+
previous["additive"] = False
|
|
328
|
+
if r.get("coverage") not in ("counter_decreased", "invalid_components"):
|
|
329
|
+
previous = r
|
|
330
|
+
latest: dict[tuple[str, str, str], dict[str, Any]] = {}
|
|
331
|
+
for r in result:
|
|
332
|
+
if r["kind"] == "quota":
|
|
333
|
+
key = (r["source"], r["stream"], r["scope"])
|
|
334
|
+
if key not in latest or timestamp(r["timestamp"]) > timestamp(latest[key]["timestamp"]):
|
|
335
|
+
latest[key] = r
|
|
336
|
+
for r in result:
|
|
337
|
+
if r["kind"] == "quota":
|
|
338
|
+
r["latest"] = latest[(r["source"], r["stream"], r["scope"])] is r
|
|
339
|
+
return sorted(result, key=lambda r: (r["timestamp"], r["id"]))
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def stable_id(value: Any) -> str:
|
|
343
|
+
return hashlib.sha256(
|
|
344
|
+
json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
|
|
345
|
+
).hexdigest()
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def import_file(
|
|
349
|
+
store: str | Path, path: str | Path, *, format: str, source: str, stream: str, role: str
|
|
350
|
+
) -> dict[str, Any]:
|
|
351
|
+
with Path(path).open("rb") as handle:
|
|
352
|
+
data = handle.read(MAX_FILE_BYTES + 1)
|
|
353
|
+
if len(data) > MAX_FILE_BYTES:
|
|
354
|
+
raise ValueError("file too large")
|
|
355
|
+
if format == "jsonl":
|
|
356
|
+
parsed = [json.loads(line) for line in data.splitlines() if line.strip()]
|
|
357
|
+
return ingest(store, parsed, source=source, stream=stream, role=role)
|
|
358
|
+
if format == "otlp":
|
|
359
|
+
from .telemetry import normalize_otlp
|
|
360
|
+
|
|
361
|
+
normalized, unsupported = normalize_otlp(json.loads(data))
|
|
362
|
+
result = ingest(store, normalized, source=source, stream=stream, role=role)
|
|
363
|
+
result["unsupported"] = unsupported
|
|
364
|
+
return result
|
|
365
|
+
raise ValueError("unsupported format")
|