agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,149 @@
1
+ -- Grants for agents.usage_event (companion to 01_schema.sql). Idempotent and
2
+ -- role-tolerant: every grant is guarded by an EXISTS check so a fresh database
3
+ -- (where the roles may not exist yet) does not abort under ON_ERROR_STOP.
4
+ --
5
+ -- Roles (create separately with their own passwords; this file only manages grants):
6
+ -- * agents_ro -> SELECT. The Grafana "agents" datasource logs in as this. NOT on
7
+ -- time_interval (window titles): only time_interval_public and the
8
+ -- active-time views, none of which carries a title or the
9
+ -- title-derived event_id.
10
+ -- * agents_writer -> USAGE on schema agents + SELECT/INSERT on usage_event, collector_scan,
11
+ -- event_session and activity_event, INSERT only on time_interval.
12
+ -- UPDATE on activity_event.attrs corrects late turn counts. No DELETE;
13
+ -- other columns and tables stay append-only + deduped by ON CONFLICT.
14
+ -- daily_note (phase 4): INSERT + SELECT on day only; agents_ro none.
15
+ -- * ledger_reader -> read-only role for a publisher: only the views and columns its queries
16
+ -- name. No usage_event, no attrs, no active minutes.
17
+ --
18
+ -- The writer is aut on each collector workstation, not a server-side job, so the database's
19
+ -- network access rules must admit every workstation that runs a collector.
20
+ --
21
+ -- A fresh LOGIN role has no USAGE on non-public schemas, and table privileges are inert
22
+ -- without it, so USAGE on schema agents is granted explicitly.
23
+ --
24
+ -- Applied via psql -f, after 01_schema.sql. Safe to re-run any number of times.
25
+
26
+ DO $$
27
+ DECLARE
28
+ have_ro boolean := EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'agents_ro');
29
+ have_writer boolean := EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'agents_writer');
30
+ have_ledger boolean := EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'ledger_reader');
31
+ BEGIN
32
+ IF have_writer THEN
33
+ GRANT USAGE ON SCHEMA agents TO agents_writer;
34
+ GRANT SELECT, INSERT ON agents.usage_event TO agents_writer;
35
+ GRANT SELECT ON agents.usage_event_dedup TO agents_writer;
36
+ -- Heartbeat now, session identity and activity in later steps. Same INSERT-only rule.
37
+ GRANT SELECT, INSERT ON agents.collector_scan TO agents_writer;
38
+ GRANT SELECT, INSERT ON agents.event_session TO agents_writer;
39
+ GRANT SELECT, INSERT ON agents.activity_event TO agents_writer;
40
+ GRANT UPDATE (attrs) ON agents.activity_event TO agents_writer;
41
+ -- Active time: aut delivers aw + manual intervals; only the postgres-run retention job
42
+ -- ever updates a row (it nulls old titles). No table SELECT: the writer's password sits on
43
+ -- every workstation, so a leaked DSN must not read other machines' titles. aut's
44
+ -- INSERT ... ON CONFLICT (event_id, ts) DO NOTHING needs SELECT on the conflict-target
45
+ -- columns only (Postgres checks the arbiter columns), so just those two.
46
+ -- The table REVOKE also drops column grants, then they are re-granted: idempotent, and it
47
+ -- undoes the first version's table-wide grant.
48
+ REVOKE SELECT ON agents.time_interval FROM agents_writer;
49
+ GRANT INSERT ON agents.time_interval TO agents_writer;
50
+ GRANT SELECT (event_id, ts) ON agents.time_interval TO agents_writer;
51
+ -- Phase 3: aut writes git outcomes, the manual /outcome and private episode corrections
52
+ -- (insert-only, ON CONFLICT DO NOTHING) and reads the linker's episodes for the review.
53
+ GRANT SELECT, INSERT ON agents.commit_outcome TO agents_writer;
54
+ GRANT SELECT, INSERT ON agents.manual_outcome TO agents_writer;
55
+ GRANT SELECT, INSERT ON agents.episode_correction TO agents_writer;
56
+ GRANT SELECT ON agents.episode TO agents_writer;
57
+ GRANT SELECT ON agents.episode_member TO agents_writer;
58
+ -- Phase 4: labels and evals are activity_event rows (granted above). Nightly notes are
59
+ -- text: insert plus SELECT on the conflict column only, same reason as time_interval.
60
+ REVOKE SELECT ON agents.daily_note FROM agents_writer;
61
+ GRANT INSERT ON agents.daily_note TO agents_writer;
62
+ GRANT SELECT (day) ON agents.daily_note TO agents_writer;
63
+ -- Phase 6: commits per day from the user's git scan, counts only. aut git-days inserts
64
+ -- new scans and reads each day's latest value, to write only the days that changed.
65
+ GRANT SELECT, INSERT ON agents.git_commit_day TO agents_writer;
66
+ GRANT SELECT ON agents.git_commits_daily TO agents_writer;
67
+ END IF;
68
+ IF have_ro THEN
69
+ GRANT USAGE ON SCHEMA agents TO agents_ro;
70
+ REVOKE ALL ON agents.work_taxonomy_mode, agents.decision_work_latest_by_taxonomy,
71
+ agents.outcome_evidence_mode, agents.decision_outcome_latest_by_contract,
72
+ agents.decision_input_status_latest FROM agents_ro;
73
+ GRANT SELECT ON agents.usage_event TO agents_ro;
74
+ GRANT SELECT ON agents.usage_event_dedup TO agents_ro;
75
+ GRANT SELECT ON agents.usage_daily TO agents_ro;
76
+ GRANT SELECT ON agents.collector_scan TO agents_ro;
77
+ GRANT SELECT ON agents.event_session TO agents_ro;
78
+ GRANT SELECT ON agents.activity_event TO agents_ro;
79
+ GRANT SELECT ON agents.collector_coverage TO agents_ro;
80
+ -- Window titles are private: agents_ro never reads the table, only views without them.
81
+ -- The REVOKE also undoes any earlier hand grant.
82
+ REVOKE ALL ON agents.time_interval FROM agents_ro;
83
+ GRANT SELECT ON agents.time_interval_public TO agents_ro;
84
+ GRANT SELECT ON agents.steer_interval TO agents_ro;
85
+ GRANT SELECT ON agents.active_interval TO agents_ro;
86
+ GRANT SELECT ON agents.active_daily_live TO agents_ro;
87
+ GRANT SELECT ON agents.active_daily TO agents_ro;
88
+ -- Phase 2b calibration: parameters and learned apps carry no title.
89
+ GRANT SELECT ON agents.time_params TO agents_ro;
90
+ GRANT SELECT ON agents.app_learned TO agents_ro;
91
+ GRANT SELECT ON agents.time_interval_effective TO agents_ro;
92
+ -- Phase 3: the episode tables stay closed; episode_monthly gives monthly counts only.
93
+ -- Phase 5 below deliberately opens per-episode rows (tokens_cost_by_episode: episode_id,
94
+ -- outcome) and per-case session ids (intervention_cases) to agents_ro, for a private
95
+ -- Grafana only: with activity_event that maps sessions to episodes and their outcomes.
96
+ REVOKE ALL ON agents.commit_outcome FROM agents_ro;
97
+ REVOKE ALL ON agents.manual_outcome FROM agents_ro;
98
+ REVOKE ALL ON agents.episode_correction FROM agents_ro;
99
+ REVOKE ALL ON agents.episode FROM agents_ro;
100
+ REVOKE ALL ON agents.episode_member FROM agents_ro;
101
+ GRANT SELECT ON agents.episode_monthly TO agents_ro;
102
+ -- Phase 4: label and eval views (ids, enums, numbers; agents_ro reads activity_event
103
+ -- already, so no new exposure). Nightly notes are private text: no access at all.
104
+ GRANT SELECT ON agents.label_latest TO agents_ro;
105
+ GRANT SELECT ON agents.eval_latest TO agents_ro;
106
+ REVOKE ALL ON agents.daily_note FROM agents_ro;
107
+ -- Phase 5 (03_effort.sql): private effort, outcome, cost and audit views for
108
+ -- a private Grafana. Counts, shares, ids and numbers; no title, text or machine. The
109
+ -- helper views and *_live twins stay ungranted: they read the phase 3 tables agents_ro
110
+ -- may not, and the twins are too slow for a panel.
111
+ GRANT SELECT ON agents.effort_period, agents.effort_daily, agents.effort_weekly,
112
+ agents.effort_monthly, agents.label_share_monthly, agents.work_mix_monthly,
113
+ agents.steer_intent_monthly, agents.outcomes_monthly, agents.episodes_revisits_weekly,
114
+ agents.parallel_steering_weekly, agents.shipping_rework_monthly, agents.coverage_monthly,
115
+ agents.built_reused_monthly, agents.learning_to_doing_monthly,
116
+ agents.follow_through_monthly, agents.tokens_cost_by_model_daily,
117
+ agents.tokens_cost_by_episode, agents.efficiency_by_episode, agents.audit_daily,
118
+ agents.intervention_cases, agents.intervention_monthly, agents.eval_history, agents.working_label_monthly, agents.working_signal_monthly,
119
+ agents.git_commits_daily, agents.steers_counted_daily, agents.steer_estimate_daily,
120
+ agents.sessions_steered_daily, agents.sessions_steered_weekly, agents.label_share_weekly,
121
+ agents.work_mix_weekly, agents.steer_intent_weekly, agents.outcomes_weekly,
122
+ agents.shipping_rework_weekly, agents.built_reused_weekly, agents.distinct_26_weeks,
123
+ agents.follow_through_weekly, agents.decision_label_weekly TO agents_ro;
124
+ END IF;
125
+ IF have_ledger THEN
126
+ GRANT USAGE ON SCHEMA agents TO ledger_reader;
127
+ -- Undo earlier label-view grants; publication reads only the aggregate projections.
128
+ REVOKE ALL ON agents.work_mix_weekly, agents.steer_intent_weekly,
129
+ agents.outcomes_weekly, agents.working_label_latest, agents.working_label_monthly,
130
+ agents.working_signal_monthly, agents.working_label_weekly,
131
+ agents.decision_label_latest, agents.work_taxonomy_mode,
132
+ agents.decision_work_latest_by_taxonomy, agents.outcome_evidence_mode,
133
+ agents.decision_outcome_latest_by_contract, agents.decision_input_status_latest FROM ledger_reader;
134
+ GRANT SELECT (kind, ts) ON agents.activity_event TO ledger_reader;
135
+ GRANT SELECT (last_ok_at) ON agents.collector_coverage TO ledger_reader;
136
+ GRANT SELECT (week, focus_blocks, parallel_peak) ON agents.effort_weekly TO ledger_reader;
137
+ GRANT SELECT (month, source, days) ON agents.coverage_monthly TO ledger_reader;
138
+ GRANT SELECT ON agents.git_commits_daily, agents.steers_counted_daily, agents.steer_estimate_daily,
139
+ agents.sessions_steered_daily, agents.sessions_steered_weekly, agents.decision_label_weekly,
140
+ agents.git_outcomes_weekly, agents.shipping_rework_weekly,
141
+ agents.built_reused_weekly, agents.distinct_26_weeks, agents.follow_through_weekly TO ledger_reader;
142
+ END IF;
143
+ END $$;
144
+
145
+ -- Calibration runs as postgres from the daily job only. Functions are executable by PUBLIC by
146
+ -- default; they run with the caller's rights, so this is belt and braces.
147
+ REVOKE EXECUTE ON FUNCTION agents.time_calibration(timestamptz) FROM PUBLIC;
148
+ REVOKE EXECUTE ON FUNCTION agents.time_calibrate(timestamptz) FROM PUBLIC;
149
+ REVOKE EXECUTE ON FUNCTION agents.episode_link() FROM PUBLIC;
@@ -0,0 +1,66 @@
1
+ -- Refresh job for the agents database (hourly in the Compose stack). Run it as postgres (psql -f,
2
+ -- ON_ERROR_STOP), so any error exits non-zero and the scheduler can alert on the failure.
3
+ -- Idempotent.
4
+ \set ON_ERROR_STOP on
5
+
6
+ -- One statement preserves both cron atomicity and a caller's transaction boundary.
7
+ DO $refresh$
8
+ DECLARE
9
+ refresh_started_at timestamptz;
10
+ extra record;
11
+ BEGIN
12
+ PERFORM pg_advisory_xact_lock(hashtext('agents.derived_refresh'));
13
+ refresh_started_at := clock_timestamp();
14
+
15
+ -- Claude Code sometimes re-logs one message under a second ts. The (event_id, ts) key keeps both
16
+ -- rows, so every raw sum counts that turn twice. Keep the row agents.usage_event_dedup keeps and
17
+ -- delete the other copies: Grafana's raw panels and usage_daily (its hourly policy re-materializes
18
+ -- the changed days) then count it once. Only that known case is settled here; any other duplicate
19
+ -- stays, for audit_daily.dup_unexplained to report.
20
+ -- One DELETE per copy, with literal values: TimescaleDB then decompresses only the compressed
21
+ -- batch that holds the row. A joined DELETE decompresses whole chunks of old history.
22
+ FOR extra IN
23
+ SELECT event_id, ts FROM (
24
+ SELECT event_id, ts,
25
+ row_number() OVER (PARTITION BY event_id ORDER BY total_tokens DESC, ts DESC) AS rn
26
+ FROM agents.usage_event
27
+ WHERE event_id IN (SELECT event_id FROM agents.usage_event GROUP BY event_id
28
+ HAVING count(*) > 1 AND bool_and(agent = 'claude_code')
29
+ AND count(DISTINCT machine) = 1)
30
+ ) ranked
31
+ WHERE rn > 1
32
+ LOOP
33
+ -- ts written as UTC ISO text: the session's DateStyle and zone abbreviations cannot shift it.
34
+ EXECUTE format('DELETE FROM agents.usage_event WHERE agent = %L AND event_id = %L AND ts = %L::timestamptz',
35
+ 'claude_code', extra.event_id,
36
+ to_char(timezone('UTC', extra.ts), 'YYYY-MM-DD HH24:MI:SS.US') || '+00');
37
+ END LOOP;
38
+
39
+ -- Window titles are kept indefinitely by default. To turn a retention limit on, add here:
40
+ -- UPDATE agents.time_interval SET title = NULL
41
+ -- WHERE ts < now() - interval '<N> days' AND title IS NOT NULL;
42
+ -- (time_interval_title_ts_idx keeps that UPDATE cheap.)
43
+
44
+ -- Calibrate the steer rules first, so the refresh below already uses the newest parameters.
45
+ -- 01_schema.sql explains the rules and the guard.
46
+ PERFORM agents.time_calibrate();
47
+
48
+ -- Rebuild the per-day totals Grafana reads (01_schema.sql explains why it is materialized).
49
+ REFRESH MATERIALIZED VIEW CONCURRENTLY agents.active_daily;
50
+
51
+ -- Rebuild episodes within the same transaction. A failed rebuild publishes no partial snapshot.
52
+ PERFORM agents.episode_link();
53
+
54
+ -- Rebuild the phase 5 views Grafana reads (03_effort.sql). After the episodes, which
55
+ -- effort and episode cost count; the model rollup first, coverage_monthly reads it.
56
+ REFRESH MATERIALIZED VIEW CONCURRENTLY agents.tokens_cost_by_model_daily;
57
+ REFRESH MATERIALIZED VIEW CONCURRENTLY agents.effort_period;
58
+ REFRESH MATERIALIZED VIEW CONCURRENTLY agents.tokens_cost_by_episode;
59
+ REFRESH MATERIALIZED VIEW CONCURRENTLY agents.audit_daily;
60
+
61
+ -- Labels may use only scans completed before this refresh began.
62
+ INSERT INTO agents.collector_scan (machine, collector, started_at, finished_at, status)
63
+ VALUES ('sink', 'episode_link', refresh_started_at, clock_timestamp(), 'ok');
64
+
65
+ END
66
+ $refresh$;
@@ -0,0 +1,180 @@
1
+ """Explicit local sources extend native readers without claiming harness completeness."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ import json
7
+ import os
8
+ from pathlib import Path
9
+ import stat
10
+ from typing import Any
11
+ from zoneinfo import ZoneInfo
12
+
13
+ from .config import config_path
14
+ from .observations import identifier
15
+
16
+ MAX_BYTES = 1024 * 1024
17
+
18
+
19
+ class SourceError(ValueError):
20
+ """The source registry or an explicit native root cannot be read safely."""
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class Source:
25
+ id: str
26
+ harness: str
27
+ engine: str
28
+ location: str
29
+ home: Path | None
30
+ transport: str = 'unknown'
31
+
32
+
33
+ @dataclass(frozen=True)
34
+ class Registry:
35
+ sources: tuple[Source, ...]
36
+ links: tuple[dict[str, str], ...]
37
+ publications: tuple[dict[str, str], ...]
38
+
39
+
40
+ def _absolute(value: Any) -> Path:
41
+ if not isinstance(value, str) or not value or '\x00' in value:
42
+ raise ValueError("invalid absolute path")
43
+ path = Path(value)
44
+ if not path.is_absolute():
45
+ raise ValueError("invalid absolute path")
46
+ return path
47
+
48
+
49
+ def registry_path(home: Path) -> Path:
50
+ if home == Path.home():
51
+ if value := os.environ.get('AUT_SOURCES'):
52
+ try:
53
+ return _absolute(value)
54
+ except ValueError:
55
+ raise SourceError("sources: AUT_SOURCES must be an absolute path") from None
56
+ return config_path().parent / 'sources.json'
57
+ return home / '.config/agent-usage-tracker/sources.json'
58
+
59
+
60
+ def _read_json(path: Path) -> Any:
61
+ descriptor = os.open(path, os.O_RDONLY | getattr(os, 'O_NONBLOCK', 0))
62
+ with os.fdopen(descriptor, 'rb') as stream:
63
+ if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode):
64
+ raise ValueError("configuration must be a regular file")
65
+ raw = stream.read(MAX_BYTES + 1)
66
+ if len(raw) > MAX_BYTES:
67
+ raise ValueError("oversized configuration")
68
+ return json.loads(raw)
69
+
70
+
71
+ def _rows(value: dict[str, Any], key: str, limit: int) -> list[Any]:
72
+ rows = value.get(key, [])
73
+ if not isinstance(rows, list) or len(rows) > limit:
74
+ raise ValueError("invalid configuration list")
75
+ return rows
76
+
77
+
78
+ def _fields(value: Any, required: set[str], optional: set[str] | None = None) -> None:
79
+ if not isinstance(value, dict) or not required <= value.keys() \
80
+ or value.keys() - required - (optional or set()):
81
+ raise ValueError("missing or unknown fields")
82
+
83
+
84
+ def load_registry(home: Path) -> Registry:
85
+ try:
86
+ value = _read_json(registry_path(home))
87
+ except FileNotFoundError:
88
+ return Registry((), (), ())
89
+ except (OSError, ValueError, RecursionError):
90
+ raise SourceError("sources: registry is unreadable or invalid") from None
91
+ try:
92
+ _fields(value, {'version'}, {'sources', 'links', 'publications'})
93
+ if type(value['version']) is not int or value['version'] != 1:
94
+ raise ValueError("unsupported version")
95
+ sources: dict[str, Source] = {}
96
+ for row in _rows(value, 'sources', 128):
97
+ _fields(row, {'id', 'harness', 'engine', 'location'}, {'home', 'transport'})
98
+ if row['location'] not in ('local', 'remote'):
99
+ raise ValueError("invalid location")
100
+ root = _absolute(row['home']) if 'home' in row else None
101
+ if root is not None and row['location'] != 'local':
102
+ raise ValueError("remote source cannot have a local home")
103
+ transport = row.get('transport', 'native_files' if root else 'unknown')
104
+ if transport not in ('native_files', 'structured_cli', 'app_server', 'acp',
105
+ 'http_events', 'otlp', 'pty', 'unknown'):
106
+ raise ValueError("invalid transport")
107
+ source = Source(identifier(row['id']), identifier(row['harness']),
108
+ identifier(row['engine']), row['location'], root, transport)
109
+ if source.id in sources and sources[source.id] != source:
110
+ raise ValueError("conflicting source identifier")
111
+ sources[source.id] = source
112
+ links: list[dict[str, str]] = []
113
+ seen: set[tuple[tuple[str, str], ...]] = set()
114
+ for row in _rows(value, 'links', 10000):
115
+ _fields(row, {'source_id', 'native_session_id', 'harness_thread_id'}, {'request_id'})
116
+ link = {key: identifier(item) for key, item in row.items()}
117
+ if link['source_id'] not in sources:
118
+ raise ValueError("unknown source identifier")
119
+ edge = tuple(sorted(link.items()))
120
+ if edge not in seen:
121
+ seen.add(edge)
122
+ links.append(link)
123
+ publications: dict[str, dict[str, str]] = {}
124
+ for row in _rows(value, 'publications', 32):
125
+ _fields(row, {'id', 'path', 'timezone'})
126
+ pub_id = identifier(row['id'])
127
+ _absolute(row['path'])
128
+ if not isinstance(row['timezone'], str):
129
+ raise ValueError("invalid timezone")
130
+ ZoneInfo(row['timezone'])
131
+ if pub_id in publications and publications[pub_id] != row:
132
+ raise ValueError("conflicting publication identifier")
133
+ publications[pub_id] = dict(row)
134
+ return Registry(tuple(sources.values()), tuple(links), tuple(publications.values()))
135
+ except (ValueError, TypeError, KeyError, RecursionError):
136
+ raise SourceError("sources: registry is invalid") from None
137
+
138
+
139
+ def _t3_claude_homes(home: Path) -> list[Path]:
140
+ try:
141
+ value = _read_json(home / '.t3/userdata/settings.json')
142
+ instances = value.get('providerInstances', {})
143
+ roots = []
144
+ for instance in instances.values():
145
+ if instance.get('driver') == 'claudeAgent':
146
+ root = instance.get('config', {}).get('homePath')
147
+ if root:
148
+ roots.append(_absolute(root))
149
+ return roots
150
+ except FileNotFoundError:
151
+ return []
152
+ except (OSError, ValueError, TypeError, AttributeError, RuntimeError, RecursionError):
153
+ raise SourceError("claude_code: T3 profile configuration is unreadable") from None
154
+
155
+
156
+ def engine_homes(home: Path, engine: str) -> list[Path]:
157
+ registry = load_registry(home)
158
+ defaults = {'claude_code': home / '.claude', 'codex': home / '.codex',
159
+ 'pi': home / '.pi/agent'}
160
+ if engine not in defaults:
161
+ return []
162
+ roots = [defaults[engine]]
163
+ try:
164
+ env = {'claude_code': 'CLAUDE_CONFIG_DIR', 'codex': 'CODEX_HOME'}.get(engine)
165
+ if home == Path.home() and env and (value := os.environ.get(env)):
166
+ roots.append(_absolute(value))
167
+ roots.extend(source.home for source in registry.sources
168
+ if source.engine == engine and source.location == 'local'
169
+ and source.home is not None
170
+ and source.transport in ('native_files', 'structured_cli'))
171
+ if engine == 'claude_code':
172
+ roots.extend(_t3_claude_homes(home))
173
+ unique: dict[Path, Path] = {}
174
+ for root in roots:
175
+ unique.setdefault(root.resolve(), root)
176
+ return list(unique.values())
177
+ except (OSError, ValueError, RuntimeError) as exc:
178
+ if isinstance(exc, SourceError):
179
+ raise
180
+ raise SourceError("sources: native root is invalid") from None
@@ -0,0 +1,191 @@
1
+ """Per-machine watermark store. Bounds work on each scan; correctness of dedup comes
2
+ from event_id + ON CONFLICT at the sink, so the watermark can be slightly over-inclusive
3
+ without ever double-counting.
4
+
5
+ Layout (JSON): {"collectors": {"<name>": {<private state>}}}; `aut label` keeps its watermark under
6
+ the name "label". A process writes back only the names it owns (see save_state).
7
+ A collector that changes its file entries keeps a `state_v` in its state and upgrades older state
8
+ once, in its own _upgrade_state, when it starts.
9
+ The watermark is only committed AFTER a successful push, so a failed push means the next
10
+ run re-reads and re-sends (idempotent at the sink).
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import contextlib
16
+ import errno
17
+ import json
18
+ import os
19
+ import stat
20
+ import sys
21
+ import tempfile
22
+ from pathlib import Path
23
+ from typing import Any, Iterable, Iterator
24
+
25
+
26
+ def state_path() -> Path:
27
+ if os.environ.get("AUT_STATE"):
28
+ return Path(os.environ["AUT_STATE"]).expanduser()
29
+ base = os.environ.get("XDG_STATE_HOME", str(Path.home() / ".local" / "state"))
30
+ return Path(base) / "agent-usage-tracker" / "state.json"
31
+
32
+
33
+ def _warn_unusable(path: Path, reason: str) -> None:
34
+ """Starting clean is the right recovery, but doing it silently is not: losing the watermark
35
+ means the next scan re-reads every log from byte 0 and re-pushes all history. The sink dedups
36
+ that on event_id, so nothing breaks, but on a busy machine it is a multi-minute scan and a
37
+ lot of redundant rows -- and if the state keeps going bad, silence hides it happening every
38
+ single run. Copy the file aside first, because save_state() overwrites it at the end of this
39
+ same run and would otherwise destroy the only evidence of what went wrong."""
40
+ kept = path.with_suffix(".corrupt")
41
+ try:
42
+ data = path.read_bytes() # copy, not move: save_state still wants the path
43
+ with os.fdopen(os.open(kept, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "wb") as fh:
44
+ fh.write(data)
45
+ detail = f"a copy is at {kept}"
46
+ except OSError as exc:
47
+ detail = f"could not copy it aside ({exc})"
48
+ print(f"aut: warning: the watermark at {path} is unusable ({reason}); {detail}. Starting "
49
+ "clean, so this run re-reads and re-pushes ALL history. The sink will dedup it, but "
50
+ "expect a slow scan.", file=sys.stderr)
51
+
52
+
53
+ def load_state() -> dict[str, Any]:
54
+ """The watermarks. A read error (a root-owned file, EIO) raises AutError: the file may be fine,
55
+ and starting clean would end with save_state replacing it."""
56
+ path = state_path()
57
+ try:
58
+ obj = json.loads(path.read_text())
59
+ except FileNotFoundError:
60
+ return {"collectors": {}} # first run on this machine: genuinely nothing yet
61
+ except OSError as exc:
62
+ from .config import AutError # here because config imports this module
63
+ raise AutError(f"aut: could not read {path} ({exc}); fix its owner or mode") from exc
64
+ except ValueError as exc: # JSONDecodeError and UnicodeDecodeError are ValueErrors
65
+ _warn_unusable(path, str(exc))
66
+ return {"collectors": {}}
67
+ if not isinstance(obj, dict) or not isinstance(obj.get("collectors"), dict):
68
+ _warn_unusable(path, f"unexpected shape: {type(obj).__name__}")
69
+ return {"collectors": {}}
70
+ return obj
71
+
72
+
73
+ class LockBusy(BlockingIOError):
74
+ """A nonblocking lock could not be acquired."""
75
+
76
+
77
+ @contextlib.contextmanager
78
+ def _locked(path: Path, *, blocking: bool = True) -> Iterator[None]:
79
+ """Hold an advisory lock on a file beside `path`, for one short read-merge-write."""
80
+ path.parent.mkdir(parents=True, exist_ok=True)
81
+ with open(path.with_suffix(".lock"), "a+b") as fh:
82
+ if sys.platform == "win32":
83
+ import msvcrt
84
+ try:
85
+ msvcrt.locking(fh.fileno(), msvcrt.LK_LOCK if blocking else msvcrt.LK_NBLCK, 1)
86
+ except OSError as exc:
87
+ if not blocking and exc.errno in (errno.EACCES, errno.EAGAIN, errno.EDEADLK):
88
+ raise LockBusy(exc.errno, "lock is already held") from exc
89
+ raise
90
+ try:
91
+ yield
92
+ finally:
93
+ msvcrt.locking(fh.fileno(), msvcrt.LK_UNLCK, 1)
94
+ else:
95
+ import fcntl
96
+ try:
97
+ fcntl.flock(fh, fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB))
98
+ except BlockingIOError as exc:
99
+ raise LockBusy(exc.errno, "lock is already held") from exc
100
+ yield
101
+
102
+
103
+ def save_state(state: dict[str, Any], names: Iterable[str]) -> None:
104
+ """Write the named collectors' watermarks (a name gone from `state` is dropped) over what is on
105
+ disk now. A scan and a long `aut label run` each hold a copy from when they started; writing
106
+ that copy back whole would roll the other's watermarks back. Two runs of one collector still
107
+ race on its own entry, which costs a re-read at most: the sink dedups on event_id. A read error
108
+ raises, because the file may hold other collectors' good watermarks."""
109
+ path = state_path()
110
+ with _locked(path):
111
+ try:
112
+ disk = json.loads(path.read_text(encoding="utf-8"))
113
+ except (FileNotFoundError, ValueError):
114
+ disk = {}
115
+ if not isinstance(disk, dict):
116
+ disk = {}
117
+ if not isinstance(disk.get("collectors"), dict):
118
+ disk["collectors"] = {}
119
+ for name in names:
120
+ if name in state["collectors"]:
121
+ disk["collectors"][name] = state["collectors"][name]
122
+ else:
123
+ disk["collectors"].pop(name, None)
124
+ atomic_write_text(path, json.dumps(disk, indent=1))
125
+
126
+
127
+ def warn_if_group_readable(path: Path) -> None:
128
+ """For a file that holds a secret. Windows has no POSIX modes, so the check is off there."""
129
+ if os.name != "nt" and path.stat().st_mode & 0o077:
130
+ print(f"aut: warning: {path} holds a secret but is group/other-accessible; "
131
+ f"run: chmod 600 {path}", file=sys.stderr)
132
+
133
+
134
+ def _plain_write_mode(path: Path) -> int:
135
+ """The mode a plain open() leaves: an existing file's own, else the umask's."""
136
+ try:
137
+ return stat.S_IMODE(path.stat().st_mode)
138
+ except FileNotFoundError:
139
+ umask = os.umask(0) # the only way to read it
140
+ os.umask(umask)
141
+ return 0o666 & ~umask
142
+
143
+
144
+ def atomic_write_text(path: Path, text: str, *, shared: bool = False,
145
+ newline: str | None = None) -> None:
146
+ """Write through a unique temp file and a rename, so a crash or a second process never leaves
147
+ a half-written file. Creates the parent directory. The file is private to this user unless
148
+ `shared`, for a file other users or services may read: then it gets the mode a plain write
149
+ would."""
150
+ path.parent.mkdir(parents=True, exist_ok=True)
151
+ fd, tmp = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp")
152
+ try:
153
+ with os.fdopen(fd, "w", encoding="utf-8", newline=newline) as fh:
154
+ fh.write(text)
155
+ if shared:
156
+ os.chmod(tmp, _plain_write_mode(path))
157
+ os.replace(tmp, path)
158
+ finally:
159
+ if os.path.exists(tmp):
160
+ os.unlink(tmp)
161
+
162
+
163
+ def load_json_dict(path: Path) -> dict[str, Any]:
164
+ """A rebuildable cache's contents; {} when it is missing or unusable (it is simply redone)."""
165
+ try:
166
+ obj = json.loads(path.read_text(encoding="utf-8"))
167
+ except (OSError, ValueError):
168
+ return {}
169
+ return obj if isinstance(obj, dict) else {}
170
+
171
+
172
+ def parse_record(raw: bytes | str) -> dict[str, Any] | None:
173
+ """One complete JSONL line as a dict, else None. read_new_lines holds back partial lines, so
174
+ there a None is a real anomaly: the caller counts it as malformed and skips it. A reader that
175
+ only shows local text just skips it, as does one of a file a crash can cut mid-write."""
176
+ try:
177
+ o = json.loads(raw)
178
+ except ValueError: # JSONDecodeError and UnicodeDecodeError are ValueErrors
179
+ return None
180
+ return o if isinstance(o, dict) else None
181
+
182
+
183
+ def collector_wm(state: dict[str, Any], name: str) -> dict[str, Any]:
184
+ """Return (creating if needed) the private watermark dict for one collector. Resets a
185
+ non-dict value (from external corruption) to an empty dict so a scan can't crash on it."""
186
+ collectors = state.setdefault("collectors", {})
187
+ wm = collectors.get(name)
188
+ if not isinstance(wm, dict):
189
+ wm = {}
190
+ collectors[name] = wm
191
+ return wm