agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,1186 @@
1
+ -- agents.usage_event: coding-agent token + model usage (Claude Code, Codex, Cursor, Pi).
2
+ --
3
+ -- Fed by the standalone `agent-usage-tracker` (aut) collector on each dev machine.
4
+ -- Isolation-first: this lives in its own database `agents` (its own failure domain), not
5
+ -- co-tenanted with unrelated data.
6
+ --
7
+ -- Idempotent + safe to re-run. Applied via psql -f (see the grants file for roles).
8
+ -- One-time bootstrap as superuser:
9
+ -- CREATE DATABASE agents;
10
+ -- \c agents
11
+ -- then apply the files with apply.sh (sink_stack/), which also creates the roles
12
+ --
13
+ -- Note: every USD figure is an estimate (API-equivalent). Subscriptions are flat-rate, so
14
+ -- these numbers are notional, never a real bill.
15
+
16
+ CREATE EXTENSION IF NOT EXISTS timescaledb;
17
+ CREATE SCHEMA IF NOT EXISTS agents;
18
+
19
+ CREATE TABLE IF NOT EXISTS agents.usage_event (
20
+ event_id text NOT NULL, -- deterministic idempotency key from the collector
21
+ ts timestamptz NOT NULL, -- from the source log line (hypertable partition col)
22
+ machine text NOT NULL, -- role-based label (laptop, buildbox, ...)
23
+ agent text NOT NULL, -- claude_code | codex | cursor | pi
24
+ kind text NOT NULL DEFAULT 'agent', -- 'agent' = assistant turn (tokens); 'human' = a prompt the user sent (0 tokens)
25
+ model text,
26
+ provider text,
27
+ input_tokens bigint NOT NULL DEFAULT 0, -- fresh (non-cached) prompt tokens
28
+ output_tokens bigint NOT NULL DEFAULT 0,
29
+ cache_read_tokens bigint NOT NULL DEFAULT 0,
30
+ cache_write_tokens bigint NOT NULL DEFAULT 0,
31
+ reasoning_tokens bigint, -- codex only; null elsewhere
32
+ total_tokens bigint NOT NULL DEFAULT 0,
33
+ est_cost_usd numeric(12,6), -- NOTIONAL / API-equivalent
34
+ project text, -- working-dir basename, never a full path
35
+ -- Timescale requires the partition column in any unique index, so the dedup key is
36
+ -- (event_id, ts). event_id is deterministic + ts is immutable, so this is stable.
37
+ PRIMARY KEY (event_id, ts)
38
+ );
39
+
40
+ SELECT create_hypertable('agents.usage_event', 'ts', if_not_exists => TRUE);
41
+
42
+ -- Upgrade path: CREATE TABLE IF NOT EXISTS above is a no-op on an existing DB, so the kind
43
+ -- column defined in it never lands there without this idempotent ADD COLUMN.
44
+ ALTER TABLE agents.usage_event ADD COLUMN IF NOT EXISTS kind text NOT NULL DEFAULT 'agent';
45
+
46
+ -- Compression keeps disk use and WAL churn low. That is what protects the disk; retention
47
+ -- never was.
48
+ --
49
+ -- Raw rows are now kept FOREVER. The old policy dropped them after 1825 days, which was only
50
+ -- ever a stand-in for "lifetime-ish". Two things make it wrong to keep:
51
+ --
52
+ -- * It cannot be undone. Everything raw carries and the daily rollup does not -- the hour of
53
+ -- day, the project, reasoning tokens, one row per turn -- is gone the moment a chunk is
54
+ -- dropped. The rollup buckets to a whole day, so it can say "41 human turns on the 3rd" but
55
+ -- never when the first one landed. That per-turn timestamp is the presence signal behind
56
+ -- active-time reporting, so a silent drop at year five would quietly destroy the evidence.
57
+ -- * It buys little. Compressed raw rows are small, so keeping them forever costs little disk.
58
+ --
59
+ -- remove_retention_policy is idempotent via if_exists, so this also retires the policy on hosts
60
+ -- that already have it rather than only affecting fresh installs.
61
+ ALTER TABLE agents.usage_event SET (
62
+ timescaledb.compress,
63
+ timescaledb.compress_segmentby = 'agent, model',
64
+ timescaledb.compress_orderby = 'ts DESC, event_id'
65
+ );
66
+ SELECT add_compression_policy('agents.usage_event', INTERVAL '14 days', if_not_exists => TRUE);
67
+ SELECT remove_retention_policy('agents.usage_event', if_exists => TRUE);
68
+
69
+ -- Daily rollup for Grafana and other stats readers. Cheap to query and kept forever; raw is
70
+ -- kept forever too now, so this is a convenience, no longer the only long-term survivor.
71
+ -- Upgrade path: a continuous aggregate cannot ALTER its GROUP BY to add a column, and the
72
+ -- CREATE ... IF NOT EXISTS below is a no-op if an OLD (pre-kind) usage_daily already exists.
73
+ -- So drop the old one first, but ONLY if it lacks the kind column (fresh installs have no
74
+ -- view yet; already-migrated DBs keep theirs untouched). DROP is fine in plpgsql; the cagg
75
+ -- CREATE stays at top level (TimescaleDB disallows it inside a transaction block).
76
+ DO $$
77
+ BEGIN
78
+ IF EXISTS (SELECT 1 FROM information_schema.columns
79
+ WHERE table_schema = 'agents' AND table_name = 'usage_daily' AND column_name = 'day')
80
+ AND NOT EXISTS (SELECT 1 FROM information_schema.columns
81
+ WHERE table_schema = 'agents' AND table_name = 'usage_daily' AND column_name = 'kind')
82
+ THEN
83
+ EXECUTE 'DROP MATERIALIZED VIEW agents.usage_daily CASCADE';
84
+ END IF;
85
+ END $$;
86
+
87
+ CREATE MATERIALIZED VIEW IF NOT EXISTS agents.usage_daily
88
+ WITH (timescaledb.continuous) AS
89
+ -- kind is a grouping dimension so "turns" can be split into agent vs human; human turns
90
+ -- carry zero tokens, so the token sums are unaffected by their presence.
91
+ SELECT time_bucket('1 day', ts) AS day,
92
+ machine, agent, model, kind,
93
+ sum(input_tokens) AS input_tokens,
94
+ sum(output_tokens) AS output_tokens,
95
+ sum(cache_read_tokens) AS cache_read_tokens,
96
+ sum(cache_write_tokens) AS cache_write_tokens,
97
+ sum(total_tokens) AS total_tokens,
98
+ sum(est_cost_usd) AS est_cost_usd,
99
+ count(*) AS turns
100
+ FROM agents.usage_event
101
+ GROUP BY day, machine, agent, model, kind
102
+ WITH NO DATA;
103
+
104
+ -- Real-time aggregation (NOT materialized_only). With a 1-day bucket and a 1h
105
+ -- end_offset, the scheduled policy only materializes buckets that are fully
106
+ -- enclosed by [start, now-1h], so TODAY's bucket is never materialized until
107
+ -- after midnight UTC. materialized_only=true would then hide the current day
108
+ -- from every Grafana panel that reads this view (a "Last 24 hours" overview
109
+ -- would blank out all day). Real-time mode computes today's partial bucket live
110
+ -- from the raw hypertable, so the dashboard always reflects current usage.
111
+ -- Idempotent: safe to re-run.
112
+ ALTER MATERIALIZED VIEW agents.usage_daily SET (timescaledb.materialized_only = false);
113
+
114
+ -- Refresh window covers ALL history (start_offset NULL). Rows can arrive days or months late
115
+ -- with old timestamps: a machine whose logs sync after days offline, or a backfill. With a
116
+ -- bounded (e.g. 3-day) window those rows never reach the rollup. The policy re-materializes
117
+ -- only invalidated ranges, so the wide
118
+ -- window costs nothing when nothing changed. if_not_exists keeps an old policy as it is, so an
119
+ -- existing policy with a bounded start_offset is removed first. Idempotent.
120
+ DO $$
121
+ BEGIN
122
+ IF EXISTS (
123
+ SELECT 1
124
+ FROM timescaledb_information.jobs j -- lists a cagg job under the view's own name
125
+ WHERE j.hypertable_schema = 'agents' AND j.hypertable_name = 'usage_daily'
126
+ AND j.proc_name = 'policy_refresh_continuous_aggregate'
127
+ AND j.config->>'start_offset' IS NOT NULL)
128
+ THEN
129
+ PERFORM remove_continuous_aggregate_policy('agents.usage_daily');
130
+ END IF;
131
+ END $$;
132
+
133
+ SELECT add_continuous_aggregate_policy('agents.usage_daily',
134
+ start_offset => NULL,
135
+ end_offset => INTERVAL '1 hour',
136
+ schedule_interval => INTERVAL '1 hour',
137
+ if_not_exists => TRUE);
138
+
139
+ -- Materialize ALL existing history once. The hourly policy above refreshes only invalidated
140
+ -- ranges, and a fresh CREATE / recreate (WITH NO DATA) invalidates nothing, so older buckets would never be materialized and
141
+ -- the rollup would silently start at "three days ago" while Grafana showed it as the full
142
+ -- history. Raw is no longer dropped, so this is recoverable now rather than terminal -- still
143
+ -- do it, or every dashboard reading usage_daily is wrong until someone notices. No-op on an
144
+ -- empty DB. Must run outside a txn block: psql -f autocommits each statement, so this is fine.
145
+ CALL refresh_continuous_aggregate('agents.usage_daily', NULL, NULL);
146
+
147
+ -- One row per event_id. Claude Code sometimes re-logs the same message with a different ts,
148
+ -- and the hypertable key is (event_id, ts) (Timescale needs ts in it), so both rows persist and
149
+ -- every sum over usage_event counts that turn twice. This view settles every duplicate at read
150
+ -- time: it keeps the row with the largest total_tokens (ts DESC breaks ties, and is unique
151
+ -- per event_id because of the primary key); agents.audit_daily checks what it settles. Derived views and new queries should
152
+ -- read this view, not usage_event. usage_daily sums raw rows (a continuous aggregate
153
+ -- cannot read a view); the refresh job (refresh.sql) deletes Claude Code's re-logged copies from
154
+ -- the table, so it and usage_daily count them once after the next run. Idempotent: CREATE OR REPLACE.
155
+ CREATE OR REPLACE VIEW agents.usage_event_dedup AS
156
+ SELECT DISTINCT ON (event_id) *
157
+ FROM agents.usage_event
158
+ ORDER BY event_id, total_tokens DESC, ts DESC;
159
+
160
+ -- Re-grant Grafana read: the DO-block recreate above uses DROP ... CASCADE, which removes the
161
+ -- agents_ro grant. The grants file is normally applied right after this file; this block
162
+ -- covers a manual `psql -f` of this file on its own. Guarded so a first-time install
163
+ -- does not error here: the roles may not exist yet when this file runs.
164
+ DO $$
165
+ BEGIN
166
+ IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'agents_ro') THEN
167
+ GRANT SELECT ON agents.usage_daily TO agents_ro;
168
+ GRANT SELECT ON agents.usage_event_dedup TO agents_ro;
169
+ END IF;
170
+ IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'agents_writer') THEN
171
+ GRANT SELECT ON agents.usage_event_dedup TO agents_writer;
172
+ END IF;
173
+ END $$;
174
+
175
+ -- One row per aut scan, per machine and collector: proof that the collector RAN, and how it
176
+ -- ended. usage_event alone cannot tell a machine whose aut died from a quiet week; this can.
177
+ -- aut writes it after each scan over the same DSN as usage_event, best effort: a failed
178
+ -- heartbeat insert never blocks or fails usage delivery. error is short (the client truncates
179
+ -- it to 300 chars and strips DSN/URL credentials). A freshness monitoring check can watch it.
180
+ -- Plain table, not a hypertable: one row per collector per scan, so roughly 4 to 600 rows per
181
+ -- machine per day depending on the scan interval (hours, or 10 to 15 min with the shipped
182
+ -- timers), on the order of 200k rows per machine per year.
183
+ -- shortcut: no retention; add a delete of rows older than 90 days (or make it a hypertable)
184
+ -- once the table passes about 1M rows.
185
+ CREATE TABLE IF NOT EXISTS agents.collector_scan (
186
+ machine text NOT NULL,
187
+ collector text NOT NULL, -- claude_code | codex | cursor | pi | episode_link
188
+ started_at timestamptz NOT NULL,
189
+ finished_at timestamptz NOT NULL,
190
+ status text NOT NULL CHECK (status IN ('ok', 'error', 'degraded')),
191
+ events integer NOT NULL DEFAULT 0, -- new events this scan
192
+ human integer NOT NULL DEFAULT 0, -- of which human prompts
193
+ newest_ts timestamptz, -- newest valid event ts seen this scan
194
+ error text,
195
+ aut_version text,
196
+ PRIMARY KEY (machine, collector, finished_at)
197
+ );
198
+ -- 'degraded': the collector ran but could not read a source that is there (a stopped aw-server, an
199
+ -- expired token). CREATE above does not change a table that exists, so widen the check here.
200
+ DO $$
201
+ BEGIN
202
+ IF NOT EXISTS (SELECT 1 FROM pg_constraint
203
+ WHERE conrelid = 'agents.collector_scan'::regclass
204
+ AND conname = 'collector_scan_status_check'
205
+ AND pg_get_constraintdef(oid) LIKE '%degraded%') THEN
206
+ ALTER TABLE agents.collector_scan DROP CONSTRAINT IF EXISTS collector_scan_status_check;
207
+ ALTER TABLE agents.collector_scan ADD CONSTRAINT collector_scan_status_check
208
+ CHECK (status IN ('ok', 'error', 'degraded'));
209
+ END IF;
210
+ END $$;
211
+ CREATE INDEX IF NOT EXISTS collector_scan_finished_at_idx ON agents.collector_scan (finished_at);
212
+
213
+ -- Session identity per event, keyed by event_id. It cannot live on usage_event: those rows are
214
+ -- never updated (the writer is INSERT only and the sink is ON CONFLICT DO NOTHING), so a column
215
+ -- added there would stay NULL on every existing row forever. Join it to agents.usage_event_dedup
216
+ -- on event_id. Nothing writes it yet; the upcoming aut steps do.
217
+ CREATE TABLE IF NOT EXISTS agents.event_session (
218
+ event_id text PRIMARY KEY,
219
+ machine text NOT NULL,
220
+ agent text NOT NULL,
221
+ session_id text NOT NULL,
222
+ parent_session_id text,
223
+ run_origin text NOT NULL
224
+ CHECK (run_origin IN ('interactive', 'automated', 'subagent', 'unknown')),
225
+ turn_id text,
226
+ first_seen timestamptz NOT NULL DEFAULT now()
227
+ );
228
+
229
+ -- Non-token activity (steer, interrupt, commit, pr, tool_error, turn, invoke, ...). The kinds
230
+ -- arrive in later steps; nothing writes this yet. attrs carries non-text evidence only (counts,
231
+ -- flags, ids), never prompt or output text. Same (event_id, ts) key rule as usage_event.
232
+ -- No compression or retention policy yet: decide once there is real volume to size it against.
233
+ CREATE TABLE IF NOT EXISTS agents.activity_event (
234
+ event_id text NOT NULL,
235
+ ts timestamptz NOT NULL,
236
+ machine text NOT NULL,
237
+ agent text NOT NULL,
238
+ kind text NOT NULL,
239
+ session_id text,
240
+ turn_id text,
241
+ attrs jsonb NOT NULL DEFAULT '{}'::jsonb,
242
+ PRIMARY KEY (event_id, ts)
243
+ );
244
+
245
+ SELECT create_hypertable('agents.activity_event', 'ts', if_not_exists => TRUE);
246
+ CREATE INDEX IF NOT EXISTS activity_event_machine_agent_session_idx
247
+ ON agents.activity_event (machine, agent, session_id);
248
+
249
+ -- One row per (machine, collector): when it last scanned cleanly, when it last failed, and
250
+ -- what the latest scan said. newest_ts is over ok scans only, so a failing scan cannot claim
251
+ -- data it never delivered. Idempotent: CREATE OR REPLACE.
252
+ CREATE OR REPLACE VIEW agents.collector_coverage AS
253
+ SELECT machine, collector,
254
+ max(finished_at) FILTER (WHERE status = 'ok') AS last_ok_at,
255
+ max(finished_at) FILTER (WHERE status = 'error') AS last_error_at,
256
+ (array_agg(status ORDER BY finished_at DESC))[1] AS last_status,
257
+ count(*) FILTER (WHERE status = 'ok' AND finished_at > now() - interval '7 days')
258
+ AS ok_scans_7d,
259
+ max(newest_ts) FILTER (WHERE status = 'ok') AS newest_ts,
260
+ (array_agg(aut_version ORDER BY finished_at DESC))[1] AS aut_version
261
+ FROM agents.collector_scan
262
+ GROUP BY machine, collector;
263
+
264
+ -- ---------------------------------------------------------------------------------------------
265
+ -- Active time (phase 2). Active time is the UNION of intervals from three sources: steer
266
+ -- islands (derived from activity_event below), ActivityWatch not-AFK window time and manual
267
+ -- entries (both rows of time_interval). Intervals are merged across all machines and agents
268
+ -- before summing, so two agents steered in parallel never count as two hours. Agent runtime is
269
+ -- never human time: only the human's own steers, window time and manual entries feed this.
270
+ -- Days are local days in the sink time zone (AUT_TZ). No view here exposes machine-level detail or titles.
271
+ -- ---------------------------------------------------------------------------------------------
272
+
273
+ -- One row per interval aut delivers: source 'aw' (ActivityWatch window time intersected with
274
+ -- not-AFK, clipped at scan range edges) or 'manual' (`aut time add`). title is PRIVATE: redacted
275
+ -- on the machine before push, and never readable by agents_ro (the public view drops it). Kept
276
+ -- for good by default; the refresh job (refresh.sql) says how to turn a limit on. Remember that
277
+ -- backups also hold titles. Same (event_id, ts) key
278
+ -- rule as usage_event. No compression policy on purpose: a later title UPDATE must stay a plain
279
+ -- row update, and compressed chunks would turn it into a decompress-and-rewrite.
280
+ -- Producers must drop pieces with end <= start before insert: aut delivers every table in one
281
+ -- transaction, so a single row failing CHECK (end_ts > ts) rolls back the whole delivery and
282
+ -- fails again on every retry of that range.
283
+ CREATE TABLE IF NOT EXISTS agents.time_interval (
284
+ event_id text NOT NULL,
285
+ ts timestamptz NOT NULL, -- interval start (hypertable partition col)
286
+ end_ts timestamptz NOT NULL,
287
+ machine text NOT NULL,
288
+ source text NOT NULL CHECK (source IN ('aw', 'manual')),
289
+ app text,
290
+ title text,
291
+ category text,
292
+ work_relevant boolean NOT NULL,
293
+ attrs jsonb NOT NULL DEFAULT '{}'::jsonb,
294
+ PRIMARY KEY (event_id, ts),
295
+ CHECK (end_ts > ts)
296
+ );
297
+
298
+ SELECT create_hypertable('agents.time_interval', 'ts', if_not_exists => TRUE);
299
+ CREATE INDEX IF NOT EXISTS time_interval_ts_end_idx ON agents.time_interval (ts, end_ts);
300
+ -- The nightly title UPDATE reads only rows that still hold a title (about one day's worth), not
301
+ -- the whole history with titles already nulled.
302
+ CREATE INDEX IF NOT EXISTS time_interval_title_ts_idx ON agents.time_interval (ts) WHERE title IS NOT NULL;
303
+
304
+ -- Everything agents_ro may read about time_interval: every column except title and event_id.
305
+ -- aut builds event_id from a hash of the title, so with machine, app, ts and end_ts beside it a
306
+ -- reader could confirm a guessed title, even after the title is nulled.
307
+ -- One-time migration: the first version exposed event_id, and CREATE OR REPLACE cannot drop a
308
+ -- column. Nothing depends on the view; the grants file (run after this file) re-grants it.
309
+ DO $$
310
+ BEGIN
311
+ IF EXISTS (SELECT 1 FROM information_schema.columns
312
+ WHERE table_schema = 'agents' AND table_name = 'time_interval_public'
313
+ AND column_name = 'event_id') THEN
314
+ EXECUTE 'DROP VIEW agents.time_interval_public';
315
+ END IF;
316
+ END $$;
317
+ CREATE OR REPLACE VIEW agents.time_interval_public AS
318
+ SELECT ts, end_ts, machine, source, app, category, work_relevant, attrs
319
+ FROM agents.time_interval;
320
+
321
+ -- Parameters of the steer rules (phase 2b). The LATEST row (by computed_at) is in force for
322
+ -- steer_lead / steer_interval and everything built on them. The 'default' row holds the
323
+ -- hand-picked phase 2 numbers; the refresh job (refresh.sql) adds a 'calibrated' row (agents.time_calibrate
324
+ -- below) only when the data is big enough and says something new. Rows are never updated or
325
+ -- deleted, so the table is the history of the parameters. No titles here: agents_ro may read it.
326
+ CREATE TABLE IF NOT EXISTS agents.time_params (
327
+ computed_at timestamptz PRIMARY KEY,
328
+ gap_minutes numeric NOT NULL, -- a new island starts after a longer idle gap
329
+ wpm numeric NOT NULL, -- typing speed behind the lead-in
330
+ lead_min_s int NOT NULL, -- lead-in clamp, seconds
331
+ lead_max_s int NOT NULL,
332
+ n_gaps int NOT NULL DEFAULT 0, -- sample sizes behind a calibrated row
333
+ n_leads int NOT NULL DEFAULT 0,
334
+ gap_agreement numeric CHECK (gap_agreement BETWEEN 0 AND 1),
335
+ window_days int NOT NULL DEFAULT 0, -- days with ActivityWatch rows in the window
336
+ source text NOT NULL CHECK (source IN ('default', 'calibrated')),
337
+ attrs jsonb NOT NULL DEFAULT '{}'::jsonb
338
+ );
339
+ INSERT INTO agents.time_params (computed_at, gap_minutes, wpm, lead_min_s, lead_max_s, source)
340
+ VALUES ('1970-01-01 00:00+00', 15, 40, 30, 600, 'default')
341
+ ON CONFLICT (computed_at) DO NOTHING;
342
+
343
+ -- Each human steer with its typing lead-in start: ts - lead, lead = words at the latest wpm
344
+ -- clamped to lead_min_s .. lead_max_s, or 1 minute when the row has no word count. A non-numeric
345
+ -- words value falls back to 1 min rather than failing every query that reads this view.
346
+ CREATE OR REPLACE VIEW agents.steer_lead AS
347
+ SELECT e.ts,
348
+ e.ts - make_interval(secs => CASE
349
+ WHEN e.attrs->>'words' ~ '^[0-9]+(\.[0-9]+)?$'
350
+ THEN least(greatest((e.attrs->>'words')::numeric * 60 / p.wpm, p.lead_min_s), p.lead_max_s)::float8
351
+ ELSE 60 END) AS start_ts
352
+ FROM agents.activity_event e
353
+ CROSS JOIN (SELECT wpm, lead_min_s, lead_max_s FROM agents.time_params ORDER BY computed_at DESC LIMIT 1) p
354
+ WHERE e.kind = 'steer';
355
+
356
+ -- Steer islands. Steers are ordered by ts across ALL agents and machines; a new island starts
357
+ -- when a steer's lead-in starts more than gap_minutes (latest time_params row) after the previous
358
+ -- steer. Island = [min(lead-in start), max(ts)].
359
+ CREATE OR REPLACE VIEW agents.steer_interval AS
360
+ WITH g AS (
361
+ SELECT ts, start_ts,
362
+ count(*) FILTER (WHERE prev_ts IS NULL OR start_ts > prev_ts + gap)
363
+ OVER (ORDER BY ts ROWS UNBOUNDED PRECEDING) AS island
364
+ FROM (SELECT l.ts, l.start_ts, lag(l.ts) OVER (ORDER BY l.ts) AS prev_ts,
365
+ p.gap_minutes * interval '1 minute' AS gap
366
+ FROM agents.steer_lead l
367
+ CROSS JOIN (SELECT gap_minutes FROM agents.time_params ORDER BY computed_at DESC LIMIT 1) p) x
368
+ )
369
+ SELECT min(start_ts) AS start_ts, max(ts) AS end_ts, count(*) AS steers
370
+ FROM g
371
+ GROUP BY island;
372
+
373
+ -- Apps learned as work (phase 2b): ActivityWatch apps in focus at the moment of at least 3
374
+ -- steers on at least 2 local days (AUT_TZ). An app is never learned when ANY of its aw rows is a
375
+ -- browser (rule browser_*, the title rules decide there) or comes from the user's own mapping
376
+ -- (rule config_app: that mapping always wins, also over the app's rows from before it). The app in
377
+ -- focus is the newest row on the steer's own machine, else the newest row on any machine (machine
378
+ -- is used here only, never exposed).
379
+ -- shortcut: looks at the 20 newest aw rows starting at or before each steer, so with many
380
+ -- machines interleaving short rows a long focus row can be missed; switch to a per-machine lookup
381
+ -- when more than about 3 machines run ActivityWatch at once.
382
+ CREATE OR REPLACE VIEW agents.app_learned AS
383
+ SELECT f.app, count(*) AS n_steers,
384
+ count(DISTINCT (s.ts AT TIME ZONE '@AUT_TZ@')::date) AS n_days,
385
+ min(s.ts) AS first_seen
386
+ FROM agents.activity_event s
387
+ CROSS JOIN LATERAL (
388
+ SELECT c.app FROM (
389
+ SELECT t.app, t.machine, t.ts, t.end_ts
390
+ FROM agents.time_interval t
391
+ WHERE t.source = 'aw' AND t.ts <= s.ts
392
+ ORDER BY t.ts DESC LIMIT 20) c
393
+ WHERE c.end_ts > s.ts
394
+ ORDER BY c.machine = s.machine DESC NULLS LAST, c.ts DESC LIMIT 1) f
395
+ WHERE s.kind = 'steer' AND f.app IS NOT NULL
396
+ GROUP BY f.app
397
+ HAVING count(*) >= 3
398
+ AND count(DISTINCT (s.ts AT TIME ZONE '@AUT_TZ@')::date) >= 2
399
+ AND f.app NOT IN (SELECT x.app FROM agents.time_interval x
400
+ WHERE x.source = 'aw' AND x.app IS NOT NULL
401
+ AND (x.attrs->>'rule' LIKE 'browser\_%' OR x.attrs->>'rule' = 'config_app'));
402
+
403
+ -- time_interval as the active-time views see it: a row the collector could not place (rule
404
+ -- unknown or macos_category) counts as work when its app is learned. Every other row keeps the
405
+ -- collector's decision. No title, no title-derived event_id (same columns as time_interval_public).
406
+ CREATE OR REPLACE VIEW agents.time_interval_effective AS
407
+ SELECT t.ts, t.end_ts, t.machine, t.source, t.app, t.category,
408
+ t.work_relevant OR (t.attrs->>'rule' IN ('unknown', 'macos_category')
409
+ AND t.app IN (SELECT app FROM agents.app_learned)) AS work_relevant,
410
+ t.attrs
411
+ FROM agents.time_interval t;
412
+
413
+ -- Merged active intervals, one row per merged interval per variant. 'all' = steer islands + all
414
+ -- time_interval rows; 'relevant' = steer islands + work_relevant rows of time_interval_effective
415
+ -- (learned apps included). Overlapping or touching intervals merge (gaps and islands on start
416
+ -- with a running max of end). sources lists which of steer / aw / manual contributed. Re-merges
417
+ -- the whole history on every read (slow over months of data, see active_daily below): drill-down only.
418
+ CREATE OR REPLACE VIEW agents.active_interval AS
419
+ WITH src AS (
420
+ SELECT v.variant, s.start_ts, s.end_ts, 'steer'::text AS source
421
+ FROM agents.steer_interval s CROSS JOIN (VALUES ('all'), ('relevant')) AS v(variant)
422
+ UNION ALL
423
+ SELECT 'all', ts, end_ts, source FROM agents.time_interval
424
+ UNION ALL
425
+ SELECT 'relevant', ts, end_ts, source FROM agents.time_interval_effective WHERE work_relevant
426
+ ), m AS (
427
+ SELECT variant, start_ts, end_ts, source,
428
+ max(end_ts) OVER (PARTITION BY variant ORDER BY start_ts, end_ts
429
+ ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) AS prev_end
430
+ FROM src
431
+ ), g AS (
432
+ SELECT variant, start_ts, end_ts, source,
433
+ count(*) FILTER (WHERE prev_end IS NULL OR start_ts > prev_end)
434
+ OVER (PARTITION BY variant ORDER BY start_ts, end_ts ROWS UNBOUNDED PRECEDING) AS grp
435
+ FROM m
436
+ )
437
+ SELECT variant, min(start_ts) AS start_ts, max(end_ts) AS end_ts,
438
+ array_agg(DISTINCT source ORDER BY source) AS sources
439
+ FROM g
440
+ GROUP BY variant, grp;
441
+
442
+ -- Minutes per local day (sink time zone, AUT_TZ), computed live. minutes_all / minutes_relevant are the merged
443
+ -- totals; the per-source columns are summed BEFORE merging (so they can add up to more than
444
+ -- minutes_all) and exist for diagnosis only. An interval that crosses local midnight is split at
445
+ -- the boundary, DST days included (the day bounds are local midnights converted back to
446
+ -- timestamptz). Exact but slow over a long history (every read re-merges all of it); Grafana
447
+ -- reads the materialized agents.active_daily below.
448
+ CREATE OR REPLACE VIEW agents.active_daily_live AS
449
+ WITH pieces AS (
450
+ SELECT variant AS bucket, start_ts, end_ts FROM agents.active_interval
451
+ UNION ALL
452
+ SELECT 'steer', start_ts, end_ts FROM agents.steer_interval
453
+ UNION ALL
454
+ SELECT source, ts, end_ts FROM agents.time_interval
455
+ ), split AS (
456
+ SELECT p.bucket, d::date AS day,
457
+ greatest(p.start_ts, d AT TIME ZONE '@AUT_TZ@') AS s,
458
+ least(p.end_ts, (d + interval '1 day') AT TIME ZONE '@AUT_TZ@') AS e
459
+ FROM pieces p
460
+ CROSS JOIN LATERAL generate_series(
461
+ date_trunc('day', p.start_ts AT TIME ZONE '@AUT_TZ@'),
462
+ date_trunc('day', p.end_ts AT TIME ZONE '@AUT_TZ@'),
463
+ interval '1 day') AS d
464
+ )
465
+ SELECT day,
466
+ round(coalesce(sum(extract(epoch FROM e - s)) FILTER (WHERE bucket = 'all'), 0) / 60, 1) AS minutes_all,
467
+ round(coalesce(sum(extract(epoch FROM e - s)) FILTER (WHERE bucket = 'relevant'), 0) / 60, 1) AS minutes_relevant,
468
+ round(coalesce(sum(extract(epoch FROM e - s)) FILTER (WHERE bucket = 'steer'), 0) / 60, 1) AS minutes_steer,
469
+ round(coalesce(sum(extract(epoch FROM e - s)) FILTER (WHERE bucket = 'aw'), 0) / 60, 1) AS minutes_aw,
470
+ round(coalesce(sum(extract(epoch FROM e - s)) FILTER (WHERE bucket = 'manual'), 0) / 60, 1) AS minutes_manual
471
+ FROM split
472
+ WHERE e > s
473
+ GROUP BY day;
474
+
475
+ -- agents.active_daily: active_daily_live materialized, so a 12-month Grafana panel reads ~365
476
+ -- rows instead of re-merging every interval. With a year of ActivityWatch data the live views
477
+ -- take seconds per read while a read of active_daily takes well under a millisecond. So
478
+ -- active_interval and active_daily_live are drill-down only (agents_ro may read them, but no
479
+ -- dashboard panel should sit on them).
480
+ -- Refreshed CONCURRENTLY (readers never block) by the refresh job (refresh.sql); the unique index on
481
+ -- day is what CONCURRENTLY needs.
482
+ -- shortcut: refreshed on refresh.sql's schedule, so today and late-pushed days lag until the next run;
483
+ -- add an hourly refresh entry, or read active_daily_live for recent days, when a live panel matters.
484
+ CREATE MATERIALIZED VIEW IF NOT EXISTS agents.active_daily AS
485
+ SELECT * FROM agents.active_daily_live
486
+ WITH DATA;
487
+ CREATE UNIQUE INDEX IF NOT EXISTS active_daily_day_idx ON agents.active_daily (day);
488
+
489
+ -- ---------------------------------------------------------------------------------------------
490
+ -- Calibration (phase 2b): the steer rules learn from ActivityWatch. Run by
491
+ -- the refresh job BEFORE the active_daily refresh. Window: aw rows and steers in the 90
492
+ -- days before p_end; window_days counts the local days that have aw rows.
493
+ --
494
+ -- Gap limit. A pair is two consecutive steers (all agents and machines); its gap runs from the
495
+ -- earlier steer to the later steer's lead-in start, which is what steer_interval compares with
496
+ -- gap_minutes. The lead-in uses the wpm and lead bounds measured in this same run (the latest
497
+ -- row's when there are no leads), so the gap limit is chosen against the lead-ins stored with it. Pairs with gap <= 0 (lead-in overlaps,
498
+ -- joined whatever the limit) or over 120 minutes are skipped, and so is a pair outside its days'
499
+ -- aw span (it must start after that day's first aw row and end before its own day's last aw row
500
+ -- end), so a day the collector only covered in part does not read as away. A gap is present when
501
+ -- aw rows (any app, any category, overlaps counted once) cover at least half of it. The limit is
502
+ -- the candidate 5..60 minutes with the best agreement between (gap <= limit) and present; ties go
503
+ -- to the smaller limit.
504
+ --
505
+ -- Typing speed. For steers with words >= 5 whose ts falls inside an aw row, the lead is how long
506
+ -- that row's app had been in focus on that machine right before ts (rows of the same app with
507
+ -- gaps under 30 s chain into one run), capped at 15 minutes. Focus time is an upper bound on
508
+ -- typing time (reading agent output in the same app counts too), so a lead slower than 10 wpm
509
+ -- (the wpm floor) or at the 15-minute cap is dropped as mostly reading; n_leads counts the leads
510
+ -- kept. wpm = median(words / lead minutes) of those, clamped to 10..120 and rounded; lead_min_s /
511
+ -- lead_max_s = 5th / 95th percentile of the kept leads, clamped to 15..900 s. When a steer sits
512
+ -- inside rows on two machines, the steer's own machine wins, else the run that started last.
513
+ -- ---------------------------------------------------------------------------------------------
514
+ CREATE OR REPLACE FUNCTION agents.time_calibration(p_end timestamptz DEFAULT now())
515
+ RETURNS TABLE (gap_minutes numeric, wpm numeric, lead_min_s int, lead_max_s int, n_gaps int,
516
+ n_leads int, gap_agreement numeric, window_days int, agreement_at_latest numeric)
517
+ LANGUAGE sql STABLE AS $$
518
+ WITH w AS (
519
+ SELECT p_end - interval '90 days' AS w_start
520
+ ), latest AS (
521
+ SELECT * FROM agents.time_params ORDER BY computed_at DESC LIMIT 1
522
+ ), aw AS (
523
+ SELECT t.machine, t.ts, t.end_ts, t.app
524
+ FROM agents.time_interval t, w
525
+ WHERE t.source = 'aw' AND t.ts >= w.w_start AND t.ts < p_end
526
+ ), days AS (
527
+ SELECT (ts AT TIME ZONE '@AUT_TZ@')::date AS day, min(ts) AS first_ts, max(end_ts) AS last_end
528
+ FROM aw GROUP BY 1
529
+ ), steers AS (
530
+ SELECT e.event_id, e.ts, e.machine,
531
+ CASE WHEN e.attrs->>'words' ~ '^[0-9]+(\.[0-9]+)?$' THEN (e.attrs->>'words')::numeric END AS words
532
+ FROM agents.activity_event e, w
533
+ WHERE e.kind = 'steer' AND e.ts >= w.w_start AND e.ts < p_end
534
+ ), runs AS (
535
+ SELECT machine, ts, end_ts, min(ts) OVER (PARTITION BY machine, run) AS run_start
536
+ FROM (SELECT machine, ts, end_ts,
537
+ count(*) FILTER (WHERE prev_app IS DISTINCT FROM app OR ts - prev_end >= interval '30 seconds')
538
+ OVER (PARTITION BY machine ORDER BY ts, end_ts) AS run
539
+ FROM (SELECT machine, ts, end_ts, app,
540
+ lag(app) OVER m AS prev_app, lag(end_ts) OVER m AS prev_end
541
+ FROM aw WINDOW m AS (PARTITION BY machine ORDER BY ts, end_ts)) a) r
542
+ ), ev AS (
543
+ -- aw rows (k = 0) and word-counted steers (k = 1, once per machine) on one timeline, so each
544
+ -- steer finds the newest row starting at or before it by sorting, not by a range join.
545
+ SELECT machine, ts, 0 AS k, end_ts, run_start, NULL::text AS sid, NULL::text AS smachine,
546
+ NULL::numeric AS words FROM runs
547
+ UNION ALL
548
+ SELECT m.machine, s.ts, 1, NULL, NULL, s.event_id, s.machine, s.words
549
+ FROM steers s CROSS JOIN (SELECT DISTINCT machine FROM aw) m
550
+ WHERE s.words >= 5
551
+ ), grp AS (
552
+ SELECT *, count(*) FILTER (WHERE k = 0) OVER (PARTITION BY machine ORDER BY ts, k) AS g FROM ev
553
+ ), hit AS (
554
+ SELECT k, sid, ts, words, machine, smachine,
555
+ max(end_ts) OVER (PARTITION BY machine, g) AS row_end,
556
+ max(run_start) OVER (PARTITION BY machine, g) AS run_start
557
+ FROM grp
558
+ ), leads AS (
559
+ SELECT DISTINCT ON (sid, ts) words, least(extract(epoch FROM ts - run_start), 900) AS lead_s
560
+ FROM hit
561
+ WHERE k = 1 AND row_end > ts AND run_start < ts
562
+ ORDER BY sid, ts, machine = smachine DESC NULLS LAST, run_start DESC
563
+ ), lstats AS (
564
+ SELECT count(*)::int AS n_leads,
565
+ round(least(greatest(percentile_cont(0.5) WITHIN GROUP (ORDER BY words * 60 / lead_s), 10), 120)::numeric) AS wpm,
566
+ round(least(greatest(percentile_cont(0.05) WITHIN GROUP (ORDER BY lead_s), 15), 900))::int AS lead_min_s,
567
+ round(least(greatest(percentile_cont(0.95) WITHIN GROUP (ORDER BY lead_s), 15), 900))::int AS lead_max_s
568
+ FROM leads
569
+ WHERE lead_s < 900 AND lead_s <= words * 6
570
+ ), lp AS (
571
+ SELECT coalesce(ls.wpm, lt.wpm) AS wpm, coalesce(ls.lead_min_s, lt.lead_min_s) AS lo,
572
+ coalesce(ls.lead_max_s, lt.lead_max_s) AS hi
573
+ FROM lstats ls, latest lt
574
+ ), pairs AS (
575
+ -- lead-in start: the steer_lead formula with this run's lead parameters
576
+ SELECT l.prev_ts, l.start_ts
577
+ FROM (SELECT s.ts, lag(s.ts) OVER (ORDER BY s.ts) AS prev_ts,
578
+ s.ts - make_interval(secs => CASE WHEN s.words IS NOT NULL
579
+ THEN least(greatest(s.words * 60 / lp.wpm, lp.lo), lp.hi)::float8 ELSE 60 END) AS start_ts
580
+ FROM steers s, lp) l
581
+ JOIN days d1 ON d1.day = (l.prev_ts AT TIME ZONE '@AUT_TZ@')::date AND l.prev_ts >= d1.first_ts
582
+ JOIN days d2 ON d2.day = (l.start_ts AT TIME ZONE '@AUT_TZ@')::date AND l.start_ts <= d2.last_end
583
+ WHERE l.start_ts > l.prev_ts AND l.start_ts <= l.prev_ts + interval '120 minutes'
584
+ ), long_rows AS MATERIALIZED (
585
+ SELECT ts, end_ts FROM aw WHERE end_ts - ts > interval '2 hours'
586
+ ), cov AS (
587
+ -- Union length of the aw rows inside each gap: each clipped row adds only what lies past the
588
+ -- running max end of the rows before it (so a row found twice adds nothing the second time).
589
+ -- The lower bound on t.ts (longest window row, at most 2 hours) keeps each index lookup short;
590
+ -- the few rows longer than 2 hours come from long_rows instead.
591
+ SELECT prev_ts, start_ts, sum(piece) AS covered
592
+ FROM (SELECT p.prev_ts, p.start_ts,
593
+ greatest(least(t.end_ts, p.start_ts) - greatest(t.ts, p.prev_ts,
594
+ max(least(t.end_ts, p.start_ts)) OVER (
595
+ PARTITION BY p.prev_ts, p.start_ts ORDER BY t.ts, t.end_ts
596
+ ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)), interval '0') AS piece
597
+ FROM pairs p
598
+ CROSS JOIN LATERAL (
599
+ SELECT t.ts, t.end_ts FROM agents.time_interval t
600
+ WHERE t.source = 'aw' AND t.ts < p.start_ts AND t.end_ts > p.prev_ts
601
+ AND t.ts >= (SELECT w_start FROM w)
602
+ AND t.ts >= p.prev_ts - (SELECT least(max(end_ts - ts), interval '2 hours') FROM aw)
603
+ UNION ALL
604
+ SELECT r.ts, r.end_ts FROM long_rows r WHERE r.ts < p.start_ts AND r.end_ts > p.prev_ts) t) x
605
+ GROUP BY prev_ts, start_ts
606
+ ), gaps AS (
607
+ SELECT p.start_ts - p.prev_ts AS g,
608
+ coalesce(c.covered, interval '0') * 2 >= p.start_ts - p.prev_ts AS present
609
+ FROM pairs p LEFT JOIN cov c USING (prev_ts, start_ts)
610
+ ), best AS (
611
+ SELECT l, avg(((g <= make_interval(mins => l)) = present)::int) AS agreement
612
+ FROM gaps CROSS JOIN generate_series(5, 60) AS l
613
+ GROUP BY l
614
+ ORDER BY agreement DESC, l
615
+ LIMIT 1
616
+ )
617
+ SELECT b.l::numeric, ls.wpm, ls.lead_min_s, ls.lead_max_s,
618
+ (SELECT count(*) FROM gaps)::int, ls.n_leads, round(b.agreement, 4),
619
+ (SELECT count(*) FROM days)::int,
620
+ (SELECT round(avg(((g.g <= lt.gap_minutes * interval '1 minute') = g.present)::int), 4) FROM gaps g, latest lt)
621
+ FROM lstats ls LEFT JOIN best b ON true
622
+ $$;
623
+
624
+ -- Writes a 'calibrated' row only when n_gaps >= 200, n_leads >= 100 and window_days >= 14 (two
625
+ -- weeks of ActivityWatch before a row restates all history) and some parameter changes
626
+ -- materially; otherwise nothing (the last row stays in force). Hysteresis, so the window sliding
627
+ -- by a day or a flat agreement curve does not restate history every night: the gap limit moves
628
+ -- only when its agreement beats the latest limit's (agreement_at_latest, same gaps) by 0.02; wpm
629
+ -- only when it moves by 10%; the two lead bounds together when either moves by 10%. Unmoved
630
+ -- parameters keep the latest row's values. Returns rows written (0 or 1). computed_at is
631
+ -- clock_timestamp(), so two runs in one transaction never share a key.
632
+ CREATE OR REPLACE FUNCTION agents.time_calibrate(p_end timestamptz DEFAULT now())
633
+ RETURNS int
634
+ LANGUAGE sql VOLATILE AS $$
635
+ WITH c AS (
636
+ SELECT * FROM agents.time_calibration(p_end)
637
+ ), latest AS (
638
+ SELECT * FROM agents.time_params ORDER BY computed_at DESC LIMIT 1
639
+ ), v AS (
640
+ SELECT CASE WHEN gap_move THEN c.gap_minutes ELSE l.gap_minutes END AS gap_minutes,
641
+ CASE WHEN gap_move THEN c.gap_agreement ELSE c.agreement_at_latest END AS gap_agreement,
642
+ CASE WHEN abs(c.wpm - l.wpm) >= 0.1 * l.wpm THEN c.wpm ELSE l.wpm END AS wpm,
643
+ CASE WHEN lead_move THEN c.lead_min_s ELSE l.lead_min_s END AS lead_min_s,
644
+ CASE WHEN lead_move THEN c.lead_max_s ELSE l.lead_max_s END AS lead_max_s,
645
+ c.n_gaps, c.n_leads, c.window_days, c.agreement_at_latest
646
+ FROM c, latest l,
647
+ LATERAL (SELECT c.gap_agreement >= c.agreement_at_latest + 0.02 AS gap_move,
648
+ abs(c.lead_min_s - l.lead_min_s) >= 0.1 * l.lead_min_s
649
+ OR abs(c.lead_max_s - l.lead_max_s) >= 0.1 * l.lead_max_s AS lead_move) m
650
+ WHERE c.n_gaps >= 200 AND c.n_leads >= 100 AND c.window_days >= 14
651
+ ), ins AS (
652
+ INSERT INTO agents.time_params (computed_at, gap_minutes, wpm, lead_min_s, lead_max_s, n_gaps,
653
+ n_leads, gap_agreement, window_days, source, attrs)
654
+ SELECT clock_timestamp(), v.gap_minutes, v.wpm, v.lead_min_s, v.lead_max_s, v.n_gaps,
655
+ v.n_leads, v.gap_agreement, v.window_days, 'calibrated',
656
+ jsonb_build_object('window_end', p_end, 'agreement_at_previous', v.agreement_at_latest)
657
+ FROM v, latest
658
+ WHERE (v.gap_minutes, v.wpm, v.lead_min_s, v.lead_max_s)
659
+ IS DISTINCT FROM (latest.gap_minutes, latest.wpm, latest.lead_min_s, latest.lead_max_s)
660
+ RETURNING 1
661
+ )
662
+ SELECT count(*)::int FROM ins
663
+ $$;
664
+
665
+ -- ---------------------------------------------------------------------------------------------
666
+ -- Episodes and outcomes (phase 3). An episode is one piece of work across sessions, days and
667
+ -- machines: the interactive sessions that built it, the subagents and scripted runs they
668
+ -- started, the commits and PRs it made, and what became of them. Nothing here holds a URL,
669
+ -- path, title, branch name, ticket id or commit subject: the cross-session keys (repo_key,
670
+ -- branch_key, pr_keys) are keyed hashes aut makes with its per-install salt, SHAs are ids.
671
+
672
+ -- Git outcome per commit, written by `aut outcomes` on a collector machine. Insert-only: a commit moves
673
+ -- through states by new rows and readers take the latest decided_at per sha. survival is the
674
+ -- share (0..1) of the commit's added lines still attributed to it 14 days after the merge.
675
+ -- sha may be the commit row's abbreviation or the full SHA: the linker matches its first 7 hex
676
+ -- digits. A commit that never gets a final state keeps its episode 'undecided', and that holds
677
+ -- its month out of episode_monthly, so every commit should end in a final state.
678
+ CREATE TABLE IF NOT EXISTS agents.commit_outcome (
679
+ sha text NOT NULL,
680
+ repo_key text NOT NULL,
681
+ state text NOT NULL
682
+ CHECK (state IN ('merged', 'reverted', 'reworked', 'open', 'abandoned', 'superseded', 'undecided')),
683
+ survival numeric CHECK (survival BETWEEN 0 AND 1),
684
+ decided_at timestamptz NOT NULL,
685
+ PRIMARY KEY (sha, repo_key, state, decided_at)
686
+ );
687
+
688
+ -- The manual /outcome (`aut outcome shipped|ruled-out|open`) on one session. Insert-only; the
689
+ -- latest row of an episode's seed sessions wins over git. repo_key (the current directory's
690
+ -- repo) limits it to that session's unit in that repo; NULL, or a repo the session has no unit
691
+ -- for, means every unit of the session.
692
+ CREATE TABLE IF NOT EXISTS agents.manual_outcome (
693
+ session_id text NOT NULL,
694
+ outcome text NOT NULL CHECK (outcome IN ('shipped', 'ruled_out', 'open')),
695
+ ts timestamptz NOT NULL DEFAULT now(),
696
+ PRIMARY KEY (session_id, ts)
697
+ );
698
+ ALTER TABLE agents.manual_outcome ADD COLUMN IF NOT EXISTS repo_key text;
699
+
700
+ -- Private hand corrections from `aut episodes review`, keys only. Applied by the linker after
701
+ -- every automatic edge, in ts order: merge joins the episodes of a_session and b_session; split
702
+ -- takes a_session's units out of their episode (one new episode per episode they were in);
703
+ -- reject drops a_session's episode (not an episode); confirm marks it as checked. A seed
704
+ -- session holds one unit per repo, possibly in different episodes, so the page names the
705
+ -- reviewed units: attrs.repo_keys (a_session) and attrs.b_repo_keys (b_session), copied from
706
+ -- episode_member.repo_keys. Without them a correction acts on every unit of the session.
707
+ CREATE TABLE IF NOT EXISTS agents.episode_correction (
708
+ id uuid PRIMARY KEY,
709
+ ts timestamptz NOT NULL DEFAULT now(),
710
+ action text NOT NULL CHECK (action IN ('merge', 'split', 'confirm', 'reject')),
711
+ a_session text NOT NULL,
712
+ b_session text,
713
+ attrs jsonb NOT NULL DEFAULT '{}'::jsonb
714
+ );
715
+
716
+ -- Linker output, rebuilt in full every night by agents.episode_link() (derived data: the
717
+ -- inputs are activity_event, event_session and the three tables above).
718
+ CREATE TABLE IF NOT EXISTS agents.episode (
719
+ episode_id uuid PRIMARY KEY,
720
+ first_ts timestamptz NOT NULL,
721
+ last_ts timestamptz NOT NULL,
722
+ n_sessions integer NOT NULL,
723
+ n_seed_sessions integer NOT NULL,
724
+ n_steers integer NOT NULL,
725
+ n_commits integer NOT NULL,
726
+ n_prs integer NOT NULL,
727
+ outcome text NOT NULL
728
+ CHECK (outcome IN ('shipped', 'reworked', 'open', 'ruled_out', 'no_code', 'undecided')),
729
+ outcome_source text NOT NULL CHECK (outcome_source IN ('git', 'manual')),
730
+ confirmed boolean NOT NULL
731
+ );
732
+ CREATE INDEX IF NOT EXISTS episode_first_ts_idx ON agents.episode (first_ts);
733
+
734
+ CREATE TABLE IF NOT EXISTS agents.episode_member (
735
+ episode_id uuid NOT NULL REFERENCES agents.episode (episode_id) ON DELETE CASCADE,
736
+ session_id text NOT NULL,
737
+ role text NOT NULL CHECK (role IN ('seed', 'child')),
738
+ PRIMARY KEY (episode_id, session_id)
739
+ );
740
+ -- The repos of the session's units in this episode (a child: its root's), for corrections.
741
+ ALTER TABLE agents.episode_member ADD COLUMN IF NOT EXISTS repo_keys text[] NOT NULL DEFAULT '{}';
742
+ CREATE INDEX IF NOT EXISTS episode_member_session_idx ON agents.episode_member (session_id);
743
+
744
+ -- The linker. Unit = one seed session x one repo_key (a seed is an interactive session: not a
745
+ -- subagent, not a scripted run, not marked role 'child' by aut). A seed session whose rows carry
746
+ -- no repo_key gets one '' unit; within a unit, session continuity is the glue. Edges, in trust
747
+ -- order: 1 a shared commit key (a link row's commit_keys: a commit its turn printed, or a seed
748
+ -- link row's sha_refs key that some turn printed; both are HMACs of a SHA's first 7 hex, so no
749
+ -- raw SHA joins), 2 a shared pr key (link pr_keys, or a pr row's host + repo_key + number),
750
+ -- 3 a shared branch_key whose next use comes within 14 days of the last. Ticket keys are no
751
+ -- edge: they may only confirm a pair another edge already joins, which changes nothing.
752
+ -- Union-find refuses an edge that would join two episodes whose spans lie more than 60 days
753
+ -- apart, so one long-lived key cannot grow a giant episode. A quoted SHA or a pr key is a soft
754
+ -- key: a unit whose soft keys reach more than 2 other units (git log output, a review, a
755
+ -- PR-watching session) is a hub and joins only the nearest of them in time, so it never fuses
756
+ -- the episodes it looked at. A commit row's SHA counts only as 7 to 40 hex digits and serves
757
+ -- the episode's commits and outcome only, never an edge. Children (subagents, scripted runs)
758
+ -- never seed or bridge: they attach to their root parent's unit, and of their rows only the
759
+ -- commits they printed count (owned by that unit). Corrections apply last. Episode id = md5 of
760
+ -- the earliest unit (session_id | repo_key), so it is stable while that unit is. Outcome: the
761
+ -- latest manual outcome wins; else reworked if any commit was reverted or reworked, shipped if
762
+ -- any merged (survival >= 0.5 where known), open if any is open, ruled_out if every commit was
763
+ -- abandoned (superseded pre-amend SHAs aside), undecided while commits wait for a decision, and
764
+ -- no_code for episodes without commits (research, review). Runs as postgres from the
765
+ -- refresh job; returns one row of counts.
766
+ CREATE OR REPLACE FUNCTION agents.episode_link()
767
+ RETURNS TABLE (units int, edges int, refused int, episodes int, members int)
768
+ LANGUAGE plpgsql VOLATILE AS $$
769
+ DECLARE
770
+ n int;
771
+ i int;
772
+ ra int;
773
+ rb int;
774
+ tgt int;
775
+ next_label int;
776
+ par int[];
777
+ labs int[];
778
+ f timestamptz[];
779
+ l timestamptz[];
780
+ ed record;
781
+ cr record;
782
+ BEGIN
783
+ PERFORM pg_advisory_xact_lock(hashtext('agents.derived_refresh'));
784
+ -- Scratch tables are dropped at the end; a failed run rolls their creation back with it.
785
+ refused := 0;
786
+
787
+ -- sha: a commit row's SHA when it is one (live rows hold junk like '1' that would join
788
+ -- unrelated sessions).
789
+ CREATE TEMP TABLE _el_row AS
790
+ SELECT session_id, ts, kind, nullif(attrs->>'repo_key', '') AS repo_key, attrs,
791
+ CASE WHEN kind = 'commit' AND lower(attrs->>'sha') ~ '^[0-9a-f]{7,40}$'
792
+ THEN lower(attrs->>'sha') END AS sha
793
+ FROM agents.activity_event
794
+ WHERE kind IN ('link', 'commit', 'pr', 'steer') AND session_id IS NOT NULL;
795
+
796
+ -- Seed or child, and the root parent a child attaches to (up to 5 levels of nesting).
797
+ CREATE TEMP TABLE _el_sess AS
798
+ WITH RECURSIVE es AS (
799
+ SELECT session_id, bool_or(run_origin IN ('subagent', 'automated')) AS scripted,
800
+ max(parent_session_id) AS parent
801
+ FROM agents.event_session GROUP BY session_id
802
+ ), ids AS (
803
+ SELECT DISTINCT session_id FROM _el_row
804
+ ), marked AS (
805
+ SELECT DISTINCT session_id FROM _el_row WHERE kind = 'link' AND attrs->>'role' = 'child'
806
+ ), up AS (
807
+ SELECT session_id, session_id AS root, 0 AS depth FROM ids
808
+ UNION ALL
809
+ SELECT up.session_id, es.parent, up.depth + 1
810
+ FROM up JOIN es ON es.session_id = up.root
811
+ WHERE es.parent IS NOT NULL AND up.depth < 5
812
+ ), top AS (
813
+ SELECT DISTINCT ON (session_id) session_id, root FROM up ORDER BY session_id, depth DESC
814
+ )
815
+ SELECT d.session_id, t.root,
816
+ coalesce(es.scripted, false) OR es.parent IS NOT NULL OR m.session_id IS NOT NULL AS is_child
817
+ FROM ids d JOIN top t USING (session_id)
818
+ LEFT JOIN es USING (session_id) LEFT JOIN marked m USING (session_id);
819
+
820
+ CREATE TEMP TABLE _el_unit AS
821
+ WITH su AS (
822
+ SELECT r.session_id, coalesce(r.repo_key, '') AS repo_key
823
+ FROM _el_row r JOIN _el_sess s USING (session_id)
824
+ WHERE NOT s.is_child AND r.kind <> 'steer'
825
+ GROUP BY 1, 2
826
+ )
827
+ SELECT (row_number() OVER (ORDER BY session_id, repo_key))::int AS uid, session_id, repo_key
828
+ FROM su
829
+ WHERE repo_key <> ''
830
+ OR NOT EXISTS (SELECT 1 FROM su s2 WHERE s2.session_id = su.session_id AND s2.repo_key <> '');
831
+ CREATE INDEX ON _el_unit (session_id, repo_key);
832
+
833
+ -- Where each seed session was, repo by repo, over time.
834
+ CREATE TEMP TABLE _el_pos AS
835
+ SELECT r.session_id, r.ts, u.uid
836
+ FROM _el_row r JOIN _el_unit u ON u.session_id = r.session_id AND u.repo_key = r.repo_key
837
+ WHERE r.kind <> 'steer';
838
+ CREATE INDEX ON _el_pos (session_id);
839
+
840
+ -- Every row to a unit of its owner (itself, or a child's root): the unit of the row's repo,
841
+ -- else (no repo, or a repo the owner has no unit for) the owner's unit nearest in time, so a
842
+ -- turn whose repo went unresolved stays with the work around it, else the owner's '' unit.
843
+ -- A child whose root has no unit is in no episode. shortcut: a scripted run with no parent
844
+ -- session (none of them carry one today) is dropped with its commits, so the interactive
845
+ -- episode it ran inside can read no_code; attach it by repo_key + time window to the
846
+ -- overlapping seed unit if such commits ever matter.
847
+ CREATE TEMP TABLE _el_asg AS
848
+ WITH o AS (
849
+ SELECT r.session_id, r.ts, r.kind, r.repo_key, r.attrs, r.sha,
850
+ CASE WHEN s.is_child THEN 'child' ELSE 'seed' END AS role,
851
+ CASE WHEN s.is_child THEN s.root ELSE r.session_id END AS owner
852
+ FROM _el_row r JOIN _el_sess s USING (session_id)
853
+ WHERE NOT (s.is_child AND r.kind = 'steer')
854
+ )
855
+ SELECT o.*, coalesce(
856
+ (SELECT u.uid FROM _el_unit u WHERE u.session_id = o.owner AND u.repo_key = o.repo_key),
857
+ (SELECT p.uid FROM _el_pos p WHERE p.session_id = o.owner
858
+ ORDER BY abs(extract(epoch FROM p.ts - o.ts)), p.uid LIMIT 1),
859
+ (SELECT u.uid FROM _el_unit u WHERE u.session_id = o.owner AND u.repo_key = '')) AS uid
860
+ FROM o;
861
+ DELETE FROM _el_asg WHERE uid IS NULL;
862
+
863
+ CREATE TEMP TABLE _el_span AS
864
+ SELECT uid, min(ts) AS t0, max(ts) AS t1 FROM _el_asg GROUP BY uid;
865
+
866
+ -- Edge keys. Children add only their printed commits (commit_keys); their quoted SHAs, PRs
867
+ -- and branches never bridge. shortcut: a commit key is the HMAC of a SHA's first 7 hex
868
+ -- digits (aut stores 7 to 9), so two different commits that share 7 digits would join
869
+ -- (about a 5% chance of one such pair over 5k commits); key longer prefixes in aut if a
870
+ -- wrong join is ever found. Only 64-hex keys count: link rows written before aut keyed
871
+ -- them hold raw hex sha_refs and no commit_keys, and join on nothing. So history only joins
872
+ -- by SHA once aut has sent its link rows (aut backfill --activity-only); a session whose
873
+ -- transcript was deleted before that keeps its commits and outcome but is never SHA-linked.
874
+ -- soft: a quoted SHA or a pr key (bridges only through the hub rule below); hard: a
875
+ -- printed commit or a branch.
876
+ CREATE TEMP TABLE _el_key AS
877
+ WITH printed AS (
878
+ SELECT a.uid, a.ts, x AS k
879
+ FROM _el_asg a, jsonb_array_elements_text(CASE WHEN jsonb_typeof(a.attrs->'commit_keys') = 'array'
880
+ THEN a.attrs->'commit_keys' ELSE '[]' END) x
881
+ WHERE a.kind = 'link' AND x ~ '^[0-9a-f]{64}$'
882
+ )
883
+ SELECT 1 AS tier, false AS soft, uid, ts, k FROM printed
884
+ UNION ALL
885
+ SELECT 1, true, a.uid, a.ts, x
886
+ FROM _el_asg a, jsonb_array_elements_text(CASE WHEN jsonb_typeof(a.attrs->'sha_refs') = 'array'
887
+ THEN a.attrs->'sha_refs' ELSE '[]' END) x
888
+ WHERE a.kind = 'link' AND a.role = 'seed' AND x ~ '^[0-9a-f]{64}$'
889
+ AND x IN (SELECT k FROM printed)
890
+ UNION ALL
891
+ SELECT 2, true, a.uid, a.ts, x
892
+ FROM _el_asg a, jsonb_array_elements_text(CASE WHEN jsonb_typeof(a.attrs->'pr_keys') = 'array'
893
+ THEN a.attrs->'pr_keys' ELSE '[]' END) x
894
+ WHERE a.kind = 'link' AND a.role = 'seed'
895
+ UNION ALL
896
+ SELECT 2, true, uid, ts, concat_ws(':', 'pr-row', attrs->>'host', attrs->>'repo_key', attrs->>'number')
897
+ FROM _el_asg WHERE kind = 'pr' AND role = 'seed'
898
+ UNION ALL
899
+ SELECT 3, false, uid, ts, attrs->>'branch_key'
900
+ FROM _el_asg WHERE kind = 'link' AND role = 'seed' AND attrs->>'branch_key' IS NOT NULL;
901
+ CREATE INDEX ON _el_key (tier, k);
902
+
903
+ -- Hard keys: each unit to the previous unit (by first use) holding the same key; for a
904
+ -- branch only when this unit's first use comes within 14 days of the latest use before it.
905
+ -- Soft keys: a unit to each unit holding what it quotes (the printer of a SHA, any other
906
+ -- holder of a pr key). A unit with more than 2 such partners is a hub and keeps only the
907
+ -- pair with its nearest partner in time; a pr pair stands only if neither side is a hub
908
+ -- that chose another partner (a SHA's printer owns its commit even when it is a hub).
909
+ CREATE TEMP TABLE _el_edge AS
910
+ WITH ku AS (
911
+ SELECT tier, k, uid, min(ts) AS t0, max(ts) AS t1 FROM _el_key WHERE NOT soft GROUP BY 1, 2, 3
912
+ ), ch AS (
913
+ SELECT tier, uid, t0, lag(uid) OVER w AS prev,
914
+ max(t1) OVER (w ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) AS prev_t1
915
+ FROM ku WINDOW w AS (PARTITION BY tier, k ORDER BY t0, uid)
916
+ ), sp AS (
917
+ SELECT s.tier, s.uid AS u, h.uid AS v, greatest(min(s.ts), min(h.ts)) AS ts,
918
+ min(abs(extract(epoch FROM s.ts - h.ts))) AS dist
919
+ FROM _el_key s JOIN _el_key h
920
+ ON h.tier = s.tier AND h.k = s.k AND h.uid <> s.uid AND (h.soft = (s.tier = 2))
921
+ WHERE s.soft GROUP BY 1, 2, 3
922
+ ), deg AS (
923
+ SELECT u, count(DISTINCT v) AS n, (array_agg(v ORDER BY dist, v))[1] AS near
924
+ FROM sp GROUP BY u
925
+ )
926
+ SELECT tier, t0 AS ts, prev AS a, uid AS b FROM ch
927
+ WHERE prev IS NOT NULL AND (tier < 3 OR t0 <= prev_t1 + interval '14 days')
928
+ UNION ALL
929
+ SELECT sp.tier, sp.ts, sp.v, sp.u
930
+ FROM sp JOIN deg du ON du.u = sp.u LEFT JOIN deg dv ON dv.u = sp.v
931
+ WHERE (du.n <= 2 OR du.near = sp.v)
932
+ AND (sp.tier = 1 OR (sp.u < sp.v AND (dv.n <= 2 OR dv.near = sp.u)));
933
+ SELECT count(*)::int INTO edges FROM _el_edge;
934
+
935
+ -- Union-find over arrays indexed by uid (1..n), with path halving.
936
+ SELECT coalesce(array_agg(uid ORDER BY uid), '{}'), coalesce(array_agg(t0 ORDER BY uid), '{}'),
937
+ coalesce(array_agg(t1 ORDER BY uid), '{}')
938
+ INTO par, f, l FROM _el_span;
939
+ n := coalesce(array_length(par, 1), 0);
940
+ units := n;
941
+ FOR ed IN SELECT a, b FROM _el_edge ORDER BY tier, ts, a, b LOOP
942
+ ra := ed.a;
943
+ WHILE par[ra] <> ra LOOP par[ra] := par[par[ra]]; ra := par[ra]; END LOOP;
944
+ rb := ed.b;
945
+ WHILE par[rb] <> rb LOOP par[rb] := par[par[rb]]; rb := par[rb]; END LOOP;
946
+ CONTINUE WHEN ra = rb;
947
+ IF greatest(f[ra], f[rb]) - least(l[ra], l[rb]) > interval '60 days' THEN
948
+ refused := refused + 1;
949
+ CONTINUE;
950
+ END IF;
951
+ IF rb < ra THEN tgt := ra; ra := rb; rb := tgt; END IF;
952
+ par[rb] := ra;
953
+ f[ra] := least(f[ra], f[rb]);
954
+ l[ra] := greatest(l[ra], l[rb]);
955
+ END LOOP;
956
+ labs := par;
957
+ FOR i IN 1..n LOOP
958
+ ra := i;
959
+ WHILE par[ra] <> ra LOOP ra := par[ra]; END LOOP;
960
+ labs[i] := ra;
961
+ END LOOP;
962
+ CREATE TEMP TABLE _el_lab AS SELECT g AS uid, labs[g] AS lab FROM generate_series(1, n) g;
963
+
964
+ -- The units each correction names: side a = a_session's units in attrs.repo_keys (all of
965
+ -- them without it), side b = b_session's in attrs.b_repo_keys.
966
+ CREATE TEMP TABLE _el_cu AS
967
+ SELECT c.id, u.uid,
968
+ CASE WHEN u.session_id = c.a_session
969
+ AND (NOT c.attrs ? 'repo_keys' OR c.attrs->'repo_keys' ? u.repo_key)
970
+ THEN 'a' ELSE 'b' END AS side
971
+ FROM agents.episode_correction c JOIN _el_unit u
972
+ ON (u.session_id = c.a_session AND (NOT c.attrs ? 'repo_keys' OR c.attrs->'repo_keys' ? u.repo_key))
973
+ OR (u.session_id = c.b_session AND (NOT c.attrs ? 'b_repo_keys' OR c.attrs->'b_repo_keys' ? u.repo_key));
974
+
975
+ -- Corrections last, in time order. Labels above n are episodes made by a split.
976
+ next_label := n;
977
+ FOR cr IN SELECT id, action FROM agents.episode_correction
978
+ WHERE action <> 'confirm' ORDER BY ts, id LOOP
979
+ IF cr.action = 'merge' THEN
980
+ CONTINUE WHEN NOT EXISTS (SELECT 1 FROM _el_cu WHERE id = cr.id AND side = 'a')
981
+ OR NOT EXISTS (SELECT 1 FROM _el_cu WHERE id = cr.id AND side = 'b');
982
+ SELECT min(x.lab) INTO tgt FROM _el_lab x JOIN _el_cu c USING (uid) WHERE c.id = cr.id;
983
+ UPDATE _el_lab SET lab = tgt
984
+ WHERE lab IN (SELECT x.lab FROM _el_lab x JOIN _el_cu c USING (uid) WHERE c.id = cr.id);
985
+ ELSIF cr.action = 'split' THEN
986
+ -- one new episode per episode the named units were in
987
+ UPDATE _el_lab x SET lab = next_label + m.r
988
+ FROM (SELECT y.uid, dense_rank() OVER (ORDER BY y.lab) AS r
989
+ FROM _el_lab y JOIN _el_cu c USING (uid)
990
+ WHERE c.id = cr.id AND c.side = 'a' AND y.lab IS NOT NULL) m
991
+ WHERE x.uid = m.uid;
992
+ next_label := next_label + n; -- n bounds the ranks; labels need not be dense
993
+ ELSE -- reject
994
+ UPDATE _el_lab SET lab = NULL
995
+ WHERE lab IN (SELECT x.lab FROM _el_lab x JOIN _el_cu c USING (uid)
996
+ WHERE c.id = cr.id AND c.side = 'a');
997
+ END IF;
998
+ END LOOP;
999
+
1000
+ CREATE TEMP TABLE _el_ep AS
1001
+ SELECT x.uid, md5(first_value(u.session_id || '|' || u.repo_key)
1002
+ OVER (PARTITION BY x.lab ORDER BY s.t0, u.session_id, u.repo_key))::uuid AS episode_id
1003
+ FROM _el_lab x JOIN _el_unit u USING (uid) JOIN _el_span s USING (uid)
1004
+ WHERE x.lab IS NOT NULL;
1005
+
1006
+ TRUNCATE agents.episode, agents.episode_member;
1007
+
1008
+ INSERT INTO agents.episode (episode_id, first_ts, last_ts, n_sessions, n_seed_sessions, n_steers,
1009
+ n_commits, n_prs, outcome, outcome_source, confirmed)
1010
+ WITH base AS (
1011
+ SELECT e.episode_id, min(s.t0) AS first_ts, max(s.t1) AS last_ts
1012
+ FROM _el_ep e JOIN _el_span s USING (uid) GROUP BY 1
1013
+ ), mem AS (
1014
+ SELECT e.episode_id, count(DISTINCT a.session_id) AS n_sessions,
1015
+ count(DISTINCT a.session_id) FILTER (WHERE a.role = 'seed') AS n_seed,
1016
+ count(*) FILTER (WHERE a.kind = 'steer') AS n_steers
1017
+ FROM _el_ep e JOIN _el_asg a USING (uid) GROUP BY 1
1018
+ ), prs AS (
1019
+ SELECT e.episode_id,
1020
+ greatest(count(DISTINCT k.k) FILTER (WHERE k.k NOT LIKE 'pr-row:%'),
1021
+ count(DISTINCT k.k) FILTER (WHERE k.k LIKE 'pr-row:%')) AS n_prs
1022
+ FROM _el_ep e JOIN _el_key k USING (uid) WHERE k.tier = 2 GROUP BY 1
1023
+ ), co AS ( -- the commit rows' 7-digit prefix, so any sha form `aut outcomes` writes joins
1024
+ SELECT DISTINCT ON (left(lower(sha), 7)) left(lower(sha), 7) AS k, state, survival
1025
+ FROM agents.commit_outcome ORDER BY left(lower(sha), 7), decided_at DESC,
1026
+ CASE state WHEN 'reverted' THEN 0 WHEN 'reworked' THEN 1 WHEN 'merged' THEN 2
1027
+ WHEN 'open' THEN 3 WHEN 'abandoned' THEN 4 WHEN 'superseded' THEN 5 ELSE 6 END,
1028
+ repo_key, sha, survival NULLS LAST
1029
+ ), cs AS (
1030
+ SELECT DISTINCT e.episode_id, left(a.sha, 7) AS k
1031
+ FROM _el_ep e JOIN _el_asg a USING (uid) WHERE a.sha IS NOT NULL
1032
+ ), git AS (
1033
+ SELECT cs.episode_id,
1034
+ count(*) FILTER (WHERE coalesce(co.state, '') <> 'superseded') AS n_commits,
1035
+ CASE WHEN bool_or(co.state IN ('reverted', 'reworked')) THEN 'reworked'
1036
+ WHEN bool_or(co.state = 'merged' AND coalesce(co.survival, 1) >= 0.5) THEN 'shipped'
1037
+ WHEN bool_or(co.state = 'open') THEN 'open'
1038
+ -- coalesce: a commit with no outcome row yet keeps the episode undecided
1039
+ WHEN bool_and(coalesce(co.state, '') IN ('abandoned', 'superseded'))
1040
+ AND bool_or(co.state = 'abandoned') THEN 'ruled_out'
1041
+ ELSE 'undecided' END AS outcome
1042
+ FROM cs LEFT JOIN co USING (k) GROUP BY 1
1043
+ ), man AS (
1044
+ SELECT DISTINCT ON (e.episode_id) e.episode_id, m.outcome
1045
+ FROM agents.manual_outcome m
1046
+ JOIN _el_unit u ON u.session_id = m.session_id
1047
+ AND (m.repo_key IS NULL OR u.repo_key = m.repo_key
1048
+ OR NOT EXISTS (SELECT 1 FROM _el_unit v
1049
+ WHERE v.session_id = m.session_id AND v.repo_key = m.repo_key))
1050
+ JOIN _el_ep e USING (uid)
1051
+ ORDER BY e.episode_id, m.ts DESC, m.session_id, m.repo_key NULLS LAST, m.outcome
1052
+ ), conf AS (
1053
+ SELECT DISTINCT e.episode_id
1054
+ FROM _el_ep e JOIN _el_cu cu USING (uid) JOIN agents.episode_correction c ON c.id = cu.id
1055
+ WHERE c.action = 'confirm' AND cu.side = 'a'
1056
+ )
1057
+ SELECT b.episode_id, b.first_ts, b.last_ts, m.n_sessions, m.n_seed, m.n_steers,
1058
+ coalesce(g.n_commits, 0), coalesce(p.n_prs, 0),
1059
+ coalesce(man.outcome, g.outcome, 'no_code'),
1060
+ CASE WHEN man.outcome IS NOT NULL THEN 'manual' ELSE 'git' END,
1061
+ conf.episode_id IS NOT NULL
1062
+ FROM base b JOIN mem m USING (episode_id)
1063
+ LEFT JOIN prs p USING (episode_id) LEFT JOIN git g USING (episode_id)
1064
+ LEFT JOIN man USING (episode_id) LEFT JOIN conf USING (episode_id);
1065
+ GET DIAGNOSTICS episodes = ROW_COUNT;
1066
+
1067
+ INSERT INTO agents.episode_member (episode_id, session_id, role, repo_keys)
1068
+ SELECT e.episode_id, a.session_id, a.role, array_agg(DISTINCT u.repo_key ORDER BY u.repo_key)
1069
+ FROM _el_ep e JOIN _el_asg a USING (uid) JOIN _el_unit u USING (uid)
1070
+ GROUP BY 1, 2, 3;
1071
+ GET DIAGNOSTICS members = ROW_COUNT;
1072
+
1073
+ DROP TABLE _el_row, _el_sess, _el_unit, _el_pos, _el_asg, _el_span, _el_key, _el_edge,
1074
+ _el_lab, _el_cu, _el_ep;
1075
+ RETURN NEXT;
1076
+ END
1077
+ $$;
1078
+
1079
+ -- Episodes per completed month (sink time zone AUT_TZ, by the month an episode started), for
1080
+ -- public summaries. A month shows only 14 days after it ends, once every episode
1081
+ -- that started in it has been quiet for 14 days and none is still undecided, so the counts do
1082
+ -- not move after they are first shown (later hand corrections still may, on purpose). A
1083
+ -- commit that never gets a final outcome holds its month back. Counts only: no session or
1084
+ -- episode id.
1085
+ CREATE OR REPLACE VIEW agents.episode_monthly AS
1086
+ SELECT month,
1087
+ count(*)::int AS episodes,
1088
+ count(*) FILTER (WHERE outcome = 'shipped')::int AS shipped,
1089
+ count(*) FILTER (WHERE outcome = 'reworked')::int AS reworked,
1090
+ count(*) FILTER (WHERE outcome = 'open')::int AS open,
1091
+ count(*) FILTER (WHERE outcome = 'ruled_out')::int AS ruled_out,
1092
+ count(*) FILTER (WHERE outcome = 'no_code')::int AS no_code,
1093
+ count(*) FILTER (WHERE outcome = 'undecided')::int AS undecided,
1094
+ count(*) FILTER (WHERE first_day <> last_day)::int AS multi_day,
1095
+ count(*) FILTER (WHERE n_seed_sessions >= 2)::int AS revisits
1096
+ FROM (SELECT date_trunc('month', first_ts AT TIME ZONE '@AUT_TZ@')::date AS month,
1097
+ (first_ts AT TIME ZONE '@AUT_TZ@')::date AS first_day,
1098
+ (last_ts AT TIME ZONE '@AUT_TZ@')::date AS last_day,
1099
+ last_ts, outcome, n_seed_sessions
1100
+ FROM agents.episode) e
1101
+ WHERE (month + interval '1 month' + interval '14 days') AT TIME ZONE '@AUT_TZ@' <= now()
1102
+ GROUP BY month
1103
+ HAVING max(last_ts) + interval '14 days' <= now()
1104
+ AND count(*) FILTER (WHERE outcome = 'undecided') = 0;
1105
+
1106
+ -- Phase 4: labels from a local model on a collector machine. No new table for them: aut writes one
1107
+ -- activity_event row per decision (kind 'label') and one per bake-off result (kind 'eval'),
1108
+ -- attrs ids, enums and numbers only, never a title or prompt text. A new model, prompt or
1109
+ -- task_ver adds rows and never overwrites, so these views pick the current one. Neither view
1110
+ -- has a machine column: labels are per unit, whichever machine wrote them.
1111
+
1112
+ -- Current label per (task, unit, unit_id): newest task_ver, then the newest labelling run
1113
+ -- (attrs labelled_at), event_id only to keep the pick stable. below_tau rows stay
1114
+ -- in, so consumers treat them as unknown (excluded from shares in numerator and denominator).
1115
+ CREATE OR REPLACE VIEW agents.label_latest AS
1116
+ SELECT DISTINCT ON (attrs->>'task', attrs->>'unit', attrs->>'unit_id')
1117
+ attrs->>'unit' AS unit,
1118
+ attrs->>'unit_id' AS unit_id,
1119
+ attrs->>'task' AS task,
1120
+ attrs->>'label' AS label,
1121
+ (attrs->>'p')::double precision AS p,
1122
+ coalesce((attrs->>'below_tau')::boolean, true) AS below_tau,
1123
+ attrs->>'model_id' AS model_id,
1124
+ (attrs->>'task_ver')::int AS task_ver,
1125
+ ts
1126
+ FROM agents.activity_event
1127
+ WHERE kind = 'label'
1128
+ AND coalesce(attrs->>'task', '') NOT LIKE 'working.%'
1129
+ ORDER BY attrs->>'task', attrs->>'unit', attrs->>'unit_id',
1130
+ (attrs->>'task_ver')::int DESC, attrs->>'labelled_at' DESC NULLS LAST, event_id DESC;
1131
+
1132
+ -- Newest eval per (task, model_id), one row per label of its per_label object. Numbers only.
1133
+ -- aut's eval row: attrs {task, task_ver, model_id, model_rev, macro_f1, kappa, ece_raw,
1134
+ -- ece_cal, per_label: {<label>: {n, precision, recall, f1, tau, coverage, share_error_pp}}}.
1135
+ CREATE OR REPLACE VIEW agents.eval_latest AS
1136
+ SELECT e.task, e.task_ver, e.model_id, e.model_rev, e.ts AS evaluated_at,
1137
+ l.key AS label,
1138
+ (l.value->>'n')::int AS n,
1139
+ (l.value->>'precision')::double precision AS "precision",
1140
+ (l.value->>'recall')::double precision AS recall,
1141
+ (l.value->>'f1')::double precision AS f1,
1142
+ (l.value->>'tau')::double precision AS tau,
1143
+ (l.value->>'coverage')::double precision AS coverage,
1144
+ (l.value->>'share_error_pp')::double precision AS share_error_pp,
1145
+ e.macro_f1, e.kappa, e.ece_raw, e.ece_cal
1146
+ FROM (SELECT DISTINCT ON (attrs->>'task', attrs->>'model_id')
1147
+ attrs->>'task' AS task,
1148
+ (attrs->>'task_ver')::int AS task_ver,
1149
+ attrs->>'model_id' AS model_id,
1150
+ attrs->>'model_rev' AS model_rev,
1151
+ (attrs->>'macro_f1')::double precision AS macro_f1,
1152
+ (attrs->>'kappa')::double precision AS kappa,
1153
+ (attrs->>'ece_raw')::double precision AS ece_raw,
1154
+ (attrs->>'ece_cal')::double precision AS ece_cal,
1155
+ attrs->'per_label' AS per_label,
1156
+ ts
1157
+ FROM agents.activity_event
1158
+ WHERE kind = 'eval'
1159
+ ORDER BY attrs->>'task', attrs->>'model_id', ts DESC, event_id DESC) e
1160
+ CROSS JOIN LATERAL jsonb_each(CASE WHEN jsonb_typeof(e.per_label) = 'object'
1161
+ THEN e.per_label ELSE '{}'::jsonb END) l;
1162
+
1163
+ -- Private nightly notes: one short draft per day, written by aut on a collector machine from that
1164
+ -- day's labels and episode facts. Text, so private like window titles: agents_ro has no access, and
1165
+ -- the writer inserts and reads only the day (enough for ON CONFLICT (day) DO NOTHING), so a
1166
+ -- leaked writer DSN cannot read the notes. Kept for good by default: no retention job.
1167
+ CREATE TABLE IF NOT EXISTS agents.daily_note (
1168
+ day date PRIMARY KEY,
1169
+ text text NOT NULL,
1170
+ model_id text NOT NULL,
1171
+ created_at timestamptz NOT NULL DEFAULT now()
1172
+ );
1173
+
1174
+ -- Commits per local day from a full git scan (all clones, the user's own identities only, counts
1175
+ -- only: no repo, name or SHA). Insert-only, a day's latest scan wins (agents.git_commits_daily).
1176
+ -- Partial scans carry recorded lower bounds; a missing day remains unknown, not a zero.
1177
+ CREATE TABLE IF NOT EXISTS agents.git_commit_day (
1178
+ day date NOT NULL,
1179
+ commits integer NOT NULL CHECK (commits >= 0),
1180
+ scanned_at timestamptz NOT NULL DEFAULT now(),
1181
+ complete boolean NOT NULL DEFAULT true,
1182
+ PRIMARY KEY (day, scanned_at)
1183
+ );
1184
+ -- Existing rows came only from successful complete scans.
1185
+ ALTER TABLE agents.git_commit_day
1186
+ ADD COLUMN IF NOT EXISTS complete boolean NOT NULL DEFAULT true;