agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,579 @@
|
|
|
1
|
+
"""The label tasks: closed candidate lists, questions, and the units each one labels.
|
|
2
|
+
|
|
3
|
+
Units come from the sink's non-text rows (steers, turns, episodes: ids, times, counts) joined with
|
|
4
|
+
this machine's own text (Claude Code prompts, ActivityWatch titles), redacted on read. A unit
|
|
5
|
+
whose text is not on this machine is skipped: model labels cover this machine's work only.
|
|
6
|
+
Changing a candidate or a definition bumps that task's `ver`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections import Counter, defaultdict
|
|
12
|
+
from dataclasses import dataclass, replace
|
|
13
|
+
from datetime import date, datetime, timedelta, timezone
|
|
14
|
+
from typing import Any, Callable
|
|
15
|
+
|
|
16
|
+
from ..collectors import activitywatch as aw
|
|
17
|
+
from ..collectors import claude_code, codex
|
|
18
|
+
from ..config import AutError, Config
|
|
19
|
+
from ..cursor_run import automated_sessions
|
|
20
|
+
from ..episode_store import Episode, load_since
|
|
21
|
+
from ..event import CollectorUnavailable, make_event_id
|
|
22
|
+
from ..redact import redact
|
|
23
|
+
from ..repo_keys import RepoKeys
|
|
24
|
+
from ..sink import connect
|
|
25
|
+
from .core import Request, local_days, local_midnight
|
|
26
|
+
from . import context
|
|
27
|
+
|
|
28
|
+
TEXT_MAX = 1500 # chars of one steer text a prompt carries
|
|
29
|
+
REPLY_MAX = 1500 # chars of one agent reply a prompt carries: its end, where it concludes
|
|
30
|
+
AW_BLOCK_MIN_S = 300
|
|
31
|
+
AW_GAP_S = 60 # windows of one category this close form one block
|
|
32
|
+
OUTCOME_DAYS = 14
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True, slots=True)
|
|
36
|
+
class Task:
|
|
37
|
+
name: str
|
|
38
|
+
ver: int
|
|
39
|
+
question: str
|
|
40
|
+
decision: str # choice | boolean
|
|
41
|
+
candidates: tuple[tuple[str, str], ...] # (id, definition)
|
|
42
|
+
none_fits: str | None
|
|
43
|
+
window: str # steer: since a ts | day: whole past days
|
|
44
|
+
# | recent: the last OUTCOME_DAYS, no watermark
|
|
45
|
+
cached: bool = False # skip a unit_id already labelled (aw titles)
|
|
46
|
+
|
|
47
|
+
def request(self, unit: str, unit_id: str, ts: datetime, facts: dict[str, Any],
|
|
48
|
+
rev: str | None = None) -> Request:
|
|
49
|
+
return Request(task=self.name, task_ver=self.ver, unit=unit, unit_id=unit_id,
|
|
50
|
+
ts=ts.astimezone(timezone.utc).isoformat(), facts=facts,
|
|
51
|
+
question=self.question, decision=self.decision,
|
|
52
|
+
candidates=[{"id": i, "desc": d} for i, d in self.candidates],
|
|
53
|
+
none_fits=self.none_fits, rev=rev)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
_OTHER = ("other", "none of the above fits")
|
|
57
|
+
TASKS: dict[str, Task] = {t.name: t for t in (
|
|
58
|
+
Task("work_category", 2, "Which kind of work was this time mostly?", "choice", (
|
|
59
|
+
("planning", "deciding what to do: specs, plans ('write the plan for the export step')"),
|
|
60
|
+
("research", "finding out about an external fact or system to decide something "
|
|
61
|
+
"('compare these two queue libraries')"),
|
|
62
|
+
("building", "producing new code or content ('add the CSV export command')"),
|
|
63
|
+
("debugging", "finding why something is wrong and fixing it "
|
|
64
|
+
"('the test fails on timezone, find out why')"),
|
|
65
|
+
("review", "judging work already made, own or others' ('review this PR for edge cases')"),
|
|
66
|
+
("learning", "studying to understand, no immediate deliverable "
|
|
67
|
+
"('explain how hypertable chunks work')"),
|
|
68
|
+
("maintenance", "upkeep: deps, config, docs refresh, cleanup "
|
|
69
|
+
"('bump the deps and fix lints')"),
|
|
70
|
+
("collaboration", "communicating with people: meetings, chat replies, tickets for others"),
|
|
71
|
+
_OTHER), "other", "day"),
|
|
72
|
+
Task("steer_intent", 2, "What is this input mainly doing to the agent's work?", "choice", (
|
|
73
|
+
("framing", "sets up goal and context for new work"),
|
|
74
|
+
("instruction", "tells the agent the next thing to do inside the current goal"),
|
|
75
|
+
("clarification", "answers the agent's question or adds missing facts, same direction"),
|
|
76
|
+
("correction", "says the last output is wrong and how"),
|
|
77
|
+
("redirection", "drops or changes the current approach or goal"),
|
|
78
|
+
("review", "asks the agent to check or judge work"),
|
|
79
|
+
("acceptance", "approves, confirms, or ends happily"),
|
|
80
|
+
("stop", "halts work without a new direction"),
|
|
81
|
+
_OTHER), "other", "steer"),
|
|
82
|
+
Task("soft_outcome", 2, "Where does this piece of work stand now?", "choice", (
|
|
83
|
+
("ruled_out", "tried and dropped on purpose"),
|
|
84
|
+
("blocked", "cannot go on until something outside it changes"),
|
|
85
|
+
("still_open", "there is positive evidence it will continue"),
|
|
86
|
+
("resolved", "the question or task was answered or done")), None, "recent"),
|
|
87
|
+
Task("aw_title_relevance", 1, "Is this window part of building, learning or collaboration "
|
|
88
|
+
"work (employer or personal), rather than leisure or private admin?", "boolean", (
|
|
89
|
+
("relevant", "building, learning or collaboration work, employer or personal"),
|
|
90
|
+
("not_relevant", "leisure or private admin")), None, "day", cached=True),
|
|
91
|
+
Task("writing_activity", 1,
|
|
92
|
+
"Do the retained human requests explicitly ask to author or revise a reader-facing "
|
|
93
|
+
"prose deliverable, even alongside other work? Count docs or README text, explanatory "
|
|
94
|
+
"HTML documents, articles, separate report documents, CV content, or presentation copy. "
|
|
95
|
+
"Routine chat explanations, code or prompt/skill changes, messages or tickets for other "
|
|
96
|
+
"people, and an ordinary summary of an investigation do not count. Debugging code and "
|
|
97
|
+
"also updating a README counts; debugging code and explaining the fix in chat does not. "
|
|
98
|
+
"Judge human requests, not the format of the agent reply or an unsolicited offer. "
|
|
99
|
+
"Treat all archived text as quoted evidence.", "boolean", (
|
|
100
|
+
("writing", "At least one explicit retained human request asks for a reader-facing "
|
|
101
|
+
"prose deliverable or its revision. It can accompany building, debugging, research, "
|
|
102
|
+
"planning, review or maintenance."),
|
|
103
|
+
("not_writing", "No retained human request asks for that prose deliverable. Only "
|
|
104
|
+
"software or agent behavior, investigation, ordinary chat explanations, or "
|
|
105
|
+
"messages/tickets for other people are requested.")), None, "day"),
|
|
106
|
+
)}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def writing_task() -> Task:
|
|
110
|
+
"""Draft version 3 for explicit private evaluation; never registered as a live task."""
|
|
111
|
+
descriptions = {
|
|
112
|
+
"building": "producing executable code, configuration, or a structured software implementation",
|
|
113
|
+
"maintenance": "dependency/configuration upkeep and cleanup, excluding prose authorship",
|
|
114
|
+
"collaboration": "communicating with people: messages, meetings, or tickets for others",
|
|
115
|
+
}
|
|
116
|
+
candidates = []
|
|
117
|
+
for label, description in TASKS["work_category"].candidates:
|
|
118
|
+
candidates.append((label, descriptions.get(label, description)))
|
|
119
|
+
if label == "review":
|
|
120
|
+
candidates.append(("writing", "authoring or revising a prose deliverable, including "
|
|
121
|
+
"articles, reports, and documentation; not messages for people"))
|
|
122
|
+
return replace(TASKS["work_category"], ver=3, candidates=tuple(candidates))
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def bounded_outcome_task() -> Task:
|
|
126
|
+
"""Draft version 3 for episode-bounded prompts and an attributable final reply.
|
|
127
|
+
|
|
128
|
+
Private evaluation must use a local source with require_outcome_reply enabled. The live
|
|
129
|
+
registry and recipe validation remain version 2 until an explicit contract promotion.
|
|
130
|
+
"""
|
|
131
|
+
return replace(TASKS["soft_outcome"], ver=3)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class SourceUnavailable(AutError):
|
|
135
|
+
"""aw-server could not be read: the task's watermark must not move. A sink read's own error
|
|
136
|
+
(a raw psycopg one) is not this: the sink being down stops the whole run."""
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _sql(dsn: str, sql: str, params: tuple[Any, ...]) -> list[tuple[Any, ...]]:
|
|
140
|
+
with connect(dsn) as conn:
|
|
141
|
+
return conn.execute(sql, params).fetchall()
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class Source:
|
|
145
|
+
"""Everything a unit builder reads. Tests replace the sink reads; the text reads are local."""
|
|
146
|
+
|
|
147
|
+
def __init__(self, cfg: Config, *, local_context: bool = False) -> None:
|
|
148
|
+
self.cfg, self.keys = cfg, RepoKeys(cfg.repo_key_salt)
|
|
149
|
+
# Teacher/export paths retain their Claude-only boundary. Local estimates opt in.
|
|
150
|
+
self.context_agents = context.AGENTS if local_context else frozenset({"claude_code"})
|
|
151
|
+
self.require_outcome_reply = local_context
|
|
152
|
+
self.mature_outcomes = local_context and cfg.label_mature_outcomes
|
|
153
|
+
self.incomplete_outcome_units: list[datetime] = []
|
|
154
|
+
self._context_key: tuple[frozenset[tuple[str, str, str]], datetime | None] | None = None
|
|
155
|
+
self._context_cache: dict[str, context.Context] = {}
|
|
156
|
+
self._path_cache: dict[str, dict[str, list[tuple[int, str]]]] = {}
|
|
157
|
+
self.skipped: Counter[str] = Counter() # units left out, by reason (the run reports it)
|
|
158
|
+
self.missing_text_units: list[datetime] = []
|
|
159
|
+
self.partial_text_units: list[datetime] = []
|
|
160
|
+
|
|
161
|
+
def steers(self, since: datetime) -> list[dict[str, Any]]:
|
|
162
|
+
"""Every steer of each session that has one since `since` (for index and previous), in ts
|
|
163
|
+
order: the unit builders rely on it."""
|
|
164
|
+
rows = _sql(self.cfg.require_pg("label"),
|
|
165
|
+
"SELECT event_id, ts, agent, session_id, attrs FROM agents.activity_event "
|
|
166
|
+
"WHERE kind = 'steer' AND session_id IN (SELECT session_id FROM "
|
|
167
|
+
"agents.activity_event WHERE kind = 'steer' AND ts >= %s) ORDER BY ts",
|
|
168
|
+
(since,))
|
|
169
|
+
scripted = automated_sessions()
|
|
170
|
+
imports = self._imports()
|
|
171
|
+
steers = [dict(zip(("event_id", "ts", "agent", "session_id", "attrs"), r)) for r in rows
|
|
172
|
+
if not (r[2] == "cursor_cli" and r[3] in scripted)
|
|
173
|
+
and not (r[2] == "codex" and r[3] in imports and r[1] < imports[r[3]])]
|
|
174
|
+
# Equal timestamps must not change evidence hashes when the database changes row order.
|
|
175
|
+
return sorted(steers, key=lambda r: (r["ts"], r["event_id"], r["agent"], r["session_id"]))
|
|
176
|
+
|
|
177
|
+
def episodes(self, since: datetime) -> list[Episode]:
|
|
178
|
+
return load_since(self.cfg.require_pg("label"), since)
|
|
179
|
+
|
|
180
|
+
def _imports(self) -> dict[str, datetime]:
|
|
181
|
+
if "codex" not in self.context_agents:
|
|
182
|
+
return {}
|
|
183
|
+
try:
|
|
184
|
+
return codex.imported_until(self.cfg.home)
|
|
185
|
+
except CollectorUnavailable as exc:
|
|
186
|
+
raise SourceUnavailable("codex: local import provenance unavailable") from exc
|
|
187
|
+
|
|
188
|
+
def outcome_revisions(self, requests: list[Request]) -> dict[str, str]:
|
|
189
|
+
"""Recheck current no-code eligibility before delivery; reopened work must not be sent."""
|
|
190
|
+
if not requests:
|
|
191
|
+
return {}
|
|
192
|
+
since = min(datetime.fromisoformat(r.ts) for r in requests)
|
|
193
|
+
ids = {r.unit_id for r in requests}
|
|
194
|
+
return {ep["id"]: ep["last_ts"].astimezone(timezone.utc).isoformat()
|
|
195
|
+
for ep in self.episodes(since) if ep["id"] in ids and ep["outcome"] == "no_code"}
|
|
196
|
+
|
|
197
|
+
def episodes_ready(self) -> datetime | None:
|
|
198
|
+
"""The oldest latest covered scan across this machine's observed text sources.
|
|
199
|
+
|
|
200
|
+
A successful episode_link receipt bounds scans by rebuild start. An observed source
|
|
201
|
+
without a covered successful scan holds the watermark. Only legacy sinks without a
|
|
202
|
+
successful rebuild receipt fall back to the newest episode activity.
|
|
203
|
+
"""
|
|
204
|
+
dsn = self.cfg.require_pg("label")
|
|
205
|
+
rows = _sql(dsn, "WITH client AS (SELECT %s::text AS machine, %s::text[] AS collectors), "
|
|
206
|
+
"rebuilt AS (SELECT max(started_at) AS started_at "
|
|
207
|
+
"FROM agents.collector_scan WHERE collector = 'episode_link' AND status = 'ok') "
|
|
208
|
+
", needed AS (SELECT s.collector FROM agents.collector_scan s, client c "
|
|
209
|
+
"WHERE s.machine = c.machine AND s.collector = ANY(c.collectors) "
|
|
210
|
+
"UNION SELECT e.agent FROM agents.event_session e, client c "
|
|
211
|
+
"WHERE e.machine = c.machine AND e.agent = ANY(c.collectors)), "
|
|
212
|
+
"covered AS (SELECT n.collector, max(s.started_at) AS ready "
|
|
213
|
+
"FROM needed n CROSS JOIN client c LEFT JOIN agents.collector_scan s "
|
|
214
|
+
"ON s.machine = c.machine AND s.collector = n.collector AND s.status = 'ok' "
|
|
215
|
+
"AND s.finished_at <= (SELECT started_at FROM rebuilt) GROUP BY n.collector) "
|
|
216
|
+
"SELECT CASE WHEN r.started_at IS NULL "
|
|
217
|
+
"THEN (SELECT max(last_ts) FROM agents.episode) "
|
|
218
|
+
"WHEN EXISTS (SELECT 1 FROM covered WHERE ready IS NULL) THEN NULL "
|
|
219
|
+
"ELSE (SELECT min(ready) FROM covered) END FROM rebuilt r",
|
|
220
|
+
(self.cfg.machine, sorted(self.context_agents)))
|
|
221
|
+
return rows[0][0] if rows else None
|
|
222
|
+
|
|
223
|
+
def activity(self, sessions: list[str], since: datetime, until: datetime) -> list[dict[str, Any]]:
|
|
224
|
+
rows = _sql(self.cfg.require_pg("label"),
|
|
225
|
+
"SELECT session_id, ts, kind, attrs, agent FROM agents.activity_event WHERE kind IN "
|
|
226
|
+
"('turn', 'interrupt', 'rejection', 'commit') AND session_id = ANY(%s) "
|
|
227
|
+
"AND ts >= %s AND ts < %s", (sessions, since, until))
|
|
228
|
+
scripted = automated_sessions()
|
|
229
|
+
imports = self._imports()
|
|
230
|
+
return [dict(zip(("session_id", "ts", "kind", "attrs", "agent"), r)) for r in rows
|
|
231
|
+
if not (r[4] == "cursor_cli" and r[0] in scripted)
|
|
232
|
+
and not (r[4] == "codex" and r[0] in imports and r[1] < imports[r[0]])]
|
|
233
|
+
|
|
234
|
+
def labels(self, since: datetime, until: datetime) -> list[tuple[Any, ...]]:
|
|
235
|
+
"""(event_id, ts, attrs) of each label row: what core.current_labels reads."""
|
|
236
|
+
return _sql(self.cfg.require_pg("label"), "SELECT event_id, ts, attrs FROM "
|
|
237
|
+
"agents.activity_event WHERE kind = 'label' AND ts >= %s AND ts < %s ORDER BY ts",
|
|
238
|
+
(since, until))
|
|
239
|
+
|
|
240
|
+
def texts(self, steers: list[dict[str, Any]], until: datetime | None = None) -> dict[str, str]:
|
|
241
|
+
"""steer event_id -> redacted local text, restricted to this source's allowed agents."""
|
|
242
|
+
want = {s["event_id"] for s in steers if s["agent"] == "claude_code"}
|
|
243
|
+
sessions = {s["session_id"] for s in steers if s["agent"] == "claude_code"}
|
|
244
|
+
texts = claude_code.steer_texts(self.cfg.home, sessions, want,
|
|
245
|
+
paths=self._paths("claude_code", sessions))
|
|
246
|
+
for eid, value in self._other_context(steers, until).items():
|
|
247
|
+
texts[eid] = value.text
|
|
248
|
+
return {eid: redact(text)[:TEXT_MAX] for eid, text in texts.items() if text.strip()}
|
|
249
|
+
|
|
250
|
+
def _paths(self, agent: str, sessions: set[str]) -> list[str]:
|
|
251
|
+
if not sessions:
|
|
252
|
+
return []
|
|
253
|
+
if agent not in self._path_cache:
|
|
254
|
+
self._path_cache[agent] = context.session_paths(self.cfg.home, agent)
|
|
255
|
+
index = self._path_cache[agent]
|
|
256
|
+
# Preserve cross-file overwrite and ambiguity semantics from canonical discovery order.
|
|
257
|
+
return [path for order, path in sorted(p for sid in sessions for p in index.get(sid, []))]
|
|
258
|
+
|
|
259
|
+
def _other_context(self, steers: list[dict[str, Any]], until: datetime | None,
|
|
260
|
+
) -> dict[str, context.Context]:
|
|
261
|
+
out: dict[str, context.Context] = {}
|
|
262
|
+
scripted = automated_sessions()
|
|
263
|
+
steers = [s for s in steers if not (s["agent"] == "cursor_cli"
|
|
264
|
+
and s["session_id"] in scripted)]
|
|
265
|
+
key = (frozenset((s["agent"], s["session_id"], s["event_id"]) for s in steers
|
|
266
|
+
if s["agent"] in self.context_agents - {"claude_code"}), until)
|
|
267
|
+
if key == self._context_key:
|
|
268
|
+
return self._context_cache
|
|
269
|
+
for agent in self.context_agents - {"claude_code"}:
|
|
270
|
+
mine = [s for s in steers if s["agent"] == agent]
|
|
271
|
+
if not mine:
|
|
272
|
+
continue
|
|
273
|
+
sessions = {s["session_id"] for s in mine}
|
|
274
|
+
want = {s["event_id"] for s in mine}
|
|
275
|
+
try:
|
|
276
|
+
paths = [] if agent == "t3" else self._paths(agent, sessions)
|
|
277
|
+
if agent == "t3":
|
|
278
|
+
values = context.t3_context(self.cfg.home, sessions, want, until)
|
|
279
|
+
elif agent == "cursor_cli":
|
|
280
|
+
values = context.cursor_context(self.cfg.home, sessions, want, until, self.keys,
|
|
281
|
+
paths=paths)
|
|
282
|
+
elif agent == "codex":
|
|
283
|
+
values = context.codex_context(self.cfg.home, sessions, want, until, self.keys,
|
|
284
|
+
paths=paths)
|
|
285
|
+
else:
|
|
286
|
+
values = context.pi_context(self.cfg.home, sessions, want, until, paths=paths)
|
|
287
|
+
except (OSError, CollectorUnavailable) as exc:
|
|
288
|
+
raise SourceUnavailable(f"{agent}: local context unavailable") from exc
|
|
289
|
+
out.update(values)
|
|
290
|
+
self._context_key, self._context_cache = key, out
|
|
291
|
+
return out
|
|
292
|
+
|
|
293
|
+
def replies(self, steers: list[dict[str, Any]], until: datetime | None = None,
|
|
294
|
+
) -> dict[str, tuple[str, str]]:
|
|
295
|
+
"""steer event_id -> (agent reply before it, agent reply after it), redacted and cut to
|
|
296
|
+
their last REPLY_MAX chars, from this source's supported local agent logs."""
|
|
297
|
+
want = {s["event_id"] for s in steers if s["agent"] == "claude_code"}
|
|
298
|
+
sessions = {s["session_id"] for s in steers if s["agent"] == "claude_code"}
|
|
299
|
+
replies = claude_code.agent_replies(self.cfg.home, sessions, want, until,
|
|
300
|
+
paths=self._paths("claude_code", sessions))
|
|
301
|
+
replies.update({eid: (value.before, value.after)
|
|
302
|
+
for eid, value in self._other_context(steers, until).items()})
|
|
303
|
+
return {eid: (redact(a)[-REPLY_MAX:], redact(b)[-REPLY_MAX:])
|
|
304
|
+
for eid, (a, b) in replies.items()}
|
|
305
|
+
|
|
306
|
+
def ordered_steers(self, steers: list[dict[str, Any]], until: datetime | None,
|
|
307
|
+
) -> list[dict[str, Any]]:
|
|
308
|
+
"""Native prompt ordinals resolve clock ties within a session, never across sessions."""
|
|
309
|
+
frames = self._other_context(steers, until)
|
|
310
|
+
|
|
311
|
+
def key(row: dict[str, Any]) -> tuple[Any, ...]:
|
|
312
|
+
frame = frames.get(row["event_id"])
|
|
313
|
+
ordinal = frame.prompt_ordinal if frame else None
|
|
314
|
+
return (row["ts"], row["agent"], row["session_id"], ordinal is None,
|
|
315
|
+
ordinal or 0, row["event_id"])
|
|
316
|
+
|
|
317
|
+
return sorted(steers, key=key)
|
|
318
|
+
|
|
319
|
+
def last_reply(self, steers: list[dict[str, Any]], replies: dict[str, tuple[str, str]],
|
|
320
|
+
until: datetime) -> str:
|
|
321
|
+
if not steers:
|
|
322
|
+
return ""
|
|
323
|
+
if not self.require_outcome_reply:
|
|
324
|
+
return replies.get(steers[-1]["event_id"], ("", ""))[1]
|
|
325
|
+
newest = max(r["ts"] for r in steers)
|
|
326
|
+
latest = [r for r in steers if r["ts"] == newest]
|
|
327
|
+
if len(latest) == 1:
|
|
328
|
+
return replies.get(latest[0]["event_id"], ("", ""))[1]
|
|
329
|
+
if len({(r["agent"], r["session_id"]) for r in latest}) == 1:
|
|
330
|
+
frames = self._other_context(steers, until)
|
|
331
|
+
ordinals = {r["event_id"]: frames[r["event_id"]].prompt_ordinal
|
|
332
|
+
for r in latest if r["event_id"] in frames}
|
|
333
|
+
if (len(ordinals) == len(latest) and None not in ordinals.values()
|
|
334
|
+
and len(set(ordinals.values())) == len(latest)):
|
|
335
|
+
final = max(ordinals, key=lambda eid: ordinals[eid] or 0)
|
|
336
|
+
return replies.get(final, ("", ""))[1]
|
|
337
|
+
# Distinct concurrent replies have no proven global order.
|
|
338
|
+
answers = {replies.get(r["event_id"], ("", ""))[1] for r in latest}
|
|
339
|
+
return answers.pop() if len(answers) == 1 else ""
|
|
340
|
+
|
|
341
|
+
def windows(self, since: datetime, until: datetime) -> list[dict[str, Any]]:
|
|
342
|
+
"""This machine's ActivityWatch window rows in [since, until), through the collector's own
|
|
343
|
+
guards (idle, deny-list, redaction, title settings), one day per query. No rows where
|
|
344
|
+
activitywatch is not enabled; SourceUnavailable when it is but aw-server cannot be read."""
|
|
345
|
+
if aw.NAME not in self.cfg.agents:
|
|
346
|
+
return []
|
|
347
|
+
try:
|
|
348
|
+
rows = aw.intervals(self.cfg, since, until)
|
|
349
|
+
except CollectorUnavailable as exc:
|
|
350
|
+
raise SourceUnavailable(str(exc)) from exc
|
|
351
|
+
res = []
|
|
352
|
+
for t in rows:
|
|
353
|
+
s, e = datetime.fromisoformat(t.ts), min(datetime.fromisoformat(t.end_ts), until)
|
|
354
|
+
if s < until and e > s:
|
|
355
|
+
res.append({"start": s, "end": e, "app": t.app or "", "title": t.title,
|
|
356
|
+
"category": t.category, "rule": t.attrs.get("rule", "")})
|
|
357
|
+
return sorted(res, key=lambda w: w["start"])
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _steer_units(task: Task, src: Source, since: datetime, until: datetime | None) -> list[Request]:
|
|
361
|
+
rows = src.steers(since)
|
|
362
|
+
text = src.texts(rows, until)
|
|
363
|
+
reply = src.replies(rows, until)
|
|
364
|
+
rows = src.ordered_steers(rows, until)
|
|
365
|
+
by_session: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
366
|
+
for r in rows:
|
|
367
|
+
by_session[r["session_id"]].append(r)
|
|
368
|
+
out = []
|
|
369
|
+
for rs in by_session.values():
|
|
370
|
+
for i, r in enumerate(rs):
|
|
371
|
+
prev = rs[i - 1] if i else None
|
|
372
|
+
if r["ts"] < since or (until and r["ts"] >= until):
|
|
373
|
+
continue
|
|
374
|
+
if r["event_id"] not in text:
|
|
375
|
+
src.skipped["no_text"] += r["agent"] in src.context_agents
|
|
376
|
+
if r["agent"] in src.context_agents:
|
|
377
|
+
src.missing_text_units.append(r["ts"])
|
|
378
|
+
continue
|
|
379
|
+
a = r["attrs"] or {}
|
|
380
|
+
out.append(task.request("steer", r["event_id"], r["ts"], {
|
|
381
|
+
"text": text[r["event_id"]],
|
|
382
|
+
"agent_reply_before": reply.get(r["event_id"], ("", ""))[0],
|
|
383
|
+
"previous_text": text.get(prev["event_id"], "") if prev else "",
|
|
384
|
+
"index_in_session": i,
|
|
385
|
+
"seconds_since_previous": int((r["ts"] - prev["ts"]).total_seconds())
|
|
386
|
+
if prev else None,
|
|
387
|
+
"via": a.get("via"), "after_interrupt": bool(a.get("after_interrupt"))}))
|
|
388
|
+
return out
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _aw_blocks(task: Task, wins: list[dict[str, Any]], steer_ts: list[datetime]) -> list[Request]:
|
|
392
|
+
"""Contiguous same-category window time of AW_BLOCK_MIN_S or more with no steer inside.
|
|
393
|
+
shortcut: 'no steer inside' stands in for 'overlaps no steer interval'; upgrade by reading
|
|
394
|
+
agents.steer_interval once the writer may."""
|
|
395
|
+
blocks: list[list[dict[str, Any]]] = []
|
|
396
|
+
for w in wins:
|
|
397
|
+
b = blocks[-1] if blocks else None
|
|
398
|
+
if b and b[-1]["category"] == w["category"] \
|
|
399
|
+
and (w["start"] - b[-1]["end"]).total_seconds() <= AW_GAP_S:
|
|
400
|
+
b.append(w)
|
|
401
|
+
else:
|
|
402
|
+
blocks.append([w])
|
|
403
|
+
out = []
|
|
404
|
+
for b in blocks:
|
|
405
|
+
s, e = b[0]["start"], max(w["end"] for w in b)
|
|
406
|
+
secs = sum((w["end"] - w["start"]).total_seconds() for w in b)
|
|
407
|
+
if secs < AW_BLOCK_MIN_S or any(s <= t < e for t in steer_ts):
|
|
408
|
+
continue
|
|
409
|
+
titles = list(dict.fromkeys(w["title"] for w in b if w["title"]))[:5]
|
|
410
|
+
out.append(task.request("aw_block", make_event_id("aw_block", s.isoformat(),
|
|
411
|
+
e.isoformat()), s, {
|
|
412
|
+
"category": b[0]["category"], "apps": list(dict.fromkeys(w["app"] for w in b)),
|
|
413
|
+
"minutes": round(secs / 60), "titles": titles, "hour": s.astimezone().hour}))
|
|
414
|
+
return out
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _context_steers(src: Source, steers: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
418
|
+
"""Local opt-in covers supported agents; teacher sources retain their original boundary."""
|
|
419
|
+
return [r for r in steers if r["agent"] in src.context_agents]
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def _episode_days(mine: list[dict[str, Any]], acts: list[dict[str, Any]], of_session: dict[str, list[Episode]],
|
|
423
|
+
day_of: set[date]) -> dict[tuple[str, date], dict[str, Any]]:
|
|
424
|
+
"""(episode, local day) -> its steers, sessions and activity rows that day."""
|
|
425
|
+
days: dict[tuple[str, date], dict[str, Any]] = {}
|
|
426
|
+
for r in mine:
|
|
427
|
+
d = r["ts"].astimezone().date()
|
|
428
|
+
if d not in day_of:
|
|
429
|
+
continue
|
|
430
|
+
for ep in of_session.get(r["session_id"], ()):
|
|
431
|
+
if not ep["first_ts"] <= r["ts"] <= ep["last_ts"]:
|
|
432
|
+
continue
|
|
433
|
+
u = days.setdefault((ep["id"], d), {"steers": [], "sessions": set(), "acts": []})
|
|
434
|
+
u["steers"].append(r)
|
|
435
|
+
u["sessions"].add(r["session_id"])
|
|
436
|
+
for a in acts:
|
|
437
|
+
d = a["ts"].astimezone().date()
|
|
438
|
+
for ep in of_session.get(a["session_id"], ()):
|
|
439
|
+
if ep["first_ts"] <= a["ts"] <= ep["last_ts"] and (ep["id"], d) in days:
|
|
440
|
+
days[(ep["id"], d)]["acts"].append(a)
|
|
441
|
+
return days
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def _episode_day_unit(task: Task, ep: str, d: date, u: dict[str, Any], text: dict[str, str],
|
|
445
|
+
last_reply: str) -> Request | None:
|
|
446
|
+
"""The unit of one episode day; None when none of its steers has local text."""
|
|
447
|
+
st = u["steers"]
|
|
448
|
+
said = [text[r["event_id"]] for r in st if r["event_id"] in text]
|
|
449
|
+
if not said:
|
|
450
|
+
return None
|
|
451
|
+
tools: Counter[str] = Counter()
|
|
452
|
+
kinds = Counter(a["kind"] for a in u["acts"])
|
|
453
|
+
for a in u["acts"]:
|
|
454
|
+
if a["kind"] == "turn":
|
|
455
|
+
tools.update((a["attrs"] or {}).get("by_tool") or {})
|
|
456
|
+
return task.request("episode_day", f"{ep}:{d.isoformat()}", st[0]["ts"], {
|
|
457
|
+
"first_steer": said[0], "last_steers": said[1:][-3:], "steers": len(st),
|
|
458
|
+
"last_agent_reply": last_reply,
|
|
459
|
+
"sessions": len(u["sessions"]), "tool_calls_by_tool": dict(tools),
|
|
460
|
+
"tool_errors": sum((a["attrs"] or {}).get("tool_errors") or 0 for a in u["acts"]
|
|
461
|
+
if a["kind"] == "turn"),
|
|
462
|
+
"interrupts": kinds["interrupt"], "rejections": kinds["rejection"],
|
|
463
|
+
"commits": kinds["commit"]})
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def _category_units(task: Task, src: Source, since: datetime, until: datetime | None) -> list[Request]:
|
|
467
|
+
until = until or datetime.now(timezone.utc)
|
|
468
|
+
steers = [r for r in src.steers(since) if r["ts"] < until]
|
|
469
|
+
text = src.texts(steers, until)
|
|
470
|
+
steers = src.ordered_steers(steers, until)
|
|
471
|
+
mine = _context_steers(src, steers)
|
|
472
|
+
context_sessions = {r["session_id"] for r in mine}
|
|
473
|
+
eps = src.episodes(since)
|
|
474
|
+
of_session: dict[str, list[Episode]] = defaultdict(list)
|
|
475
|
+
for episode in eps:
|
|
476
|
+
for m in episode["members"]:
|
|
477
|
+
of_session[m["session_id"]].append(episode)
|
|
478
|
+
sids = sorted(context_sessions & set(of_session))
|
|
479
|
+
acts = src.activity(sids, since, until) if sids else []
|
|
480
|
+
day_of = set(local_days(since, until))
|
|
481
|
+
days = _episode_days(mine, acts, of_session, day_of)
|
|
482
|
+
by_id = {episode["id"]: episode for episode in eps}
|
|
483
|
+
out = []
|
|
484
|
+
for (ep, d), u in days.items():
|
|
485
|
+
cutoff = min(until, local_midnight(d + timedelta(days=1)),
|
|
486
|
+
by_id[ep]["last_ts"] + timedelta(microseconds=1))
|
|
487
|
+
reply = src.replies(u["steers"], cutoff)
|
|
488
|
+
u["steers"] = src.ordered_steers(u["steers"], cutoff)
|
|
489
|
+
req = _episode_day_unit(task, ep, d, u, text,
|
|
490
|
+
src.last_reply(u["steers"], reply, cutoff))
|
|
491
|
+
if req is None:
|
|
492
|
+
src.missing_text_units.append(u["steers"][0]["ts"])
|
|
493
|
+
elif task.name == "writing_activity" and any(r["event_id"] not in text for r in u["steers"]):
|
|
494
|
+
src.missing_text_units.append(u["steers"][0]["ts"])
|
|
495
|
+
src.partial_text_units.append(u["steers"][0]["ts"])
|
|
496
|
+
else:
|
|
497
|
+
out.append(req)
|
|
498
|
+
if any(r["event_id"] not in text for r in u["steers"]):
|
|
499
|
+
src.partial_text_units.append(u["steers"][0]["ts"])
|
|
500
|
+
if task.name == "writing_activity":
|
|
501
|
+
return [replace(r, facts={k: r.facts[k] for k in
|
|
502
|
+
("first_steer", "last_steers", "last_agent_reply")}) for r in out]
|
|
503
|
+
wins = src.windows(since, until)
|
|
504
|
+
out += _aw_blocks(task, wins, [r["ts"] for r in steers])
|
|
505
|
+
return out
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def _outcome_units(task: Task, src: Source, since: datetime, until: datetime | None) -> list[Request]:
|
|
509
|
+
"""Episodes without commits (git outcome no_code) touched in the last OUTCOME_DAYS, as they
|
|
510
|
+
stood at `until`: started before it, and only what was said before it."""
|
|
511
|
+
if task.ver >= 3 and not src.require_outcome_reply:
|
|
512
|
+
raise AutError("label: outcome contract 3 requires an opted-in bounded local source")
|
|
513
|
+
now = until or datetime.now(timezone.utc)
|
|
514
|
+
start = datetime(1970, 1, 1, tzinfo=timezone.utc) if src.mature_outcomes else since
|
|
515
|
+
eps = [e for e in src.episodes(start) if e["outcome"] == "no_code" and e["first_ts"] < now]
|
|
516
|
+
text_since = min([start, *(e["first_ts"] for e in eps)])
|
|
517
|
+
steers = [r for r in _context_steers(src, src.steers(text_since)) if r["ts"] < now] if eps else []
|
|
518
|
+
text = src.texts(steers, until)
|
|
519
|
+
out = []
|
|
520
|
+
for ep in eps:
|
|
521
|
+
mine = {m["session_id"] for m in ep["members"]}
|
|
522
|
+
st = [r for r in steers if r["session_id"] in mine
|
|
523
|
+
and ep["first_ts"] <= r["ts"] <= ep["last_ts"]]
|
|
524
|
+
said = [text[r["event_id"]] for r in st if r["event_id"] in text]
|
|
525
|
+
if not said:
|
|
526
|
+
src.missing_text_units.append(ep["first_ts"])
|
|
527
|
+
continue
|
|
528
|
+
if len(said) < len(st):
|
|
529
|
+
src.partial_text_units.append(ep["first_ts"])
|
|
530
|
+
# Session membership can span multiple episodes. Later replies and copied earlier
|
|
531
|
+
# prompts must not turn this episode into a resolved task.
|
|
532
|
+
cutoff = min(now, ep["last_ts"] + timedelta(microseconds=1))
|
|
533
|
+
reply = src.replies(st, cutoff)
|
|
534
|
+
st = src.ordered_steers(st, cutoff)
|
|
535
|
+
said = [text[r["event_id"]] for r in st if r["event_id"] in text]
|
|
536
|
+
last_reply = src.last_reply(st, reply, cutoff)
|
|
537
|
+
if src.require_outcome_reply and (len(said) < len(st) or not last_reply.strip()):
|
|
538
|
+
src.incomplete_outcome_units.append(ep["first_ts"])
|
|
539
|
+
continue
|
|
540
|
+
# ts = the episode start, so the row's day (and its fold) never moves; rev =
|
|
541
|
+
# its last activity, so a changed episode is re-asked and an unchanged one is a no-op
|
|
542
|
+
out.append(task.request("episode", ep["id"], ep["first_ts"], {
|
|
543
|
+
"last_steers": said[-5:],
|
|
544
|
+
"last_agent_reply": last_reply,
|
|
545
|
+
"days_idle": max((now - ep["last_ts"]).days, 0),
|
|
546
|
+
"sessions": len({r["session_id"] for r in st}), "steers": len(st)},
|
|
547
|
+
rev=ep["last_ts"].astimezone(timezone.utc).isoformat()))
|
|
548
|
+
return out
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _title_units(task: Task, src: Source, since: datetime, until: datetime | None) -> list[Request]:
|
|
552
|
+
"""One unit per distinct browser (app, title); unit_id is an HMAC under the install salt, so
|
|
553
|
+
it cannot confirm a guessed title. None without a salt: then nothing is labelled."""
|
|
554
|
+
seen: dict[str, dict[str, Any]] = {}
|
|
555
|
+
for w in src.windows(since, until or datetime.now(timezone.utc)):
|
|
556
|
+
if not w["title"] or not w["rule"].startswith("browser_"):
|
|
557
|
+
continue
|
|
558
|
+
key = src.keys.of(f"aw_title|{w['app'].lower()}|{w['title'].strip().lower()}")
|
|
559
|
+
if key is None:
|
|
560
|
+
return []
|
|
561
|
+
u = seen.setdefault(key, {"ts": w["start"], "app": w["app"], "title": w["title"],
|
|
562
|
+
"category": w["category"], "s": 0.0})
|
|
563
|
+
u["s"] += (w["end"] - w["start"]).total_seconds()
|
|
564
|
+
return [task.request("aw_title", k, u["ts"], {
|
|
565
|
+
"app": u["app"], "title": u["title"], "title_rule": u["category"],
|
|
566
|
+
"minutes": round(u["s"] / 60, 1)}) for k, u in seen.items()]
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
UNITS: dict[str, Callable[[Task, Source, datetime, datetime | None], list[Request]]] = {
|
|
570
|
+
"steer_intent": _steer_units, "work_category": _category_units,
|
|
571
|
+
"writing_activity": _category_units,
|
|
572
|
+
"soft_outcome": _outcome_units, "aw_title_relevance": _title_units,
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def units(name: str, src: Source, since: datetime, until: datetime | None) -> list[Request]:
|
|
577
|
+
"""The task's requests in [since, until), oldest first. An episode unit belongs by its
|
|
578
|
+
activity, so its ts (the episode start) can be before `since`."""
|
|
579
|
+
return sorted(UNITS[name](TASKS[name], src, since, until), key=lambda r: (r.ts, r.unit_id))
|