agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""Repo identity and its salted key: what a row stores instead of a repo, branch or ticket name."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import hmac
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import secrets
|
|
10
|
+
import subprocess # nosec B404 # one git_argv call below, no shell
|
|
11
|
+
import sys
|
|
12
|
+
from urllib.parse import urlparse
|
|
13
|
+
|
|
14
|
+
from .config import AutError
|
|
15
|
+
from .git_safe import git_argv, git_env
|
|
16
|
+
from .sink import connect
|
|
17
|
+
from .state import state_path, warn_if_group_readable
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def repo_identity(url: str) -> str:
|
|
21
|
+
"""host/owner/repo, lowercased, with credentials, port and a .git suffix dropped, so the https
|
|
22
|
+
and scp-style remotes of one repo (and a PR's host + repository) give the same key."""
|
|
23
|
+
m = re.match(r"^[\w.-]+@([^:/]+):(.+)$", url) # git@host:owner/repo
|
|
24
|
+
if m:
|
|
25
|
+
host, path = m.groups()
|
|
26
|
+
else:
|
|
27
|
+
p = urlparse(url) # raises ValueError on a malformed URL
|
|
28
|
+
host, path = p.hostname or "", p.path
|
|
29
|
+
return f"{host}/{path.strip('/').removesuffix('.git')}".lower()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _salt_text(raw: bytes) -> str:
|
|
33
|
+
"""The salt a salt file holds; "" when it is not text or too short to be one."""
|
|
34
|
+
try:
|
|
35
|
+
text = raw.decode("utf-8").strip()
|
|
36
|
+
except UnicodeDecodeError:
|
|
37
|
+
return ""
|
|
38
|
+
return text if len(text) >= 32 else ""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def require_readable_salt(configured: str | None, dsn: str | None = None,
|
|
42
|
+
machine: str | None = None) -> None:
|
|
43
|
+
"""Raises before a run that sends rows, when the salt file is there but cannot be read or is
|
|
44
|
+
corrupt: NULL keys are permanent once the watermark moves on and the sink's ON CONFLICT DO
|
|
45
|
+
NOTHING applies. A corrupt file is never replaced here: a new salt changes every key. Raises
|
|
46
|
+
too when the file is missing but the sink already holds this machine's keys (a new laptop
|
|
47
|
+
without the old salt): a new salt would split every repo's old and new rows for good."""
|
|
48
|
+
if configured:
|
|
49
|
+
return
|
|
50
|
+
p = state_path().parent / "repo_key.salt"
|
|
51
|
+
try:
|
|
52
|
+
raw = p.read_bytes()
|
|
53
|
+
except FileNotFoundError:
|
|
54
|
+
refuse_new_salt(configured, dsn, machine)
|
|
55
|
+
return
|
|
56
|
+
except OSError as exc:
|
|
57
|
+
raise AutError(f"aut: could not read {p} ({exc}); fix its owner or mode, or every repo_key "
|
|
58
|
+
"of this run would be NULL for good") from exc
|
|
59
|
+
if not _salt_text(raw):
|
|
60
|
+
raise AutError(f"aut: {p} is too short or not text, so every repo_key of this run would be "
|
|
61
|
+
"NULL for good; restore it, or delete it to start new keys (old rows will "
|
|
62
|
+
"no longer match new ones)")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def refuse_new_salt(configured: str | None, dsn: str | None, machine: str | None) -> None:
|
|
66
|
+
"""Called before any command that can create repo_key.salt (RepoKeys makes it on first use: a
|
|
67
|
+
scan, a dry run, a label run, outcomes). Raises when there is no salt yet but the sink already
|
|
68
|
+
holds this machine's repo keys. With no sink to ask (webhook only), it warns instead."""
|
|
69
|
+
p = state_path().parent / "repo_key.salt"
|
|
70
|
+
if configured or p.exists():
|
|
71
|
+
return
|
|
72
|
+
if not dsn:
|
|
73
|
+
print(f"aut: warning: no {p} and no pg_dsn to ask the sink, so a new repo_key.salt is "
|
|
74
|
+
"made. If this machine sent rows before (a new laptop), stop and restore the old "
|
|
75
|
+
"salt first (docs/new-laptop.md).", file=sys.stderr)
|
|
76
|
+
return
|
|
77
|
+
if _sink_has_keys(dsn, machine or ""):
|
|
78
|
+
raise AutError(
|
|
79
|
+
f"aut: {p} is missing, but the sink already holds repo keys from machine "
|
|
80
|
+
f"'{machine}'. A new salt would never match them. Copy the old repo_key.salt "
|
|
81
|
+
"there, or set repo_key_salt in config.toml to its contents, then run again "
|
|
82
|
+
"(docs/new-laptop.md). Nothing was sent.")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _sink_has_keys(dsn: str, machine: str) -> bool:
|
|
86
|
+
"""Whether any activity row of this machine carries a repo_key. A read error raises: going on
|
|
87
|
+
would mint a new salt before anyone could tell."""
|
|
88
|
+
with connect(dsn) as conn:
|
|
89
|
+
return conn.execute("SELECT 1 FROM agents.activity_event WHERE machine = %s "
|
|
90
|
+
"AND attrs->>'repo_key' IS NOT NULL LIMIT 1", (machine,)).fetchone() \
|
|
91
|
+
is not None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _load_salt(configured: str | None, create: bool = True) -> bytes:
|
|
95
|
+
"""The config's repo_key_salt, else a random salt kept next to the state file (created 0600 on
|
|
96
|
+
first use, unless not `create`). b"" when neither can be had: repo keys are then NULL, never an
|
|
97
|
+
unsalted hash."""
|
|
98
|
+
if configured:
|
|
99
|
+
return configured.encode("utf-8")
|
|
100
|
+
p = state_path().parent / "repo_key.salt"
|
|
101
|
+
try:
|
|
102
|
+
if not p.exists():
|
|
103
|
+
if not create:
|
|
104
|
+
return b""
|
|
105
|
+
# Written in full to a temp file, then linked into place: a crash or a second process
|
|
106
|
+
# never sees a half-written salt, and the first link wins a race.
|
|
107
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
108
|
+
tmp = p.with_name(f"{p.name}.{os.getpid()}.tmp")
|
|
109
|
+
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
110
|
+
with os.fdopen(fd, "w") as fh:
|
|
111
|
+
fh.write(secrets.token_hex(32))
|
|
112
|
+
try:
|
|
113
|
+
os.link(tmp, p)
|
|
114
|
+
except FileExistsError: # another process linked first: its salt wins
|
|
115
|
+
pass
|
|
116
|
+
finally:
|
|
117
|
+
tmp.unlink(missing_ok=True)
|
|
118
|
+
salt = _salt_text(p.read_bytes())
|
|
119
|
+
warn_if_group_readable(p)
|
|
120
|
+
except OSError as exc:
|
|
121
|
+
print(f"aut: warning: could not read or create {p} ({exc}); repo_key stays NULL until it "
|
|
122
|
+
"is fixed", file=sys.stderr)
|
|
123
|
+
return b""
|
|
124
|
+
if not salt:
|
|
125
|
+
print(f"aut: warning: {p} is too short or not text; repo_key stays NULL until it is fixed",
|
|
126
|
+
file=sys.stderr)
|
|
127
|
+
return b""
|
|
128
|
+
return salt.encode("utf-8")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class RepoKeys:
|
|
132
|
+
"""repo_key = HMAC-SHA256(salt, host/owner/repo), so a list of candidate repo names cannot be
|
|
133
|
+
tested against stored keys without the salt. Cached per cwd for one collect() pass. With
|
|
134
|
+
create False a missing salt file stays missing and the keys are NULL (a dry run)."""
|
|
135
|
+
|
|
136
|
+
def __init__(self, salt: str | None, *, create: bool = True) -> None:
|
|
137
|
+
self._configured, self._create = salt, create
|
|
138
|
+
self._salt: bytes | None = None
|
|
139
|
+
self._cwd: dict[object, str | None] = {}
|
|
140
|
+
|
|
141
|
+
def of(self, identity: str) -> str | None:
|
|
142
|
+
if self._salt is None:
|
|
143
|
+
self._salt = _load_salt(self._configured, self._create)
|
|
144
|
+
if not self._salt:
|
|
145
|
+
return None
|
|
146
|
+
return hmac.new(self._salt, identity.encode("utf-8"), hashlib.sha256).hexdigest()
|
|
147
|
+
|
|
148
|
+
def of_cwd(self, cwd: object) -> str | None:
|
|
149
|
+
"""Key of the origin remote for a working dir; None when the directory or the remote is
|
|
150
|
+
gone. The URL itself is never stored or logged."""
|
|
151
|
+
if not isinstance(cwd, str):
|
|
152
|
+
return None
|
|
153
|
+
if cwd not in self._cwd:
|
|
154
|
+
self._cwd[cwd] = self._origin_key(cwd) if os.path.isdir(cwd) else None
|
|
155
|
+
return self._cwd[cwd]
|
|
156
|
+
|
|
157
|
+
def _origin_key(self, cwd: str) -> str | None:
|
|
158
|
+
try:
|
|
159
|
+
r = subprocess.run( # nosec B603 # constant args after -C cwd, no shell
|
|
160
|
+
git_argv(cwd, "remote", "get-url", "origin"),
|
|
161
|
+
capture_output=True, timeout=2, env=git_env())
|
|
162
|
+
url = r.stdout.decode("utf-8", "replace").strip()
|
|
163
|
+
return self.of(repo_identity(url)) if r.returncode == 0 and url else None
|
|
164
|
+
except (OSError, subprocess.SubprocessError, ValueError):
|
|
165
|
+
return None # no git, a stuck remote config, a malformed URL: the key is NULL
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
"""`aut episodes review [--month YYYY-MM]`: hand-check one month's episodes on a local page.
|
|
2
|
+
|
|
3
|
+
The page lists the month's episodes (agents.episode / episode_member). Each seed session shows
|
|
4
|
+
what THIS machine's logs (Claude Code, Codex, Pi, Cursor CLI) and their git commands say: the
|
|
5
|
+
agent, the project folder, when it ran and for how long, how many prompts, the first and last
|
|
6
|
+
prompt (redacted, about 200 characters), the commit subjects and PR titles it made, the files it
|
|
7
|
+
edited most, its titles and branch names. That text is read when the page is built and exists
|
|
8
|
+
only in the page served from 127.0.0.1 under a random token; it is never written to Postgres, a
|
|
9
|
+
log, the terminal or a file.
|
|
10
|
+
|
|
11
|
+
To ask less, the page shows a sample of 30 episodes spread over outcome and size (a link shows
|
|
12
|
+
all), and leaves out episodes already confirmed or rejected. Nothing but session ids and repo
|
|
13
|
+
keys is written back, as agents.episode_correction rows: right -> confirm, should split -> split
|
|
14
|
+
(per chosen session), should merge -> merge (with the chosen episode), not building work ->
|
|
15
|
+
reject. "don't know", the default, writes nothing.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import html
|
|
21
|
+
import random
|
|
22
|
+
import uuid
|
|
23
|
+
from collections import Counter
|
|
24
|
+
from datetime import datetime, timedelta, timezone
|
|
25
|
+
from typing import Any, Callable
|
|
26
|
+
|
|
27
|
+
from . import episode_store, local_page
|
|
28
|
+
from .episode_store import Episode, EpisodeMember
|
|
29
|
+
from .collectors.session_digest import SessionView, local_info
|
|
30
|
+
from .config import AutError, Config
|
|
31
|
+
|
|
32
|
+
_ACTIONS = {"right": "confirm", "reject": "reject"}
|
|
33
|
+
SAMPLE = 30
|
|
34
|
+
_CSS = """body { overflow-wrap: anywhere; }
|
|
35
|
+
fieldset { border: 1px solid var(--line); border-radius: 6px; margin: 0 0 1rem; }
|
|
36
|
+
.meta, .none { color: var(--mut); font-size: 13px; }
|
|
37
|
+
ul { margin: .3rem 0; padding-left: 1.2rem; } li { margin-bottom: .5rem; }
|
|
38
|
+
label { margin-right: 1rem; display: inline-block; }
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _seeds(ep: Episode) -> list[EpisodeMember]:
|
|
43
|
+
return [m for m in ep["members"] if m["role"] == "seed"] or ep["members"]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def sample(episodes: list[Episode], n: int = SAMPLE) -> list[int]:
|
|
47
|
+
"""Indices of up to n episodes, taken in turn from each (outcome, size) group so every kind
|
|
48
|
+
shows up; the same month gives the same sample."""
|
|
49
|
+
groups: dict[tuple[str, int], list[int]] = {}
|
|
50
|
+
for i, ep in enumerate(episodes):
|
|
51
|
+
k = ep["n_sessions"] or 0
|
|
52
|
+
groups.setdefault((str(ep["outcome"]), 0 if k <= 1 else 1 if k <= 3 else 2), []).append(i)
|
|
53
|
+
rng = random.Random(0) # nosec B311 # a repeatable review sample, not security
|
|
54
|
+
queues = [rng.sample(g, len(g)) for _k, g in sorted(groups.items())]
|
|
55
|
+
picked: list[int] = []
|
|
56
|
+
while len(picked) < n and any(queues):
|
|
57
|
+
for q in queues:
|
|
58
|
+
if q and len(picked) < n:
|
|
59
|
+
picked.append(q.pop())
|
|
60
|
+
return sorted(picked)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _plural(n: int, word: str) -> str:
|
|
64
|
+
return f"{n} {word}{'' if n == 1 else 's'}"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _summary(ep: Episode, views: list[SessionView]) -> str:
|
|
68
|
+
days = (ep["last_ts"].date() - ep["first_ts"].date()).days + 1
|
|
69
|
+
when = f"on {ep['first_ts']:%a %d %b}" if days == 1 else f"over {days} days"
|
|
70
|
+
folders = Counter(v["folder"] for v in views if v.get("folder"))
|
|
71
|
+
where = f" in {folders.most_common(1)[0][0]}" if folders else ""
|
|
72
|
+
return (f"{_plural(ep['n_sessions'], 'session')} {when}{where}, "
|
|
73
|
+
f"{_plural(ep['n_commits'], 'commit')}, {_plural(ep['n_prs'], 'PR')}")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _start_and_duration(v: SessionView) -> str:
|
|
77
|
+
start, m = v.get("start"), v.get("minutes")
|
|
78
|
+
if start is None or m is None: # a session with no local log: both unset
|
|
79
|
+
return ""
|
|
80
|
+
return f"{start:%a %d %b %H:%M}, ran " + (f"{m // 60}h {m % 60}m" if m >= 60 else f"{m}m")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _session_html(v: SessionView, sid: str, i: int) -> str:
|
|
84
|
+
e = html.escape
|
|
85
|
+
head = "; ".join(v.get("titles", [])[-3:]) or v.get("first") or "(nothing in the local logs)"
|
|
86
|
+
meta = [x for x in (v.get("agent"), v.get("folder"), _start_and_duration(v),
|
|
87
|
+
_plural(v["n_prompts"], "prompt") if v.get("agent") else "",
|
|
88
|
+
", ".join(v.get("branches", [])[:3])) if x]
|
|
89
|
+
out = [f"<li><b>{e(head)}</b>"
|
|
90
|
+
f" <span class=meta>{e(' · '.join(meta))}</span>"]
|
|
91
|
+
for label, said in (("First prompt", v.get("first")), ("Last prompt", v.get("last"))):
|
|
92
|
+
if said:
|
|
93
|
+
out.append(f"<div><span class=meta>{label}:</span> {e(said)}</div>")
|
|
94
|
+
for label, items in (("Commits", v.get("commits")), ("PRs", v.get("prs")),
|
|
95
|
+
("Files", v.get("files"))):
|
|
96
|
+
if items:
|
|
97
|
+
out.append(f"<div class=meta>{label}: {e('; '.join(items))}</div>")
|
|
98
|
+
out.append(f'<label><input type=checkbox name="split{i}" value="{e(sid)}"> take out</label>'
|
|
99
|
+
"</li>")
|
|
100
|
+
return "".join(out)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
_ANSWERS = (("skip", "don't know"), ("right", "right"), ("split", "should split"),
|
|
104
|
+
("merge", "should merge with #"),
|
|
105
|
+
("reject", "not building work (admin, personal, a quick question)"))
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def render(episodes: list[Episode], info: dict[str, SessionView], token: str, heading: str,
|
|
109
|
+
shown: list[int] | None = None, answered: int = 0, kind: str = "episodes") -> str:
|
|
110
|
+
"""The page. `shown`: indices of the episodes on it (default all); form fields keep each
|
|
111
|
+
episode's index in `episodes`, so the sample and the full page post the same answers. `kind`
|
|
112
|
+
names them in the note."""
|
|
113
|
+
e = html.escape
|
|
114
|
+
shown = list(range(len(episodes))) if shown is None else shown
|
|
115
|
+
whole = len(shown) == len(episodes)
|
|
116
|
+
note = (f"All {len(episodes)} {kind}." if whole else
|
|
117
|
+
f"A sample of {len(shown)} of {len(episodes)} {kind}, spread over outcome "
|
|
118
|
+
f'and size. <a href="/{e(token)}?all=1">Show all</a>.')
|
|
119
|
+
if answered:
|
|
120
|
+
note += f" {answered} already answered (not shown)."
|
|
121
|
+
parts = [local_page.head("Episode review", "60rem", _CSS) + f"""<h1>Episodes of {e(heading)}</h1>
|
|
122
|
+
<p>Private page on this machine only. Only your answers (session ids and repo keys) are sent.
|
|
123
|
+
{note}</p>
|
|
124
|
+
<form method="post" action="/{e(token)}">"""]
|
|
125
|
+
for i in shown:
|
|
126
|
+
ep = episodes[i]
|
|
127
|
+
views = [info.get(m["session_id"], {}) for m in _seeds(ep)]
|
|
128
|
+
parts.append(f"<fieldset><legend>#{i} {ep['first_ts']:%Y-%m-%d} to "
|
|
129
|
+
f"{ep['last_ts']:%Y-%m-%d}, {e(str(ep['outcome']))}</legend>"
|
|
130
|
+
f"<div>{e(_summary(ep, views))}</div><ul>")
|
|
131
|
+
parts += [_session_html(v, m["session_id"], i) for m, v in zip(_seeds(ep), views)]
|
|
132
|
+
kids = len(ep["members"]) - len(_seeds(ep))
|
|
133
|
+
parts.append(f"</ul><div class=none>{kids} child sessions</div>" if kids else "</ul>")
|
|
134
|
+
for v, label in _ANSWERS:
|
|
135
|
+
parts.append(f'<label><input type=radio name="a{i}" value="{v}"'
|
|
136
|
+
f'{" checked" if v == "skip" else ""}> {label}</label>')
|
|
137
|
+
parts.append(f'<input name="merge{i}" size=3 aria-label="episode number to merge with">'
|
|
138
|
+
"</fieldset>")
|
|
139
|
+
parts.append("<button type=submit>Save answers</button></form></body></html>")
|
|
140
|
+
return "".join(parts)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _correction(now: datetime, action: str, a: EpisodeMember,
|
|
144
|
+
b: EpisodeMember | None = None) -> dict[str, Any]:
|
|
145
|
+
attrs = {"repo_keys": a["repo_keys"]}
|
|
146
|
+
if b:
|
|
147
|
+
attrs["b_repo_keys"] = b["repo_keys"]
|
|
148
|
+
return {"id": str(uuid.uuid4()), "ts": now, "action": action, "a_session": a["session_id"],
|
|
149
|
+
"b_session": b["session_id"] if b else None, "attrs": attrs}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _field(form: dict[str, list[str]], name: str) -> str:
|
|
153
|
+
return (form.get(name) or [""])[0]
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _answer_rows(form: dict[str, list[str]], i: int, episodes: list[Episode],
|
|
157
|
+
now: datetime) -> list[dict[str, Any]]:
|
|
158
|
+
"""The rows of episode i's answer."""
|
|
159
|
+
ans = _field(form, f"a{i}")
|
|
160
|
+
seeds = _seeds(episodes[i])
|
|
161
|
+
if ans == "split":
|
|
162
|
+
by_id = {m["session_id"]: m for m in seeds}
|
|
163
|
+
return [_correction(now, "split", by_id[s]) for s in form.get(f"split{i}", [])
|
|
164
|
+
if s in by_id]
|
|
165
|
+
if ans == "merge":
|
|
166
|
+
j = _field(form, f"merge{i}").strip().lstrip("#")
|
|
167
|
+
ok = j.isdecimal() and len(j) < 10 and int(j) < len(episodes) and int(j) != i
|
|
168
|
+
return [_correction(now, "merge", seeds[0], _seeds(episodes[int(j)])[0])] if ok else []
|
|
169
|
+
return [_correction(now, _ACTIONS[ans], seeds[0])] if ans in _ACTIONS else []
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def corrections(form: dict[str, list[str]], episodes: list[Episode],
|
|
173
|
+
now: datetime) -> tuple[list[dict[str, Any]], dict[str, int]]:
|
|
174
|
+
"""episode_correction rows for the answers, and {reviewed, right, saved}: saved counts the rows,
|
|
175
|
+
since a split writes one per session. Answers that name a
|
|
176
|
+
session or episode the page did not show are dropped."""
|
|
177
|
+
rows: list[dict[str, Any]] = []
|
|
178
|
+
reviewed = right = 0
|
|
179
|
+
for i in range(len(episodes)):
|
|
180
|
+
new = _answer_rows(form, i, episodes, now)
|
|
181
|
+
rows += new
|
|
182
|
+
reviewed += bool(new)
|
|
183
|
+
right += bool(new) and _field(form, f"a{i}") == "right"
|
|
184
|
+
return rows, {"reviewed": reviewed, "right": right, "saved": len(rows)}
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def month_pages(episodes: list[Episode], info: dict[str, SessionView], heading: str,
|
|
188
|
+
answered: int = 0) -> Callable[[str], dict[str, str]]:
|
|
189
|
+
"""The sample at /<token>, every episode at /<token>?all=1."""
|
|
190
|
+
return lambda token: {
|
|
191
|
+
"": render(episodes, info, token, heading, sample(episodes), answered, "open episodes"),
|
|
192
|
+
"?all=1": render(episodes, info, token, heading, None, answered, "open episodes")}
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def serve(episodes: list[Episode], write: Callable[[list[dict[str, Any]]], None],
|
|
196
|
+
pages: Callable[[str], dict[str, str]], *, open_browser: bool = True,
|
|
197
|
+
on_ready: Callable[[str], None] | None = None,
|
|
198
|
+
now: datetime | None = None) -> dict[str, int]:
|
|
199
|
+
"""Serve pages(token) (what follows /<token> in the path -> html) until the answers are posted;
|
|
200
|
+
write(rows) stores them. Returns {reviewed, right, saved}."""
|
|
201
|
+
token = local_page.new_token()
|
|
202
|
+
built = {k: v.encode() for k, v in pages(token).items()}
|
|
203
|
+
result = {"reviewed": 0, "right": 0, "saved": 0}
|
|
204
|
+
|
|
205
|
+
def on_post(form: dict[str, list[str]]) -> bytes:
|
|
206
|
+
rows, summary = corrections(form, episodes, now or datetime.now(timezone.utc))
|
|
207
|
+
write(rows)
|
|
208
|
+
result.update(summary)
|
|
209
|
+
return (f"<p>Saved. Marked right: {summary['right']} of {summary['reviewed']} reviewed. "
|
|
210
|
+
"You can close this tab.</p>").encode()
|
|
211
|
+
|
|
212
|
+
local_page.serve(token, built.get, on_post, "local page at {url} (Ctrl-C to stop without saving)",
|
|
213
|
+
open_browser=open_browser, on_ready=on_ready)
|
|
214
|
+
return result
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _month_bounds(month: str | None) -> tuple[str, datetime, datetime]:
|
|
218
|
+
"""(label, start, end) of a local calendar month; default the last completed one."""
|
|
219
|
+
if month:
|
|
220
|
+
try:
|
|
221
|
+
start = datetime.strptime(month, "%Y-%m")
|
|
222
|
+
except ValueError:
|
|
223
|
+
raise AutError("episodes review: --month is YYYY-MM") from None
|
|
224
|
+
else:
|
|
225
|
+
start = (datetime.now().replace(day=1) - timedelta(days=1)).replace(day=1)
|
|
226
|
+
start = start.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
|
227
|
+
end = start.replace(year=start.year + start.month // 12, month=start.month % 12 + 1)
|
|
228
|
+
return f"{start:%Y-%m}", start.astimezone(), end.astimezone()
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def run_episode_review(cfg: Config, month: str | None) -> dict[str, Any]:
|
|
232
|
+
dsn = cfg.require_pg("episodes review")
|
|
233
|
+
label, start, end = _month_bounds(month)
|
|
234
|
+
episodes = episode_store.load_month(dsn, start, end)
|
|
235
|
+
if not episodes:
|
|
236
|
+
raise AutError(f"episodes review: no episodes start in {label}")
|
|
237
|
+
done = episode_store.answered_sessions(dsn, [_seeds(ep)[0]["session_id"] for ep in episodes])
|
|
238
|
+
left = [ep for ep in episodes if _seeds(ep)[0]["session_id"] not in done]
|
|
239
|
+
if not left:
|
|
240
|
+
raise AutError(f"episodes review: all {len(episodes)} episodes of {label} are answered")
|
|
241
|
+
info = local_info(cfg.home, {m["session_id"] for ep in left for m in _seeds(ep)})
|
|
242
|
+
summary = serve(left, lambda rows: episode_store.insert_corrections(dsn, rows),
|
|
243
|
+
month_pages(left, info, label, len(episodes) - len(left)))
|
|
244
|
+
return {**summary, "month": label, "open": len(left),
|
|
245
|
+
"answered_before": len(episodes) - len(left)}
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def print_review(report: dict[str, Any]) -> None:
|
|
249
|
+
share = report["right"] / report["reviewed"] if report["reviewed"] else 0.0
|
|
250
|
+
print(f"episodes review {report['month']}: {report['reviewed']} of {report['open']} episodes "
|
|
251
|
+
f"reviewed ({report['answered_before']} answered before), {report['right']} marked "
|
|
252
|
+
f"right ({share:.0%})")
|