agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,499 @@
1
+ """`aut git-days`: the owner's commits per ledger day (ledger_timezone), from every clone, to agents.git_commit_day.
2
+
3
+ Reads branches, remotes, tags and all registered worktree heads (no merges, stash or notes),
4
+ including detached heads outside scan roots, in each object store under [git_days] roots and,
5
+ over `ssh -o BatchMode=yes`, under each remote's roots, with the same hardened git (git_safe). No
6
+ fetch. A commit counts once (by SHA) when its author email is one of the owner's identities, or a
7
+ shared author's in a repo whose name matches that author's repo_regex; a bot never counts.
8
+ Only counts and completeness leave the Mac. Raw repo names, SHAs and emails stay in memory.
9
+ Salted commit identities and per-source coverage stay in a private local cache.
10
+
11
+ The table is insert-only and a day's latest scan wins, so a run inserts, with one scanned_at, the
12
+ days from `since` to yesterday that have no row yet or whose count went up. A stored day never goes
13
+ down by itself: a lower count is only counted in the summary, and `--allow-lower` rewrites it.
14
+ Partial scans retain known commits without inventing zero days. Each source keeps its last
15
+ complete coverage. Missing sources remain retry-eligible. `--allow-lower` requires a fresh
16
+ complete scan of every configured source and resets the private identity cache.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import json
22
+ import os
23
+ import re
24
+ import shlex
25
+ import subprocess # nosec B404 # argv lists only; the remote script goes to ssh on stdin
26
+ import sys
27
+ from dataclasses import dataclass, field
28
+ from contextlib import nullcontext
29
+ from datetime import date, datetime, timedelta, timezone, tzinfo
30
+ from pathlib import Path, PurePosixPath
31
+ from typing import Any, Callable, Iterable
32
+ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
33
+
34
+ from .config import AutError, Config
35
+ from .git_safe import HARDENED_ENV, git_argv
36
+ from .git_day_cache import Snapshot, Status, load_cache, merge_sources
37
+ from .push import JobBeat, job_heartbeat, redacted_error
38
+ from .sink import check_ledger_zone, connect, write_many
39
+ from .repo_keys import RepoKeys, require_readable_salt
40
+ from .state import _locked, atomic_write_text, collector_wm, load_state, save_state, state_path
41
+
42
+ NAME = "git_days" # its aut state entry and heartbeat collector
43
+ LOG_ARGS = ("log", "--branches", "--remotes", "--tags", "--no-merges", "--format=%H%x09%aI%x09%ae")
44
+ _PRUNE = ("node_modules", ".venv", "venv", "vendor", "__pycache__")
45
+ _MAX_DEPTH = 6 # folders below a root; its .git sits one deeper
46
+ _BOT = re.compile(r"\[bot\]|^noreply@anthropic\.com$")
47
+ MIN_STORE_SHARE = 0.5
48
+ KEEP_STORES = 5 # good runs whose best store count the share rule uses
49
+ DUE_AFTER = timedelta(hours=20)
50
+ RETRY_AFTER = timedelta(hours=6) # after an attempt that failed or was refused
51
+ _REMOTE_SCAN_VERSION = 1 # retry older partial remote reads once after this repair
52
+ SSH_CONNECT_S = 10
53
+ REMOTE_TIMEOUT_S = 900
54
+ INSERT_SQL = "INSERT INTO agents.git_commit_day (day, commits, scanned_at, complete) VALUES (%s, %s, %s, %s)"
55
+ READ_ENV = {**HARDENED_ENV, "GIT_NO_LAZY_FETCH": "1", "GIT_ALLOW_PROTOCOL": "", "LC_ALL": "C"}
56
+
57
+
58
+ def _read_argv(path: str, sub: str, *args: str) -> list[str]:
59
+ # Configured physical roots authorize reads of shared clones, never wildcard or global trust.
60
+ argv = git_argv(path, sub, *args)
61
+ return [*argv[:3], "-c", "safe.directory=", "-c", f"safe.directory={path}", *argv[3:]]
62
+
63
+
64
+ def _read_git(path: str, sub: str, *args: str) -> subprocess.CompletedProcess[str] | None:
65
+ try:
66
+ return subprocess.run(_read_argv(path, sub, *args), capture_output=True, text=True,
67
+ errors="replace", timeout=300, env={**os.environ, **READ_ENV})
68
+ except (OSError, subprocess.SubprocessError):
69
+ return None
70
+
71
+
72
+ @dataclass(slots=True)
73
+ class Found:
74
+ stores: int = 0 # distinct object stores read
75
+ failed: int = 0 # hosts and stores that could not be read
76
+ commits: dict[str, date] = field(default_factory=dict) # counted SHA -> local day
77
+ sources: list[tuple[str, Found, bool]] = field(default_factory=list)
78
+
79
+
80
+ def repo_name(common: str) -> str:
81
+ """The repo's folder name from its git common dir: /x/name/.git, /x/name.git, .git/modules/name."""
82
+ p = PurePosixPath(common.replace("\\", "/"))
83
+ return (p.parent.name if p.name == ".git" else p.name).removesuffix(".git")
84
+
85
+
86
+ def _owned(cfg: Config, email: str, repo: str) -> bool:
87
+ if not email or _BOT.search(email):
88
+ return False
89
+ return email in cfg.git_days_identities or any(
90
+ re.search(e, email, re.IGNORECASE) and re.search(r, repo)
91
+ for e, r in cfg.git_days_shared_authors)
92
+
93
+
94
+ def _take(log: str, repo: str, cfg: Config, tz: tzinfo, into: dict[str, date]) -> None:
95
+ for line in log.splitlines():
96
+ sha, _, rest = line.partition("\t")
97
+ when, _, email = rest.partition("\t")
98
+ if sha in into or not _owned(cfg, email.strip().lower(), repo):
99
+ continue
100
+ try:
101
+ into[sha] = datetime.fromisoformat(when).astimezone(tz).date()
102
+ except ValueError:
103
+ continue
104
+
105
+
106
+ def _scan_local(cfg: Config, tz: tzinfo, found: Found) -> None:
107
+ seen: set[str] = set()
108
+
109
+ def unreadable(_exc: OSError) -> None:
110
+ found.failed += 1
111
+
112
+ for root in cfg.git_days_roots:
113
+ base = os.path.realpath(os.path.expanduser(root))
114
+ if base in cfg.git_days_exclude_dirs:
115
+ continue
116
+ if not os.path.isdir(base): # missing or unmounted: its repos would read as zero
117
+ found.failed += 1
118
+ continue
119
+ for dirpath, dirnames, filenames in os.walk(base, onerror=unreadable):
120
+ depth = dirpath[len(base):].count(os.sep)
121
+ if ".git" in dirnames or ".git" in filenames:
122
+ rev = _read_git(dirpath, "rev-parse", "--path-format=absolute", "--git-common-dir")
123
+ common = rev.stdout.strip() if rev is not None and rev.returncode == 0 else \
124
+ "" if rev is not None and ("not a git repository" in rev.stderr or
125
+ "invalid gitfile format" in rev.stderr) else None
126
+ if common is None:
127
+ found.failed += 1
128
+ elif common and os.path.realpath(common) not in seen:
129
+ seen.add(os.path.realpath(common))
130
+ found.stores += 1
131
+ wt = _read_git(dirpath, "worktree", "list", "--porcelain")
132
+ worktrees = wt.stdout if wt is not None and wt.returncode == 0 else None
133
+ heads = [line[5:] for line in (worktrees or "").splitlines()
134
+ if line.startswith("HEAD ") and line[5:].strip("0")]
135
+ log = _read_git(dirpath, *LOG_ARGS, *heads) if worktrees is not None else None
136
+ if log is None or log.returncode:
137
+ found.failed += 1
138
+ else:
139
+ _take(log.stdout, repo_name(common), cfg, tz, found.commits)
140
+ dirnames[:] = [] if depth >= _MAX_DEPTH else \
141
+ [n for n in dirnames if n not in (*_PRUNE, ".git") and
142
+ os.path.join(dirpath, n) not in cfg.git_days_exclude_dirs]
143
+
144
+
145
+ _DIR = "\0" # stands for the folder in a git argv; no real argument holds a NUL
146
+
147
+
148
+ def _sh(argv: list[str]) -> str:
149
+ return " ".join('"$d"' if a == _DIR else '"safe.directory=$d"'
150
+ if a == "safe.directory=" + _DIR else shlex.quote(a) for a in argv)
151
+
152
+
153
+ def _sh_root(root: str) -> str:
154
+ if root == "~" or root.startswith("~/"):
155
+ return '"$HOME"' + (("/" + shlex.quote(root[2:])) if root[2:] else "")
156
+ return shlex.quote(root)
157
+
158
+
159
+ def remote_script(roots: Iterable[str], exclude_dirs: Iterable[str] = ()) -> str:
160
+ """POSIX sh doing what _scan_local does: \\1<common dir> starts a store's log lines, \\2 marks a
161
+ missing root, a folder find could not read or a store git could not read, \\3 a git older than
162
+ 2.31 (no --path-format). A stale .git that holds no repository is skipped, as locally. Capture
163
+ rev-parse errors in memory so read-only scans need no writable remote scratch space.
164
+ Other stderr stays separate and is never printed (it names paths)."""
165
+ env = " ".join(f"{k}={shlex.quote(v)}" for k, v in READ_ENV.items())
166
+ prune = " -o ".join(f"-name {n}" for n in _PRUNE)
167
+ for path in exclude_dirs:
168
+ pattern = "".join("\\" + c if c in "\\*?[]" else c for c in path)
169
+ prune += " -o -path " + shlex.quote(pattern)
170
+ rev = _sh(_read_argv(_DIR, "rev-parse", "--path-format=absolute", "--git-common-dir"))
171
+ return f"""export {env} LC_ALL=C
172
+ set -- $(git version)
173
+ v=${{3:-0.0}}
174
+ case $v in [0-9]*.[0-9]*) ;; *) v=0.0 ;; esac
175
+ maj=${{v%%.*}}; min=${{v#*.}}; min=${{min%%.*}}
176
+ if [ "$maj" -lt 2 ] || {{ [ "$maj" -eq 2 ] && [ "$min" -lt 31 ]; }}; then printf '\\003\\n'; exit 0; fi
177
+ fail=$(printf '\\002')
178
+ seen='
179
+ '
180
+ for root in {" ".join(map(_sh_root, roots))}; do
181
+ if [ -d "$root" ]; then
182
+ root=$(CDPATH= cd "$root" && pwd -P) || {{ printf '\\002\\n'; continue; }}
183
+ find -H "$root" -maxdepth {_MAX_DEPTH + 1} \\( {prune} \\) -prune \
184
+ -o -name .git -prune -print || printf '\\002\\n'
185
+ else
186
+ printf '\\002\\n'
187
+ fi
188
+ done | while IFS= read -r g; do
189
+ if [ "$g" = "$fail" ]; then printf '\\002\\n'; continue; fi
190
+ d=${{g%/.git}}
191
+ if ! c=$({rev} 2>&1); then
192
+ case $c in *"not a git repository"*|*"invalid gitfile format"*) ;; \
193
+ *) printf '\\002\\n' ;; esac
194
+ continue
195
+ fi
196
+ case $c in /*) ;; *) printf '\\002\\n'; continue ;; esac
197
+ [ -d "$c" ] || {{ printf '\\002\\n'; continue; }}
198
+ case $seen in *"
199
+ $c
200
+ "*) continue ;; esac
201
+ seen="$seen$c
202
+ "
203
+ printf '\\001%s\\n' "$c"
204
+ if ! heads=$({_sh(_read_argv(_DIR, 'worktree', 'list', '--porcelain'))}); then
205
+ printf '\\002\\n'; continue
206
+ fi
207
+ heads=$(printf '%s\\n' "$heads" | sed -n '/^HEAD 0*$/d;s/^HEAD //p')
208
+ {_sh(_read_argv(_DIR, *LOG_ARGS))} $heads || printf '\\002\\n'
209
+ done
210
+ exit 0
211
+ """
212
+
213
+
214
+ def _ssh_argv(host: str) -> list[str]:
215
+ return ["ssh", "-o", "BatchMode=yes", "-o", "ClearAllForwardings=yes",
216
+ "-o", f"ConnectTimeout={SSH_CONNECT_S}", host,
217
+ "sh", "-s"]
218
+
219
+
220
+ def _scan_remote(i: int, host: str, roots: tuple[str, ...], cfg: Config, tz: tzinfo,
221
+ found: Found) -> None:
222
+ try:
223
+ r = subprocess.run( # nosec B603 # no shell here; the host is validated by config
224
+ _ssh_argv(host), input=remote_script(roots, cfg.git_days_exclude_dirs),
225
+ capture_output=True, text=True,
226
+ errors="replace", timeout=REMOTE_TIMEOUT_S)
227
+ code: int | None = r.returncode
228
+ except (OSError, subprocess.SubprocessError):
229
+ code = None
230
+ if code != 0:
231
+ found.failed += 1
232
+ print(f"aut: warning: git-days: remotes[{i}] could not be read "
233
+ f"({'no answer' if code is None else f'exit {code}'}); source coverage is incomplete",
234
+ file=sys.stderr)
235
+ return
236
+ if "\x03" in r.stdout.splitlines():
237
+ found.failed += 1
238
+ print(f"aut: warning: git-days: remotes[{i}] has git older than 2.31 (or none); "
239
+ "source coverage is incomplete", file=sys.stderr)
240
+ return
241
+ repo, log = "", ""
242
+ for line in r.stdout.splitlines() + ["\x01"]:
243
+ if line.startswith("\x01") or line == "\x02":
244
+ if repo:
245
+ _take(log, repo, cfg, tz, found.commits)
246
+ repo, log = "", ""
247
+ if line == "\x02":
248
+ found.failed += 1
249
+ elif line[1:]:
250
+ found.stores += 1
251
+ repo = repo_name(line[1:])
252
+ elif repo:
253
+ log += line + "\n"
254
+
255
+
256
+ def scan_all(cfg: Config, tz: tzinfo, *, defer_remotes: bool = False) -> Found:
257
+ found = Found()
258
+ if cfg.git_days_roots:
259
+ local = Found()
260
+ _scan_local(cfg, tz, local)
261
+ found.sources.append((json.dumps(['local', cfg.git_days_roots]), local, False))
262
+ for i, (host, roots) in enumerate(cfg.git_days_remotes):
263
+ remote = Found()
264
+ if not defer_remotes:
265
+ _scan_remote(i, host, roots, cfg, tz, remote)
266
+ found.sources.append((json.dumps(['remote', host, roots]), remote, defer_remotes))
267
+ for _, source, _ in found.sources:
268
+ found.stores += source.stores
269
+ found.failed += source.failed
270
+ found.commits.update(source.commits)
271
+ return found
272
+
273
+
274
+ def day_counts(commits: dict[str, date], since: date, until: date) -> dict[date, int]:
275
+ """Every day from since to until, zeros included."""
276
+ days = {since + timedelta(n): 0 for n in range((until - since).days + 1)}
277
+ for day in commits.values():
278
+ if day in days:
279
+ days[day] += 1
280
+ return days
281
+
282
+
283
+ def to_write(days: dict[date, int], stored: dict[date, int], *,
284
+ allow_lower: bool = False) -> tuple[list[tuple[date, int]], list[int]]:
285
+ """(rows: the days with no row yet or a higher count, the drop of each day that would go
286
+ down). allow_lower: every changed day is a row."""
287
+ rows, drops = [], []
288
+ for d, n in sorted(days.items()):
289
+ old = stored.get(d)
290
+ if old is not None and n < old and not allow_lower:
291
+ drops.append(old - n)
292
+ elif old != n:
293
+ rows.append((d, n))
294
+ return rows, drops
295
+
296
+
297
+ def refusal(found: Found, prev_stores: list[int] | None, *,
298
+ allow_lower: bool = False) -> str | None:
299
+ """Why this scan must not write, or None. prev_stores: the last good runs' store counts."""
300
+ if found.failed:
301
+ return (f"{found.failed} root(s), host(s), folder(s) or object store(s) could not be read "
302
+ "(a partial scan)")
303
+ if not found.stores:
304
+ return "the scan found no repositories"
305
+ best = max(prev_stores or [0])
306
+ if not allow_lower and found.stores < best * MIN_STORE_SHARE:
307
+ return (f"fewer object stores than half the best of the last good runs ({found.stores} < "
308
+ f"{best})")
309
+ return None
310
+
311
+
312
+ def coverage_rows(days: dict[date, int], complete: dict[date, bool],
313
+ stored: dict[date, tuple[int, bool]], *,
314
+ allow_lower: bool = False) -> tuple[list[tuple[date, int, bool]], list[int]]:
315
+ candidates = {d: n for d, n in days.items() if n or complete[d]}
316
+ changed, drops = to_write(candidates, {d: n for d, (n, _) in stored.items()},
317
+ allow_lower=allow_lower)
318
+ rows = [(d, n, complete[d]) for d, n in changed]
319
+ rows.extend((d, n, True) for d, n in candidates.items()
320
+ if complete[d] and stored.get(d) == (n, False))
321
+ return sorted(rows), drops
322
+
323
+
324
+ def load_stored(dsn: str) -> dict[date, tuple[int, bool]]:
325
+ with connect(dsn) as conn:
326
+ return {d: (n, complete) for d, n, complete in
327
+ conn.execute("SELECT day, commits, complete FROM agents.git_commits_daily").fetchall()}
328
+
329
+
330
+ def _ledger_zone(cfg: Config) -> tzinfo:
331
+ """The ledger's day, as in the sink's views."""
332
+ try:
333
+ return ZoneInfo(cfg.ledger_timezone)
334
+ except (ZoneInfoNotFoundError, ValueError):
335
+ raise AutError(f"git-days: ledger_timezone {cfg.ledger_timezone!r} is unknown here: check "
336
+ "the name, or install the time zone database (the system tzdata package, "
337
+ "or pip install tzdata)") from None
338
+
339
+
340
+ def run(cfg: Config, *, dry_run: bool, allow_lower: bool = False, now: datetime | None = None,
341
+ tz: tzinfo | None = None, local_only: bool = False,
342
+ defer_remotes_until: date | None = None) -> dict[str, Any]:
343
+ now = now or datetime.now(timezone.utc)
344
+ # the heartbeat comes first, so the watchdog also sees a run that fails before the scan
345
+ with job_heartbeat(cfg, NAME, datetime.now(timezone.utc), dry_run=dry_run) as beat:
346
+ if not cfg.git_days_identities:
347
+ raise AutError("git-days needs [git_days] identities in config.toml")
348
+ dsn = None if dry_run else cfg.require_pg("git-days")
349
+ if dsn:
350
+ check_ledger_zone(dsn, cfg.ledger_timezone)
351
+ tz = tz or _ledger_zone(cfg)
352
+ deferred = local_only or (defer_remotes_until is not None
353
+ and now.astimezone(tz).date() < defer_remotes_until)
354
+ cache_path = state_path().with_name('git-days-sources.json')
355
+ if not dry_run:
356
+ if cache_path.exists() and not RepoKeys(cfg.repo_key_salt, create=False).of('git-days'):
357
+ raise AutError('git-days: restore the salt belonging to the identity cache')
358
+ require_readable_salt(cfg.repo_key_salt, dsn, cfg.machine)
359
+ keys = RepoKeys(cfg.repo_key_salt, create=not dry_run)
360
+ if dry_run and not cache_path.exists() and keys.of('git-days') is None:
361
+ keys = RepoKeys('git-days temporary preview; never persisted')
362
+
363
+ def key(identity: str) -> str:
364
+ value = keys.of('git-days/v1/' + identity)
365
+ if value is None:
366
+ raise AutError('git-days: a readable repository salt is required')
367
+ return value
368
+
369
+ policy = json.dumps([cfg.git_days_identities, cfg.git_days_shared_authors,
370
+ cfg.git_days_exclude_dirs, cfg.git_days_since.isoformat(), str(tz)])
371
+ with nullcontext() if dry_run else _locked(cache_path, blocking=False):
372
+ return _collect(cfg, now, tz, dsn, beat, cache_path, key, policy,
373
+ deferred=deferred, allow_lower=allow_lower)
374
+
375
+
376
+ def _collect(cfg: Config, now: datetime, tz: tzinfo, dsn: str | None, beat: JobBeat,
377
+ cache_path: Path, key: Callable[[str], str], policy: str, *,
378
+ deferred: bool, allow_lower: bool) -> dict[str, Any]:
379
+ try:
380
+ cache = load_cache(cache_path, key('policy/' + policy), reset=allow_lower)
381
+ except ValueError as exc:
382
+ raise AutError(str(exc)) from exc
383
+ state = load_state()
384
+ wm = collector_wm(state, NAME)
385
+ kept = wm.get("stores")
386
+ prev = [n for n in kept if isinstance(n, int)] if isinstance(kept, list) else []
387
+ if dsn is not None and cfg.git_days_remotes and not deferred:
388
+ wm.update(last_attempt=now.isoformat(), remote_scan_version=_REMOTE_SCAN_VERSION)
389
+ save_state(state, [NAME])
390
+ found = scan_all(cfg, tz, defer_remotes=deferred)
391
+ snapshots: list[Snapshot] = []
392
+ for identity, source, skipped in found.sources:
393
+ sid = key('source/' + identity)
394
+ prior_source = cache['sources'].get(sid)
395
+ history = prior_source['stores'] if prior_source else []
396
+ why_source = refusal(source, history, allow_lower=allow_lower)
397
+ status: Status = 'deferred' if skipped else 'complete' if why_source is None else \
398
+ 'partial' if source.commits else 'failed'
399
+ snapshots.append(Snapshot(sid, status, {key('commit/' + sha): day
400
+ for sha, day in source.commits.items()}, source.stores))
401
+ partial = any(s.status != 'complete' for s in snapshots)
402
+ why = None if partial else refusal(found, prev, allow_lower=allow_lower)
403
+ if not found.stores and not cache['commits']:
404
+ why = 'the scan found no repositories'
405
+ if allow_lower and partial:
406
+ why = '--allow-lower needs every configured source freshly complete'
407
+ until = now.astimezone(tz).date() - timedelta(days=1)
408
+ if why:
409
+ if dsn is not None:
410
+ raise AutError(f'git-days: not writing: {why}')
411
+ return {'stores': found.stores, 'failed': found.failed,
412
+ 'days': day_counts(found.commits, cfg.git_days_since, until),
413
+ 'written': 0, 'refused': why, 'partial': partial, 'lowered': 0, 'lowered_by': 0}
414
+ updated, complete = merge_sources(cache, snapshots, cfg.git_days_since, until,
415
+ now, reset=allow_lower)
416
+ source_ids = sorted(s.source_id for s in snapshots)
417
+ if not partial:
418
+ updated['baseline_sources'] = source_ids
419
+ if prev and updated['baseline_sources'] != source_ids:
420
+ # Old aggregate history must pass once for these sources before partial reads prove zeros.
421
+ complete = dict.fromkeys(complete, False)
422
+ days = day_counts({sha: date.fromisoformat(day) for sha, day in updated['commits'].items()},
423
+ cfg.git_days_since, until)
424
+ report = {"stores": found.stores, "failed": found.failed, "days": days, "written": 0,
425
+ "refused": None, "partial": partial, "complete": complete, "lowered": 0, "lowered_by": 0}
426
+ if dsn is None:
427
+ return report
428
+ rows, drops = coverage_rows(days, complete, load_stored(dsn), allow_lower=allow_lower)
429
+ report["lowered"], report["lowered_by"] = len(drops), sum(drops)
430
+ write_many(dsn, INSERT_SQL, [(d, n, now, c) for d, n, c in rows])
431
+ atomic_write_text(cache_path, json.dumps(updated, sort_keys=True))
432
+ beat.events = report["written"] = len(rows)
433
+ wm.update(last_attempt=now.isoformat(), partial_pending=partial)
434
+ if partial:
435
+ beat.status, beat.error = 'degraded', 'GitSourcesPartial'
436
+ else:
437
+ wm.update(stores=([] if allow_lower else prev)[-(KEEP_STORES - 1):] + [found.stores],
438
+ last_ok=now.isoformat())
439
+ save_state(state, [NAME])
440
+ return report
441
+
442
+
443
+ def print_report(report: dict[str, Any], *, dry_run: bool) -> None:
444
+ head = "git-days DRY-RUN (nothing written)" if dry_run else \
445
+ f"git-days: wrote {report['written']} days"
446
+ print(f"{head}: {report['stores']} object stores read, {report['failed']} could not be read")
447
+ if report.get('partial'):
448
+ print(' source coverage is incomplete; known commits retained, missing days remain unknown')
449
+ if report.get("lowered"):
450
+ print(f" {report['lowered']} day(s) would go down ({report['lowered_by']:,} commits in "
451
+ "all); not written. `aut git-days --allow-lower` rewrites them")
452
+ if not dry_run:
453
+ return
454
+ years: dict[int, list[int]] = {}
455
+ for day, n in report["days"].items():
456
+ y = years.setdefault(day.year, [0, 0])
457
+ y[0] += n
458
+ y[1] += n > 0
459
+ for year, (n, k) in sorted(years.items()):
460
+ print(f" {year} {n:>7,} commits on {k} days")
461
+ if report["refused"]:
462
+ print(f" a real run would not write: {report['refused']}")
463
+
464
+
465
+ def _older(stamp: object, now: datetime, gap: timedelta) -> bool:
466
+ try:
467
+ return now - datetime.fromisoformat(str(stamp)) >= gap
468
+ except (TypeError, ValueError): # never ran, or an unusable stamp
469
+ return True
470
+
471
+
472
+ def run_if_due(cfg: Config, now: datetime | None = None, *,
473
+ defer_remotes_until: date | None = None) -> None:
474
+ """From `aut scan`: once the last good run is DUE_AFTER old and the last attempt RETRY_AFTER
475
+ old. A fixed remote reader retries older partial attempts once. Never fails the scan.
476
+ A manual `aut git-days` calls run() and always runs."""
477
+ if not cfg.git_days_enabled:
478
+ return
479
+ try:
480
+ now = now or datetime.now(timezone.utc)
481
+ state = load_state()
482
+ wm = collector_wm(state, NAME)
483
+ deferred = (defer_remotes_until is not None
484
+ and now.astimezone(_ledger_zone(cfg)).date() < defer_remotes_until)
485
+ new_remote_reader = (wm.get("partial_pending") is True and bool(cfg.git_days_remotes)
486
+ and not deferred
487
+ and wm.get("remote_scan_version") != _REMOTE_SCAN_VERSION)
488
+ if not ((wm.get('partial_pending') is True or _older(wm.get("last_ok"), now, DUE_AFTER))
489
+ and (new_remote_reader or _older(wm.get("last_attempt"), now, RETRY_AFTER))):
490
+ return
491
+ wm["last_attempt"] = now.isoformat()
492
+ if cfg.git_days_remotes and not deferred:
493
+ wm["remote_scan_version"] = _REMOTE_SCAN_VERSION
494
+ save_state(state, [NAME])
495
+ report = run(cfg, dry_run=False, defer_remotes_until=defer_remotes_until) \
496
+ if defer_remotes_until else run(cfg, dry_run=False)
497
+ print_report(report, dry_run=False)
498
+ except Exception as exc:
499
+ print(f"aut: warning: git-days did not run: {redacted_error(exc)}", file=sys.stderr)
@@ -0,0 +1,58 @@
1
+ """Every git command aut runs goes through here. aut runs git in clones it did not write (session
2
+ working dirs, everything under outcome_repo_roots), and a clone's own config, attributes and hooks
3
+ can name commands for git to run: diff.external, a textconv driver, core.fsmonitor,
4
+ core.sshCommand, core.askPass, remote.<name>.uploadpack, hooks (reference-transaction fires on
5
+ fetch), an ext:: remote, gpg.program (log.showSignature), core.gitProxy (git://),
6
+ core.alternateRefsCommand, a credential helper. These are switched off on the command line, which
7
+ outranks repo config. The system config is skipped; the user's global config (HOME) still applies,
8
+ so fetch keeps their ssh setup and gets their credential helpers back after the reset.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import os
14
+ import subprocess # nosec B404 # one fixed argv below, no shell
15
+
16
+ _CONFIG = ("core.fsmonitor=", "core.hooksPath=/dev/null", "protocol.ext.allow=never",
17
+ "diff.external=", "core.sshCommand=ssh", "core.askPass=", "gc.auto=0",
18
+ "maintenance.auto=false", "log.showSignature=false", "core.alternateRefsCommand=",
19
+ # core.gitProxy is first-match-wins, so the repo's value beats -c: refuse git:// itself,
20
+ # and with it every transport but these (a `foo::` remote helper too)
21
+ "protocol.allow=never", "protocol.https.allow=always", "protocol.ssh.allow=always",
22
+ "protocol.file.allow=always")
23
+ _NO_DRIVERS = ("--no-textconv", "--no-ext-diff")
24
+ # remote.<name>.uploadpack is multi-valued and the repo's value wins over -c: the flag beats both
25
+ _SUB_FLAGS = {"log": _NO_DRIVERS, "show": _NO_DRIVERS, "diff": _NO_DRIVERS,
26
+ "blame": ("--no-textconv",), "fetch": ("--upload-pack=git-upload-pack",)}
27
+
28
+
29
+ def git_argv(path: str, sub: str, *args: str) -> list[str]:
30
+ """argv for `git -C path <sub> <args>` with repo-defined commands switched off."""
31
+ cfg = (*_CONFIG, *_credential_config()) if sub == "fetch" else _CONFIG
32
+ return ["git", "-C", path, *(x for c in cfg for x in ("-c", c)), sub,
33
+ *_SUB_FLAGS.get(sub, ()), *args]
34
+
35
+
36
+ def _credential_config() -> tuple[str, ...]:
37
+ """An empty credential.helper clears every helper read so far (the repo's, url-scoped ones
38
+ too), then the user's global ones go back in their order."""
39
+ try:
40
+ r = subprocess.run( # nosec B603, B607 # fixed argv, no shell; git from the user's PATH
41
+ ["git", "config", "--global", "--includes", "-z", "--get-regexp",
42
+ r"^credential\..*helper$"], cwd=os.path.expanduser("~"),
43
+ capture_output=True, text=True, timeout=10, env=git_env())
44
+ out = r.stdout if r.returncode == 0 else ""
45
+ except (OSError, subprocess.SubprocessError):
46
+ out = ""
47
+ pairs = (e.split("\n", 1) for e in out.split("\0") if e)
48
+ return ("credential.helper=", *(f"{k}={v[0] if v else ''}" for k, *v in pairs))
49
+
50
+
51
+ # GIT_ALLOW_PROTOCOL replaces every protocol.*.allow setting, the repo's own included: a repo
52
+ # setting protocol.git.allow=always would otherwise beat the -c protocol.allow=never above.
53
+ HARDENED_ENV = {"GIT_CONFIG_NOSYSTEM": "1", "GIT_TERMINAL_PROMPT": "0",
54
+ "GIT_ALLOW_PROTOCOL": "https:ssh:file"}
55
+
56
+
57
+ def git_env() -> dict[str, str]:
58
+ return dict(os.environ, **HARDENED_ENV)