agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,499 @@
|
|
|
1
|
+
"""`aut git-days`: the owner's commits per ledger day (ledger_timezone), from every clone, to agents.git_commit_day.
|
|
2
|
+
|
|
3
|
+
Reads branches, remotes, tags and all registered worktree heads (no merges, stash or notes),
|
|
4
|
+
including detached heads outside scan roots, in each object store under [git_days] roots and,
|
|
5
|
+
over `ssh -o BatchMode=yes`, under each remote's roots, with the same hardened git (git_safe). No
|
|
6
|
+
fetch. A commit counts once (by SHA) when its author email is one of the owner's identities, or a
|
|
7
|
+
shared author's in a repo whose name matches that author's repo_regex; a bot never counts.
|
|
8
|
+
Only counts and completeness leave the Mac. Raw repo names, SHAs and emails stay in memory.
|
|
9
|
+
Salted commit identities and per-source coverage stay in a private local cache.
|
|
10
|
+
|
|
11
|
+
The table is insert-only and a day's latest scan wins, so a run inserts, with one scanned_at, the
|
|
12
|
+
days from `since` to yesterday that have no row yet or whose count went up. A stored day never goes
|
|
13
|
+
down by itself: a lower count is only counted in the summary, and `--allow-lower` rewrites it.
|
|
14
|
+
Partial scans retain known commits without inventing zero days. Each source keeps its last
|
|
15
|
+
complete coverage. Missing sources remain retry-eligible. `--allow-lower` requires a fresh
|
|
16
|
+
complete scan of every configured source and resets the private identity cache.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import re
|
|
24
|
+
import shlex
|
|
25
|
+
import subprocess # nosec B404 # argv lists only; the remote script goes to ssh on stdin
|
|
26
|
+
import sys
|
|
27
|
+
from dataclasses import dataclass, field
|
|
28
|
+
from contextlib import nullcontext
|
|
29
|
+
from datetime import date, datetime, timedelta, timezone, tzinfo
|
|
30
|
+
from pathlib import Path, PurePosixPath
|
|
31
|
+
from typing import Any, Callable, Iterable
|
|
32
|
+
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
|
33
|
+
|
|
34
|
+
from .config import AutError, Config
|
|
35
|
+
from .git_safe import HARDENED_ENV, git_argv
|
|
36
|
+
from .git_day_cache import Snapshot, Status, load_cache, merge_sources
|
|
37
|
+
from .push import JobBeat, job_heartbeat, redacted_error
|
|
38
|
+
from .sink import check_ledger_zone, connect, write_many
|
|
39
|
+
from .repo_keys import RepoKeys, require_readable_salt
|
|
40
|
+
from .state import _locked, atomic_write_text, collector_wm, load_state, save_state, state_path
|
|
41
|
+
|
|
42
|
+
NAME = "git_days" # its aut state entry and heartbeat collector
|
|
43
|
+
LOG_ARGS = ("log", "--branches", "--remotes", "--tags", "--no-merges", "--format=%H%x09%aI%x09%ae")
|
|
44
|
+
_PRUNE = ("node_modules", ".venv", "venv", "vendor", "__pycache__")
|
|
45
|
+
_MAX_DEPTH = 6 # folders below a root; its .git sits one deeper
|
|
46
|
+
_BOT = re.compile(r"\[bot\]|^noreply@anthropic\.com$")
|
|
47
|
+
MIN_STORE_SHARE = 0.5
|
|
48
|
+
KEEP_STORES = 5 # good runs whose best store count the share rule uses
|
|
49
|
+
DUE_AFTER = timedelta(hours=20)
|
|
50
|
+
RETRY_AFTER = timedelta(hours=6) # after an attempt that failed or was refused
|
|
51
|
+
_REMOTE_SCAN_VERSION = 1 # retry older partial remote reads once after this repair
|
|
52
|
+
SSH_CONNECT_S = 10
|
|
53
|
+
REMOTE_TIMEOUT_S = 900
|
|
54
|
+
INSERT_SQL = "INSERT INTO agents.git_commit_day (day, commits, scanned_at, complete) VALUES (%s, %s, %s, %s)"
|
|
55
|
+
READ_ENV = {**HARDENED_ENV, "GIT_NO_LAZY_FETCH": "1", "GIT_ALLOW_PROTOCOL": "", "LC_ALL": "C"}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _read_argv(path: str, sub: str, *args: str) -> list[str]:
|
|
59
|
+
# Configured physical roots authorize reads of shared clones, never wildcard or global trust.
|
|
60
|
+
argv = git_argv(path, sub, *args)
|
|
61
|
+
return [*argv[:3], "-c", "safe.directory=", "-c", f"safe.directory={path}", *argv[3:]]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _read_git(path: str, sub: str, *args: str) -> subprocess.CompletedProcess[str] | None:
|
|
65
|
+
try:
|
|
66
|
+
return subprocess.run(_read_argv(path, sub, *args), capture_output=True, text=True,
|
|
67
|
+
errors="replace", timeout=300, env={**os.environ, **READ_ENV})
|
|
68
|
+
except (OSError, subprocess.SubprocessError):
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(slots=True)
|
|
73
|
+
class Found:
|
|
74
|
+
stores: int = 0 # distinct object stores read
|
|
75
|
+
failed: int = 0 # hosts and stores that could not be read
|
|
76
|
+
commits: dict[str, date] = field(default_factory=dict) # counted SHA -> local day
|
|
77
|
+
sources: list[tuple[str, Found, bool]] = field(default_factory=list)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def repo_name(common: str) -> str:
|
|
81
|
+
"""The repo's folder name from its git common dir: /x/name/.git, /x/name.git, .git/modules/name."""
|
|
82
|
+
p = PurePosixPath(common.replace("\\", "/"))
|
|
83
|
+
return (p.parent.name if p.name == ".git" else p.name).removesuffix(".git")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _owned(cfg: Config, email: str, repo: str) -> bool:
|
|
87
|
+
if not email or _BOT.search(email):
|
|
88
|
+
return False
|
|
89
|
+
return email in cfg.git_days_identities or any(
|
|
90
|
+
re.search(e, email, re.IGNORECASE) and re.search(r, repo)
|
|
91
|
+
for e, r in cfg.git_days_shared_authors)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _take(log: str, repo: str, cfg: Config, tz: tzinfo, into: dict[str, date]) -> None:
|
|
95
|
+
for line in log.splitlines():
|
|
96
|
+
sha, _, rest = line.partition("\t")
|
|
97
|
+
when, _, email = rest.partition("\t")
|
|
98
|
+
if sha in into or not _owned(cfg, email.strip().lower(), repo):
|
|
99
|
+
continue
|
|
100
|
+
try:
|
|
101
|
+
into[sha] = datetime.fromisoformat(when).astimezone(tz).date()
|
|
102
|
+
except ValueError:
|
|
103
|
+
continue
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _scan_local(cfg: Config, tz: tzinfo, found: Found) -> None:
|
|
107
|
+
seen: set[str] = set()
|
|
108
|
+
|
|
109
|
+
def unreadable(_exc: OSError) -> None:
|
|
110
|
+
found.failed += 1
|
|
111
|
+
|
|
112
|
+
for root in cfg.git_days_roots:
|
|
113
|
+
base = os.path.realpath(os.path.expanduser(root))
|
|
114
|
+
if base in cfg.git_days_exclude_dirs:
|
|
115
|
+
continue
|
|
116
|
+
if not os.path.isdir(base): # missing or unmounted: its repos would read as zero
|
|
117
|
+
found.failed += 1
|
|
118
|
+
continue
|
|
119
|
+
for dirpath, dirnames, filenames in os.walk(base, onerror=unreadable):
|
|
120
|
+
depth = dirpath[len(base):].count(os.sep)
|
|
121
|
+
if ".git" in dirnames or ".git" in filenames:
|
|
122
|
+
rev = _read_git(dirpath, "rev-parse", "--path-format=absolute", "--git-common-dir")
|
|
123
|
+
common = rev.stdout.strip() if rev is not None and rev.returncode == 0 else \
|
|
124
|
+
"" if rev is not None and ("not a git repository" in rev.stderr or
|
|
125
|
+
"invalid gitfile format" in rev.stderr) else None
|
|
126
|
+
if common is None:
|
|
127
|
+
found.failed += 1
|
|
128
|
+
elif common and os.path.realpath(common) not in seen:
|
|
129
|
+
seen.add(os.path.realpath(common))
|
|
130
|
+
found.stores += 1
|
|
131
|
+
wt = _read_git(dirpath, "worktree", "list", "--porcelain")
|
|
132
|
+
worktrees = wt.stdout if wt is not None and wt.returncode == 0 else None
|
|
133
|
+
heads = [line[5:] for line in (worktrees or "").splitlines()
|
|
134
|
+
if line.startswith("HEAD ") and line[5:].strip("0")]
|
|
135
|
+
log = _read_git(dirpath, *LOG_ARGS, *heads) if worktrees is not None else None
|
|
136
|
+
if log is None or log.returncode:
|
|
137
|
+
found.failed += 1
|
|
138
|
+
else:
|
|
139
|
+
_take(log.stdout, repo_name(common), cfg, tz, found.commits)
|
|
140
|
+
dirnames[:] = [] if depth >= _MAX_DEPTH else \
|
|
141
|
+
[n for n in dirnames if n not in (*_PRUNE, ".git") and
|
|
142
|
+
os.path.join(dirpath, n) not in cfg.git_days_exclude_dirs]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
_DIR = "\0" # stands for the folder in a git argv; no real argument holds a NUL
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _sh(argv: list[str]) -> str:
|
|
149
|
+
return " ".join('"$d"' if a == _DIR else '"safe.directory=$d"'
|
|
150
|
+
if a == "safe.directory=" + _DIR else shlex.quote(a) for a in argv)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _sh_root(root: str) -> str:
|
|
154
|
+
if root == "~" or root.startswith("~/"):
|
|
155
|
+
return '"$HOME"' + (("/" + shlex.quote(root[2:])) if root[2:] else "")
|
|
156
|
+
return shlex.quote(root)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def remote_script(roots: Iterable[str], exclude_dirs: Iterable[str] = ()) -> str:
|
|
160
|
+
"""POSIX sh doing what _scan_local does: \\1<common dir> starts a store's log lines, \\2 marks a
|
|
161
|
+
missing root, a folder find could not read or a store git could not read, \\3 a git older than
|
|
162
|
+
2.31 (no --path-format). A stale .git that holds no repository is skipped, as locally. Capture
|
|
163
|
+
rev-parse errors in memory so read-only scans need no writable remote scratch space.
|
|
164
|
+
Other stderr stays separate and is never printed (it names paths)."""
|
|
165
|
+
env = " ".join(f"{k}={shlex.quote(v)}" for k, v in READ_ENV.items())
|
|
166
|
+
prune = " -o ".join(f"-name {n}" for n in _PRUNE)
|
|
167
|
+
for path in exclude_dirs:
|
|
168
|
+
pattern = "".join("\\" + c if c in "\\*?[]" else c for c in path)
|
|
169
|
+
prune += " -o -path " + shlex.quote(pattern)
|
|
170
|
+
rev = _sh(_read_argv(_DIR, "rev-parse", "--path-format=absolute", "--git-common-dir"))
|
|
171
|
+
return f"""export {env} LC_ALL=C
|
|
172
|
+
set -- $(git version)
|
|
173
|
+
v=${{3:-0.0}}
|
|
174
|
+
case $v in [0-9]*.[0-9]*) ;; *) v=0.0 ;; esac
|
|
175
|
+
maj=${{v%%.*}}; min=${{v#*.}}; min=${{min%%.*}}
|
|
176
|
+
if [ "$maj" -lt 2 ] || {{ [ "$maj" -eq 2 ] && [ "$min" -lt 31 ]; }}; then printf '\\003\\n'; exit 0; fi
|
|
177
|
+
fail=$(printf '\\002')
|
|
178
|
+
seen='
|
|
179
|
+
'
|
|
180
|
+
for root in {" ".join(map(_sh_root, roots))}; do
|
|
181
|
+
if [ -d "$root" ]; then
|
|
182
|
+
root=$(CDPATH= cd "$root" && pwd -P) || {{ printf '\\002\\n'; continue; }}
|
|
183
|
+
find -H "$root" -maxdepth {_MAX_DEPTH + 1} \\( {prune} \\) -prune \
|
|
184
|
+
-o -name .git -prune -print || printf '\\002\\n'
|
|
185
|
+
else
|
|
186
|
+
printf '\\002\\n'
|
|
187
|
+
fi
|
|
188
|
+
done | while IFS= read -r g; do
|
|
189
|
+
if [ "$g" = "$fail" ]; then printf '\\002\\n'; continue; fi
|
|
190
|
+
d=${{g%/.git}}
|
|
191
|
+
if ! c=$({rev} 2>&1); then
|
|
192
|
+
case $c in *"not a git repository"*|*"invalid gitfile format"*) ;; \
|
|
193
|
+
*) printf '\\002\\n' ;; esac
|
|
194
|
+
continue
|
|
195
|
+
fi
|
|
196
|
+
case $c in /*) ;; *) printf '\\002\\n'; continue ;; esac
|
|
197
|
+
[ -d "$c" ] || {{ printf '\\002\\n'; continue; }}
|
|
198
|
+
case $seen in *"
|
|
199
|
+
$c
|
|
200
|
+
"*) continue ;; esac
|
|
201
|
+
seen="$seen$c
|
|
202
|
+
"
|
|
203
|
+
printf '\\001%s\\n' "$c"
|
|
204
|
+
if ! heads=$({_sh(_read_argv(_DIR, 'worktree', 'list', '--porcelain'))}); then
|
|
205
|
+
printf '\\002\\n'; continue
|
|
206
|
+
fi
|
|
207
|
+
heads=$(printf '%s\\n' "$heads" | sed -n '/^HEAD 0*$/d;s/^HEAD //p')
|
|
208
|
+
{_sh(_read_argv(_DIR, *LOG_ARGS))} $heads || printf '\\002\\n'
|
|
209
|
+
done
|
|
210
|
+
exit 0
|
|
211
|
+
"""
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _ssh_argv(host: str) -> list[str]:
|
|
215
|
+
return ["ssh", "-o", "BatchMode=yes", "-o", "ClearAllForwardings=yes",
|
|
216
|
+
"-o", f"ConnectTimeout={SSH_CONNECT_S}", host,
|
|
217
|
+
"sh", "-s"]
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _scan_remote(i: int, host: str, roots: tuple[str, ...], cfg: Config, tz: tzinfo,
|
|
221
|
+
found: Found) -> None:
|
|
222
|
+
try:
|
|
223
|
+
r = subprocess.run( # nosec B603 # no shell here; the host is validated by config
|
|
224
|
+
_ssh_argv(host), input=remote_script(roots, cfg.git_days_exclude_dirs),
|
|
225
|
+
capture_output=True, text=True,
|
|
226
|
+
errors="replace", timeout=REMOTE_TIMEOUT_S)
|
|
227
|
+
code: int | None = r.returncode
|
|
228
|
+
except (OSError, subprocess.SubprocessError):
|
|
229
|
+
code = None
|
|
230
|
+
if code != 0:
|
|
231
|
+
found.failed += 1
|
|
232
|
+
print(f"aut: warning: git-days: remotes[{i}] could not be read "
|
|
233
|
+
f"({'no answer' if code is None else f'exit {code}'}); source coverage is incomplete",
|
|
234
|
+
file=sys.stderr)
|
|
235
|
+
return
|
|
236
|
+
if "\x03" in r.stdout.splitlines():
|
|
237
|
+
found.failed += 1
|
|
238
|
+
print(f"aut: warning: git-days: remotes[{i}] has git older than 2.31 (or none); "
|
|
239
|
+
"source coverage is incomplete", file=sys.stderr)
|
|
240
|
+
return
|
|
241
|
+
repo, log = "", ""
|
|
242
|
+
for line in r.stdout.splitlines() + ["\x01"]:
|
|
243
|
+
if line.startswith("\x01") or line == "\x02":
|
|
244
|
+
if repo:
|
|
245
|
+
_take(log, repo, cfg, tz, found.commits)
|
|
246
|
+
repo, log = "", ""
|
|
247
|
+
if line == "\x02":
|
|
248
|
+
found.failed += 1
|
|
249
|
+
elif line[1:]:
|
|
250
|
+
found.stores += 1
|
|
251
|
+
repo = repo_name(line[1:])
|
|
252
|
+
elif repo:
|
|
253
|
+
log += line + "\n"
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def scan_all(cfg: Config, tz: tzinfo, *, defer_remotes: bool = False) -> Found:
|
|
257
|
+
found = Found()
|
|
258
|
+
if cfg.git_days_roots:
|
|
259
|
+
local = Found()
|
|
260
|
+
_scan_local(cfg, tz, local)
|
|
261
|
+
found.sources.append((json.dumps(['local', cfg.git_days_roots]), local, False))
|
|
262
|
+
for i, (host, roots) in enumerate(cfg.git_days_remotes):
|
|
263
|
+
remote = Found()
|
|
264
|
+
if not defer_remotes:
|
|
265
|
+
_scan_remote(i, host, roots, cfg, tz, remote)
|
|
266
|
+
found.sources.append((json.dumps(['remote', host, roots]), remote, defer_remotes))
|
|
267
|
+
for _, source, _ in found.sources:
|
|
268
|
+
found.stores += source.stores
|
|
269
|
+
found.failed += source.failed
|
|
270
|
+
found.commits.update(source.commits)
|
|
271
|
+
return found
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def day_counts(commits: dict[str, date], since: date, until: date) -> dict[date, int]:
|
|
275
|
+
"""Every day from since to until, zeros included."""
|
|
276
|
+
days = {since + timedelta(n): 0 for n in range((until - since).days + 1)}
|
|
277
|
+
for day in commits.values():
|
|
278
|
+
if day in days:
|
|
279
|
+
days[day] += 1
|
|
280
|
+
return days
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def to_write(days: dict[date, int], stored: dict[date, int], *,
|
|
284
|
+
allow_lower: bool = False) -> tuple[list[tuple[date, int]], list[int]]:
|
|
285
|
+
"""(rows: the days with no row yet or a higher count, the drop of each day that would go
|
|
286
|
+
down). allow_lower: every changed day is a row."""
|
|
287
|
+
rows, drops = [], []
|
|
288
|
+
for d, n in sorted(days.items()):
|
|
289
|
+
old = stored.get(d)
|
|
290
|
+
if old is not None and n < old and not allow_lower:
|
|
291
|
+
drops.append(old - n)
|
|
292
|
+
elif old != n:
|
|
293
|
+
rows.append((d, n))
|
|
294
|
+
return rows, drops
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def refusal(found: Found, prev_stores: list[int] | None, *,
|
|
298
|
+
allow_lower: bool = False) -> str | None:
|
|
299
|
+
"""Why this scan must not write, or None. prev_stores: the last good runs' store counts."""
|
|
300
|
+
if found.failed:
|
|
301
|
+
return (f"{found.failed} root(s), host(s), folder(s) or object store(s) could not be read "
|
|
302
|
+
"(a partial scan)")
|
|
303
|
+
if not found.stores:
|
|
304
|
+
return "the scan found no repositories"
|
|
305
|
+
best = max(prev_stores or [0])
|
|
306
|
+
if not allow_lower and found.stores < best * MIN_STORE_SHARE:
|
|
307
|
+
return (f"fewer object stores than half the best of the last good runs ({found.stores} < "
|
|
308
|
+
f"{best})")
|
|
309
|
+
return None
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def coverage_rows(days: dict[date, int], complete: dict[date, bool],
|
|
313
|
+
stored: dict[date, tuple[int, bool]], *,
|
|
314
|
+
allow_lower: bool = False) -> tuple[list[tuple[date, int, bool]], list[int]]:
|
|
315
|
+
candidates = {d: n for d, n in days.items() if n or complete[d]}
|
|
316
|
+
changed, drops = to_write(candidates, {d: n for d, (n, _) in stored.items()},
|
|
317
|
+
allow_lower=allow_lower)
|
|
318
|
+
rows = [(d, n, complete[d]) for d, n in changed]
|
|
319
|
+
rows.extend((d, n, True) for d, n in candidates.items()
|
|
320
|
+
if complete[d] and stored.get(d) == (n, False))
|
|
321
|
+
return sorted(rows), drops
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def load_stored(dsn: str) -> dict[date, tuple[int, bool]]:
|
|
325
|
+
with connect(dsn) as conn:
|
|
326
|
+
return {d: (n, complete) for d, n, complete in
|
|
327
|
+
conn.execute("SELECT day, commits, complete FROM agents.git_commits_daily").fetchall()}
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _ledger_zone(cfg: Config) -> tzinfo:
|
|
331
|
+
"""The ledger's day, as in the sink's views."""
|
|
332
|
+
try:
|
|
333
|
+
return ZoneInfo(cfg.ledger_timezone)
|
|
334
|
+
except (ZoneInfoNotFoundError, ValueError):
|
|
335
|
+
raise AutError(f"git-days: ledger_timezone {cfg.ledger_timezone!r} is unknown here: check "
|
|
336
|
+
"the name, or install the time zone database (the system tzdata package, "
|
|
337
|
+
"or pip install tzdata)") from None
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def run(cfg: Config, *, dry_run: bool, allow_lower: bool = False, now: datetime | None = None,
|
|
341
|
+
tz: tzinfo | None = None, local_only: bool = False,
|
|
342
|
+
defer_remotes_until: date | None = None) -> dict[str, Any]:
|
|
343
|
+
now = now or datetime.now(timezone.utc)
|
|
344
|
+
# the heartbeat comes first, so the watchdog also sees a run that fails before the scan
|
|
345
|
+
with job_heartbeat(cfg, NAME, datetime.now(timezone.utc), dry_run=dry_run) as beat:
|
|
346
|
+
if not cfg.git_days_identities:
|
|
347
|
+
raise AutError("git-days needs [git_days] identities in config.toml")
|
|
348
|
+
dsn = None if dry_run else cfg.require_pg("git-days")
|
|
349
|
+
if dsn:
|
|
350
|
+
check_ledger_zone(dsn, cfg.ledger_timezone)
|
|
351
|
+
tz = tz or _ledger_zone(cfg)
|
|
352
|
+
deferred = local_only or (defer_remotes_until is not None
|
|
353
|
+
and now.astimezone(tz).date() < defer_remotes_until)
|
|
354
|
+
cache_path = state_path().with_name('git-days-sources.json')
|
|
355
|
+
if not dry_run:
|
|
356
|
+
if cache_path.exists() and not RepoKeys(cfg.repo_key_salt, create=False).of('git-days'):
|
|
357
|
+
raise AutError('git-days: restore the salt belonging to the identity cache')
|
|
358
|
+
require_readable_salt(cfg.repo_key_salt, dsn, cfg.machine)
|
|
359
|
+
keys = RepoKeys(cfg.repo_key_salt, create=not dry_run)
|
|
360
|
+
if dry_run and not cache_path.exists() and keys.of('git-days') is None:
|
|
361
|
+
keys = RepoKeys('git-days temporary preview; never persisted')
|
|
362
|
+
|
|
363
|
+
def key(identity: str) -> str:
|
|
364
|
+
value = keys.of('git-days/v1/' + identity)
|
|
365
|
+
if value is None:
|
|
366
|
+
raise AutError('git-days: a readable repository salt is required')
|
|
367
|
+
return value
|
|
368
|
+
|
|
369
|
+
policy = json.dumps([cfg.git_days_identities, cfg.git_days_shared_authors,
|
|
370
|
+
cfg.git_days_exclude_dirs, cfg.git_days_since.isoformat(), str(tz)])
|
|
371
|
+
with nullcontext() if dry_run else _locked(cache_path, blocking=False):
|
|
372
|
+
return _collect(cfg, now, tz, dsn, beat, cache_path, key, policy,
|
|
373
|
+
deferred=deferred, allow_lower=allow_lower)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _collect(cfg: Config, now: datetime, tz: tzinfo, dsn: str | None, beat: JobBeat,
|
|
377
|
+
cache_path: Path, key: Callable[[str], str], policy: str, *,
|
|
378
|
+
deferred: bool, allow_lower: bool) -> dict[str, Any]:
|
|
379
|
+
try:
|
|
380
|
+
cache = load_cache(cache_path, key('policy/' + policy), reset=allow_lower)
|
|
381
|
+
except ValueError as exc:
|
|
382
|
+
raise AutError(str(exc)) from exc
|
|
383
|
+
state = load_state()
|
|
384
|
+
wm = collector_wm(state, NAME)
|
|
385
|
+
kept = wm.get("stores")
|
|
386
|
+
prev = [n for n in kept if isinstance(n, int)] if isinstance(kept, list) else []
|
|
387
|
+
if dsn is not None and cfg.git_days_remotes and not deferred:
|
|
388
|
+
wm.update(last_attempt=now.isoformat(), remote_scan_version=_REMOTE_SCAN_VERSION)
|
|
389
|
+
save_state(state, [NAME])
|
|
390
|
+
found = scan_all(cfg, tz, defer_remotes=deferred)
|
|
391
|
+
snapshots: list[Snapshot] = []
|
|
392
|
+
for identity, source, skipped in found.sources:
|
|
393
|
+
sid = key('source/' + identity)
|
|
394
|
+
prior_source = cache['sources'].get(sid)
|
|
395
|
+
history = prior_source['stores'] if prior_source else []
|
|
396
|
+
why_source = refusal(source, history, allow_lower=allow_lower)
|
|
397
|
+
status: Status = 'deferred' if skipped else 'complete' if why_source is None else \
|
|
398
|
+
'partial' if source.commits else 'failed'
|
|
399
|
+
snapshots.append(Snapshot(sid, status, {key('commit/' + sha): day
|
|
400
|
+
for sha, day in source.commits.items()}, source.stores))
|
|
401
|
+
partial = any(s.status != 'complete' for s in snapshots)
|
|
402
|
+
why = None if partial else refusal(found, prev, allow_lower=allow_lower)
|
|
403
|
+
if not found.stores and not cache['commits']:
|
|
404
|
+
why = 'the scan found no repositories'
|
|
405
|
+
if allow_lower and partial:
|
|
406
|
+
why = '--allow-lower needs every configured source freshly complete'
|
|
407
|
+
until = now.astimezone(tz).date() - timedelta(days=1)
|
|
408
|
+
if why:
|
|
409
|
+
if dsn is not None:
|
|
410
|
+
raise AutError(f'git-days: not writing: {why}')
|
|
411
|
+
return {'stores': found.stores, 'failed': found.failed,
|
|
412
|
+
'days': day_counts(found.commits, cfg.git_days_since, until),
|
|
413
|
+
'written': 0, 'refused': why, 'partial': partial, 'lowered': 0, 'lowered_by': 0}
|
|
414
|
+
updated, complete = merge_sources(cache, snapshots, cfg.git_days_since, until,
|
|
415
|
+
now, reset=allow_lower)
|
|
416
|
+
source_ids = sorted(s.source_id for s in snapshots)
|
|
417
|
+
if not partial:
|
|
418
|
+
updated['baseline_sources'] = source_ids
|
|
419
|
+
if prev and updated['baseline_sources'] != source_ids:
|
|
420
|
+
# Old aggregate history must pass once for these sources before partial reads prove zeros.
|
|
421
|
+
complete = dict.fromkeys(complete, False)
|
|
422
|
+
days = day_counts({sha: date.fromisoformat(day) for sha, day in updated['commits'].items()},
|
|
423
|
+
cfg.git_days_since, until)
|
|
424
|
+
report = {"stores": found.stores, "failed": found.failed, "days": days, "written": 0,
|
|
425
|
+
"refused": None, "partial": partial, "complete": complete, "lowered": 0, "lowered_by": 0}
|
|
426
|
+
if dsn is None:
|
|
427
|
+
return report
|
|
428
|
+
rows, drops = coverage_rows(days, complete, load_stored(dsn), allow_lower=allow_lower)
|
|
429
|
+
report["lowered"], report["lowered_by"] = len(drops), sum(drops)
|
|
430
|
+
write_many(dsn, INSERT_SQL, [(d, n, now, c) for d, n, c in rows])
|
|
431
|
+
atomic_write_text(cache_path, json.dumps(updated, sort_keys=True))
|
|
432
|
+
beat.events = report["written"] = len(rows)
|
|
433
|
+
wm.update(last_attempt=now.isoformat(), partial_pending=partial)
|
|
434
|
+
if partial:
|
|
435
|
+
beat.status, beat.error = 'degraded', 'GitSourcesPartial'
|
|
436
|
+
else:
|
|
437
|
+
wm.update(stores=([] if allow_lower else prev)[-(KEEP_STORES - 1):] + [found.stores],
|
|
438
|
+
last_ok=now.isoformat())
|
|
439
|
+
save_state(state, [NAME])
|
|
440
|
+
return report
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def print_report(report: dict[str, Any], *, dry_run: bool) -> None:
|
|
444
|
+
head = "git-days DRY-RUN (nothing written)" if dry_run else \
|
|
445
|
+
f"git-days: wrote {report['written']} days"
|
|
446
|
+
print(f"{head}: {report['stores']} object stores read, {report['failed']} could not be read")
|
|
447
|
+
if report.get('partial'):
|
|
448
|
+
print(' source coverage is incomplete; known commits retained, missing days remain unknown')
|
|
449
|
+
if report.get("lowered"):
|
|
450
|
+
print(f" {report['lowered']} day(s) would go down ({report['lowered_by']:,} commits in "
|
|
451
|
+
"all); not written. `aut git-days --allow-lower` rewrites them")
|
|
452
|
+
if not dry_run:
|
|
453
|
+
return
|
|
454
|
+
years: dict[int, list[int]] = {}
|
|
455
|
+
for day, n in report["days"].items():
|
|
456
|
+
y = years.setdefault(day.year, [0, 0])
|
|
457
|
+
y[0] += n
|
|
458
|
+
y[1] += n > 0
|
|
459
|
+
for year, (n, k) in sorted(years.items()):
|
|
460
|
+
print(f" {year} {n:>7,} commits on {k} days")
|
|
461
|
+
if report["refused"]:
|
|
462
|
+
print(f" a real run would not write: {report['refused']}")
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _older(stamp: object, now: datetime, gap: timedelta) -> bool:
|
|
466
|
+
try:
|
|
467
|
+
return now - datetime.fromisoformat(str(stamp)) >= gap
|
|
468
|
+
except (TypeError, ValueError): # never ran, or an unusable stamp
|
|
469
|
+
return True
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def run_if_due(cfg: Config, now: datetime | None = None, *,
|
|
473
|
+
defer_remotes_until: date | None = None) -> None:
|
|
474
|
+
"""From `aut scan`: once the last good run is DUE_AFTER old and the last attempt RETRY_AFTER
|
|
475
|
+
old. A fixed remote reader retries older partial attempts once. Never fails the scan.
|
|
476
|
+
A manual `aut git-days` calls run() and always runs."""
|
|
477
|
+
if not cfg.git_days_enabled:
|
|
478
|
+
return
|
|
479
|
+
try:
|
|
480
|
+
now = now or datetime.now(timezone.utc)
|
|
481
|
+
state = load_state()
|
|
482
|
+
wm = collector_wm(state, NAME)
|
|
483
|
+
deferred = (defer_remotes_until is not None
|
|
484
|
+
and now.astimezone(_ledger_zone(cfg)).date() < defer_remotes_until)
|
|
485
|
+
new_remote_reader = (wm.get("partial_pending") is True and bool(cfg.git_days_remotes)
|
|
486
|
+
and not deferred
|
|
487
|
+
and wm.get("remote_scan_version") != _REMOTE_SCAN_VERSION)
|
|
488
|
+
if not ((wm.get('partial_pending') is True or _older(wm.get("last_ok"), now, DUE_AFTER))
|
|
489
|
+
and (new_remote_reader or _older(wm.get("last_attempt"), now, RETRY_AFTER))):
|
|
490
|
+
return
|
|
491
|
+
wm["last_attempt"] = now.isoformat()
|
|
492
|
+
if cfg.git_days_remotes and not deferred:
|
|
493
|
+
wm["remote_scan_version"] = _REMOTE_SCAN_VERSION
|
|
494
|
+
save_state(state, [NAME])
|
|
495
|
+
report = run(cfg, dry_run=False, defer_remotes_until=defer_remotes_until) \
|
|
496
|
+
if defer_remotes_until else run(cfg, dry_run=False)
|
|
497
|
+
print_report(report, dry_run=False)
|
|
498
|
+
except Exception as exc:
|
|
499
|
+
print(f"aut: warning: git-days did not run: {redacted_error(exc)}", file=sys.stderr)
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Every git command aut runs goes through here. aut runs git in clones it did not write (session
|
|
2
|
+
working dirs, everything under outcome_repo_roots), and a clone's own config, attributes and hooks
|
|
3
|
+
can name commands for git to run: diff.external, a textconv driver, core.fsmonitor,
|
|
4
|
+
core.sshCommand, core.askPass, remote.<name>.uploadpack, hooks (reference-transaction fires on
|
|
5
|
+
fetch), an ext:: remote, gpg.program (log.showSignature), core.gitProxy (git://),
|
|
6
|
+
core.alternateRefsCommand, a credential helper. These are switched off on the command line, which
|
|
7
|
+
outranks repo config. The system config is skipped; the user's global config (HOME) still applies,
|
|
8
|
+
so fetch keeps their ssh setup and gets their credential helpers back after the reset.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
import subprocess # nosec B404 # one fixed argv below, no shell
|
|
15
|
+
|
|
16
|
+
_CONFIG = ("core.fsmonitor=", "core.hooksPath=/dev/null", "protocol.ext.allow=never",
|
|
17
|
+
"diff.external=", "core.sshCommand=ssh", "core.askPass=", "gc.auto=0",
|
|
18
|
+
"maintenance.auto=false", "log.showSignature=false", "core.alternateRefsCommand=",
|
|
19
|
+
# core.gitProxy is first-match-wins, so the repo's value beats -c: refuse git:// itself,
|
|
20
|
+
# and with it every transport but these (a `foo::` remote helper too)
|
|
21
|
+
"protocol.allow=never", "protocol.https.allow=always", "protocol.ssh.allow=always",
|
|
22
|
+
"protocol.file.allow=always")
|
|
23
|
+
_NO_DRIVERS = ("--no-textconv", "--no-ext-diff")
|
|
24
|
+
# remote.<name>.uploadpack is multi-valued and the repo's value wins over -c: the flag beats both
|
|
25
|
+
_SUB_FLAGS = {"log": _NO_DRIVERS, "show": _NO_DRIVERS, "diff": _NO_DRIVERS,
|
|
26
|
+
"blame": ("--no-textconv",), "fetch": ("--upload-pack=git-upload-pack",)}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def git_argv(path: str, sub: str, *args: str) -> list[str]:
|
|
30
|
+
"""argv for `git -C path <sub> <args>` with repo-defined commands switched off."""
|
|
31
|
+
cfg = (*_CONFIG, *_credential_config()) if sub == "fetch" else _CONFIG
|
|
32
|
+
return ["git", "-C", path, *(x for c in cfg for x in ("-c", c)), sub,
|
|
33
|
+
*_SUB_FLAGS.get(sub, ()), *args]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _credential_config() -> tuple[str, ...]:
|
|
37
|
+
"""An empty credential.helper clears every helper read so far (the repo's, url-scoped ones
|
|
38
|
+
too), then the user's global ones go back in their order."""
|
|
39
|
+
try:
|
|
40
|
+
r = subprocess.run( # nosec B603, B607 # fixed argv, no shell; git from the user's PATH
|
|
41
|
+
["git", "config", "--global", "--includes", "-z", "--get-regexp",
|
|
42
|
+
r"^credential\..*helper$"], cwd=os.path.expanduser("~"),
|
|
43
|
+
capture_output=True, text=True, timeout=10, env=git_env())
|
|
44
|
+
out = r.stdout if r.returncode == 0 else ""
|
|
45
|
+
except (OSError, subprocess.SubprocessError):
|
|
46
|
+
out = ""
|
|
47
|
+
pairs = (e.split("\n", 1) for e in out.split("\0") if e)
|
|
48
|
+
return ("credential.helper=", *(f"{k}={v[0] if v else ''}" for k, *v in pairs))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# GIT_ALLOW_PROTOCOL replaces every protocol.*.allow setting, the repo's own included: a repo
|
|
52
|
+
# setting protocol.git.allow=always would otherwise beat the -c protocol.allow=never above.
|
|
53
|
+
HARDENED_ENV = {"GIT_CONFIG_NOSYSTEM": "1", "GIT_TERMINAL_PROMPT": "0",
|
|
54
|
+
"GIT_ALLOW_PROTOCOL": "https:ssh:file"}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def git_env() -> dict[str, str]:
|
|
58
|
+
return dict(os.environ, **HARDENED_ENV)
|