consequence-ledger 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- consequence_ledger-0.3.0.dist-info/METADATA +191 -0
- consequence_ledger-0.3.0.dist-info/RECORD +24 -0
- consequence_ledger-0.3.0.dist-info/WHEEL +5 -0
- consequence_ledger-0.3.0.dist-info/entry_points.txt +2 -0
- consequence_ledger-0.3.0.dist-info/licenses/LICENSE +21 -0
- consequence_ledger-0.3.0.dist-info/top_level.txt +1 -0
- ledger/__init__.py +2 -0
- ledger/__main__.py +3 -0
- ledger/audit.py +187 -0
- ledger/cli.py +637 -0
- ledger/db.py +70 -0
- ledger/demo.py +306 -0
- ledger/estimate.py +412 -0
- ledger/gitio.py +102 -0
- ledger/incidents.py +202 -0
- ledger/label.py +386 -0
- ledger/pagerduty.py +102 -0
- ledger/plan.py +62 -0
- ledger/policy.py +52 -0
- ledger/report.py +330 -0
- ledger/segments.py +142 -0
- ledger/sentry.py +91 -0
- ledger/squash.py +92 -0
- ledger/status.py +56 -0
ledger/cli.py
ADDED
|
@@ -0,0 +1,637 @@
|
|
|
1
|
+
"""Command-line interface: python -m ledger <command>."""
|
|
2
|
+
import argparse
|
|
3
|
+
import datetime as dt
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
import tempfile
|
|
7
|
+
from contextlib import closing
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from . import __version__
|
|
11
|
+
from . import report as report_mod
|
|
12
|
+
from .db import connect, db_path
|
|
13
|
+
from .estimate import DEFAULT_FOLDS, MIN_CROSSFIT, early_readout, estimate, load_rows, merge_balance
|
|
14
|
+
from .gitio import GitError, branch_key, current_branch, git, repo_root, safe_rev
|
|
15
|
+
from .label import DEFAULT_MAX_BLAMED, label
|
|
16
|
+
from .policy import record_decision
|
|
17
|
+
from .segments import breakdown, recommend
|
|
18
|
+
|
|
19
|
+
DETACHED = "(detached HEAD)"
|
|
20
|
+
HOOK_MARKER = "consequence-ledger hook v2"
|
|
21
|
+
HOOK = f"""#!/bin/sh
|
|
22
|
+
# {HOOK_MARKER}: tag commits with the decision that is active on the current branch.
|
|
23
|
+
# Finds the shared ledger from any worktree; does nothing where no decision is active.
|
|
24
|
+
[ "$2" = "merge" ] && exit 0
|
|
25
|
+
common="$(git rev-parse --git-common-dir 2>/dev/null)" || exit 0
|
|
26
|
+
root="$(cd "$common" 2>/dev/null && pwd)" || exit 0
|
|
27
|
+
branch="$(git symbolic-ref --short -q HEAD)" || branch="(detached HEAD)"
|
|
28
|
+
key="$(printf '%s' "$branch" | git hash-object --stdin)"
|
|
29
|
+
f="$(dirname "$root")/.ledger/active/$key"
|
|
30
|
+
[ -f "$f" ] || exit 0
|
|
31
|
+
id="$(head -n 1 "$f")"
|
|
32
|
+
grep -q "^Ledger-Decision-Id:" "$1" && exit 0
|
|
33
|
+
git interpret-trailers --in-place --trailer "Ledger-Decision-Id: $id" "$1"
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class CliError(Exception):
|
|
38
|
+
"""A problem the user can fix; shown as one line without a traceback."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def parse_date(value):
|
|
42
|
+
if value is None:
|
|
43
|
+
return None
|
|
44
|
+
if value.isdigit():
|
|
45
|
+
return int(value)
|
|
46
|
+
try:
|
|
47
|
+
d = dt.datetime.fromisoformat(value)
|
|
48
|
+
except ValueError:
|
|
49
|
+
raise CliError(f"could not read the date {value!r}; use 2026-06-30, 2026-06-30T12:00:00Z or unix seconds") from None
|
|
50
|
+
if d.tzinfo is None:
|
|
51
|
+
d = d.replace(tzinfo=dt.timezone.utc)
|
|
52
|
+
return int(d.timestamp())
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def ledger_dir(repo):
|
|
56
|
+
return Path(repo) / ".ledger"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def need_ledger(repo):
|
|
60
|
+
"""Commands that read or write decisions refuse to run, and litter, where no ledger exists."""
|
|
61
|
+
if not db_path(repo).exists():
|
|
62
|
+
raise CliError(f"no ledger in {Path(repo).resolve()}: run `ledger init` in the repository first")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def friendly_git_error(e, repo):
|
|
66
|
+
text = str(e)
|
|
67
|
+
if "not a git repository" in text:
|
|
68
|
+
return f"{Path(repo).resolve()} is not a git repository"
|
|
69
|
+
if any(s in text for s in ("unknown revision", "ambiguous argument", "bad revision", "does not have any commits")):
|
|
70
|
+
return "this repository has no commits yet, or the revision you gave does not exist"
|
|
71
|
+
return text
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def write_json(path, data):
|
|
75
|
+
Path(path).write_text(json.dumps(data, indent=2, default=str))
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def active_dir(repo):
|
|
79
|
+
return ledger_dir(repo) / "active"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def workdir(args):
|
|
83
|
+
"""The folder the user ran the command from; its checked-out branch decides which decision applies."""
|
|
84
|
+
return getattr(args, "workdir", args.repo)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def cmd_init(args):
|
|
88
|
+
git(args.repo, "rev-parse", "--git-dir") # a clear error if this is not a repository
|
|
89
|
+
repo = Path(args.repo)
|
|
90
|
+
connect(db_path(repo)).close()
|
|
91
|
+
legacy = ledger_dir(repo) / "current_decision" # v1 kept one decision per repository
|
|
92
|
+
legacy.unlink(missing_ok=True)
|
|
93
|
+
hooks = Path(git(repo, "rev-parse", "--git-path", "hooks").strip())
|
|
94
|
+
hooks = hooks if hooks.is_absolute() else repo / hooks
|
|
95
|
+
hooks.mkdir(parents=True, exist_ok=True)
|
|
96
|
+
hook = hooks / "prepare-commit-msg"
|
|
97
|
+
existing = hook.read_text(encoding="utf-8", errors="replace") if hook.exists() else ""
|
|
98
|
+
if existing and "consequence-ledger" not in existing:
|
|
99
|
+
print(f"warning: {hook} exists; add the Ledger-Decision-Id trailer step yourself", file=sys.stderr)
|
|
100
|
+
else:
|
|
101
|
+
# Always LF: git's sh can't run a script with CRLF line endings. (Path.write_text(newline=) needs 3.10.)
|
|
102
|
+
with open(hook, "w", encoding="utf-8", newline="\n") as f:
|
|
103
|
+
f.write(HOOK)
|
|
104
|
+
hook.chmod(0o755)
|
|
105
|
+
if existing and HOOK_MARKER not in existing:
|
|
106
|
+
print("upgraded the commit hook: decisions are now tracked per branch")
|
|
107
|
+
print(f"ledger ready in {ledger_dir(repo)}")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def cmd_decide(args):
|
|
111
|
+
need_ledger(args.repo)
|
|
112
|
+
arms = [a.strip() for a in args.arms.split(",") if a.strip()]
|
|
113
|
+
if len(arms) < 2 or len(set(arms)) != len(arms):
|
|
114
|
+
raise CliError("--arms needs at least two different names, e.g. --arms claude-code,codex")
|
|
115
|
+
try:
|
|
116
|
+
scores = [float(s) for s in args.scores.split(",")] if args.scores else None
|
|
117
|
+
except ValueError:
|
|
118
|
+
raise CliError("--scores must be numbers separated by commas, one per arm, e.g. --scores 1.0,0.5") from None
|
|
119
|
+
if scores is not None and len(scores) != len(arms):
|
|
120
|
+
raise CliError(f"--scores has {len(scores)} values but --arms has {len(arms)}; give one score per arm")
|
|
121
|
+
try:
|
|
122
|
+
context = json.loads(args.context)
|
|
123
|
+
except ValueError:
|
|
124
|
+
raise CliError("""--context must be JSON, e.g. --context '{"stratum": "hard"}'""") from None
|
|
125
|
+
if not isinstance(context, dict):
|
|
126
|
+
raise CliError("--context must be a JSON object such as {\"stratum\": \"hard\"}")
|
|
127
|
+
branch = current_branch(workdir(args))
|
|
128
|
+
with closing(connect(db_path(args.repo))) as con:
|
|
129
|
+
d = record_decision(con, arms, scores, context, args.epsilon, args.temperature, args.policy_version)
|
|
130
|
+
con.commit()
|
|
131
|
+
# One active decision per branch, so parallel agents on parallel branches or worktrees never share one.
|
|
132
|
+
active = active_dir(args.repo)
|
|
133
|
+
active.mkdir(parents=True, exist_ok=True)
|
|
134
|
+
with open(active / branch_key(branch), "w", encoding="utf-8", newline="\n") as f:
|
|
135
|
+
f.write(f"{d['id']}\n{branch or DETACHED}\n")
|
|
136
|
+
print(json.dumps({**d, "branch": branch or DETACHED}, indent=2))
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def cmd_done(args):
|
|
140
|
+
active = active_dir(args.repo)
|
|
141
|
+
if args.all:
|
|
142
|
+
files = list(active.glob("*")) if active.exists() else []
|
|
143
|
+
for f in files:
|
|
144
|
+
f.unlink()
|
|
145
|
+
print(f"cleared {len(files)} active decision(s); new commits will not be tagged")
|
|
146
|
+
return
|
|
147
|
+
branch = current_branch(workdir(args))
|
|
148
|
+
target = active / branch_key(branch)
|
|
149
|
+
existed = target.exists()
|
|
150
|
+
target.unlink(missing_ok=True)
|
|
151
|
+
name = branch or DETACHED
|
|
152
|
+
print(f"no active decision on {name}; new commits there will not be tagged" if existed
|
|
153
|
+
else f"{name} had no active decision")
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def cmd_attach(args):
|
|
157
|
+
need_ledger(args.repo)
|
|
158
|
+
try:
|
|
159
|
+
sha = git(args.repo, "rev-parse", "--verify", safe_rev(args.commit) + "^{commit}").strip()
|
|
160
|
+
except GitError:
|
|
161
|
+
raise CliError(f"no commit matches {args.commit!r} in this repository") from None
|
|
162
|
+
ts = int(git(args.repo, "show", "-s", "--format=%ct", sha).strip())
|
|
163
|
+
with closing(connect(db_path(args.repo))) as con:
|
|
164
|
+
if not con.execute("SELECT 1 FROM decisions WHERE id=?", (args.decision,)).fetchone():
|
|
165
|
+
raise CliError(f"unknown decision {args.decision!r}; `ledger status` and .ledger/ledger.db hold the ids")
|
|
166
|
+
con.execute("INSERT OR REPLACE INTO commits VALUES (?,?,?,'manual')", (sha, args.decision, ts))
|
|
167
|
+
con.commit()
|
|
168
|
+
print(f"{sha[:12]} -> {args.decision}")
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def fix_pattern(style, regex=None):
|
|
172
|
+
import re
|
|
173
|
+
from .label import FIX_STYLES
|
|
174
|
+
if not regex:
|
|
175
|
+
return FIX_STYLES[style]
|
|
176
|
+
try:
|
|
177
|
+
return re.compile(regex, re.I)
|
|
178
|
+
except re.error as e:
|
|
179
|
+
raise CliError(f"--fix-regex is not a valid regular expression: {e}") from None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def cmd_import(args):
|
|
183
|
+
from .incidents import import_deploys, import_incidents
|
|
184
|
+
need_ledger(args.repo)
|
|
185
|
+
if not Path(args.file).is_file():
|
|
186
|
+
raise CliError(f"file not found: {args.file}")
|
|
187
|
+
with closing(connect(db_path(args.repo))) as con:
|
|
188
|
+
if args.kind == "incidents":
|
|
189
|
+
n = import_incidents(con, args.file)
|
|
190
|
+
else:
|
|
191
|
+
n = import_deploys(con, args.file)
|
|
192
|
+
print(f"imported {n} {args.kind}; run `ledger label` to link them to decisions")
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def cmd_sync(args):
|
|
196
|
+
from .pagerduty import PagerDutyError
|
|
197
|
+
from .pagerduty import sync as pd_sync
|
|
198
|
+
from .sentry import SentryError
|
|
199
|
+
from .sentry import sync as sentry_sync
|
|
200
|
+
need_ledger(args.repo)
|
|
201
|
+
try:
|
|
202
|
+
with closing(connect(db_path(args.repo))) as con:
|
|
203
|
+
if args.source == "sentry":
|
|
204
|
+
if not args.org:
|
|
205
|
+
raise CliError("--org is required for sentry (the slug in your sentry.io address)")
|
|
206
|
+
result = sentry_sync(con, args.org, args.days, args.query,
|
|
207
|
+
levels=tuple(args.levels.split(",")) if args.levels else ())
|
|
208
|
+
else:
|
|
209
|
+
result = pd_sync(con, args.days, tuple(u for u in args.urgencies.split(",") if u))
|
|
210
|
+
except (SentryError, PagerDutyError) as e:
|
|
211
|
+
raise CliError(str(e)) from None
|
|
212
|
+
print(f"fetched {result['fetched']} issues, imported {result['imported']} as incidents. "
|
|
213
|
+
f"{args.source.title()} gives no suspect commits, so link them with an export or --incident-linking.")
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def cmd_status(args):
|
|
217
|
+
from .status import status
|
|
218
|
+
need_ledger(args.repo)
|
|
219
|
+
with closing(connect(db_path(args.repo))) as con:
|
|
220
|
+
r = status(con)
|
|
221
|
+
print(f"{r['decisions']} decisions, {r['attributed']} with commits, {r['mature']} with a closed "
|
|
222
|
+
f"window; {r['incidents']} incidents, {r['deploys']} deploys")
|
|
223
|
+
for arm, a in r["arms"].items():
|
|
224
|
+
print(f" {arm:<16}{a['decisions']:>6} decisions (lowest chance {a['lowest_propensity'] * 100:.1f}%)")
|
|
225
|
+
active = active_decisions(args.repo)
|
|
226
|
+
if active:
|
|
227
|
+
print("\nactive now (commits on these branches are being tagged):")
|
|
228
|
+
for branch, decision_id in active:
|
|
229
|
+
print(f" {branch:<28}{decision_id}")
|
|
230
|
+
print()
|
|
231
|
+
for note in r["notes"] or ["Nothing needs attention."]:
|
|
232
|
+
print(f"- {note}")
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def active_decisions(repo):
|
|
236
|
+
"""[(branch, decision id)] for every branch with an active decision."""
|
|
237
|
+
folder = active_dir(repo)
|
|
238
|
+
found = []
|
|
239
|
+
for f in sorted(folder.glob("*")) if folder.exists() else []:
|
|
240
|
+
lines = f.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
241
|
+
if lines:
|
|
242
|
+
found.append((lines[1] if len(lines) > 1 else "?", lines[0]))
|
|
243
|
+
return sorted(found)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def cmd_plan(args):
|
|
247
|
+
from .plan import plan
|
|
248
|
+
try:
|
|
249
|
+
grid = [int(x) for x in args.grid.split(",")] if args.grid else None
|
|
250
|
+
except ValueError:
|
|
251
|
+
raise CliError("--grid must be whole numbers separated by commas, e.g. --grid 500,1000,2000") from None
|
|
252
|
+
r = plan(args.base_rate, args.effect, args.arms, args.epsilon, args.prefer, grid, args.reps, args.seed, args.power)
|
|
253
|
+
print(f"Detecting {r['effect'] * 100:+.1f}pp from a {r['base_rate'] * 100:.0f}% base rate, {r['arms']} arms, "
|
|
254
|
+
f"epsilon {r['epsilon']}, {r['reps']} simulations per row\n")
|
|
255
|
+
print(f"{'decisions':>10}{'power (one look)':>20}{'power (checked repeatedly)':>30}")
|
|
256
|
+
for row in r["table"]:
|
|
257
|
+
print(f"{row['n']:>10}{row['power_fixed']:>20.2f}{row['power_anytime']:>30.2f}")
|
|
258
|
+
for label_, key in (("one look", "n_for_target_fixed"), ("checked repeatedly", "n_for_target_anytime")):
|
|
259
|
+
n = r[key]
|
|
260
|
+
print(f"\n{r['target'] * 100:.0f}% power, {label_}: " + (f"about {n} finished decisions" if n else "not reached in this grid"))
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def run_label(repo, horizon, as_of, rev="HEAD", ignore_file=None, fix_re=None, early_days=7,
|
|
264
|
+
incident_linking="off", max_blamed=DEFAULT_MAX_BLAMED):
|
|
265
|
+
ignore = []
|
|
266
|
+
if ignore_file:
|
|
267
|
+
path = Path(ignore_file) if Path(ignore_file).is_absolute() else Path(repo) / ignore_file
|
|
268
|
+
if path.exists():
|
|
269
|
+
ignore = [ln.split()[0] for ln in path.read_text().splitlines()
|
|
270
|
+
if ln.strip() and not ln.startswith("#")]
|
|
271
|
+
with closing(connect(db_path(repo))) as con:
|
|
272
|
+
stats = label(repo, con, horizon, as_of, rev, ignore, fix_re or fix_pattern("conventional"), early_days,
|
|
273
|
+
incident_linking, max_blamed)
|
|
274
|
+
write_json(ledger_dir(repo) / "label.json", stats)
|
|
275
|
+
return stats
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def run_estimate(repo, baseline=None, boot=500, clip=None, stratum="stratum", seed=0, cluster=None, planned_n=2000,
|
|
279
|
+
folds=DEFAULT_FOLDS):
|
|
280
|
+
with closing(connect(db_path(repo))) as con:
|
|
281
|
+
rows = load_rows(con, stratum, cluster)
|
|
282
|
+
if not rows:
|
|
283
|
+
raise CliError("no finished decisions to estimate from yet. Decisions count once their window has passed "
|
|
284
|
+
"and `ledger label` has run; `ledger status` shows what is missing")
|
|
285
|
+
result = estimate(rows, baseline=baseline, boot=boot, clip=clip, seed=seed, planned_n=planned_n, folds=folds)
|
|
286
|
+
result["merge_balance"] = merge_balance(con)
|
|
287
|
+
result["early"] = early_readout(con, result["baseline"], stratum, cluster, planned_n)
|
|
288
|
+
write_json(ledger_dir(repo) / "estimate.json", result)
|
|
289
|
+
return result
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def run_report(repo, out=None, primary="dr", title="Consequence Ledger"):
|
|
293
|
+
d = ledger_dir(repo)
|
|
294
|
+
for name in ("estimate.json", "label.json"):
|
|
295
|
+
if not (d / name).exists():
|
|
296
|
+
raise CliError(f"nothing to report yet: run `ledger label` and `ledger estimate` first (missing {name})")
|
|
297
|
+
est = json.loads((d / "estimate.json").read_text())
|
|
298
|
+
stats = json.loads((d / "label.json").read_text())
|
|
299
|
+
truth = json.loads((d / "truth.json").read_text()) if (d / "truth.json").exists() else None
|
|
300
|
+
validation = json.loads((d / "validation.json").read_text()) if (d / "validation.json").exists() else None
|
|
301
|
+
out = Path(out or d / "report.html")
|
|
302
|
+
out.write_text(report_mod.render(est, stats, title, primary, truth, {"repo": Path(repo).name},
|
|
303
|
+
validation), encoding="utf-8")
|
|
304
|
+
return out
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def print_estimate(result, primary="dr"):
|
|
308
|
+
print(f"\n{'arm':<12}{'n':>6}{'naive':>9}{'ipw':>9}{'snipw':>9}{primary:>9} 95% CI vs {result['baseline']}")
|
|
309
|
+
for arm, e in result["arms"].items():
|
|
310
|
+
est = e["estimates"]
|
|
311
|
+
vs = e["vs_baseline"][primary]
|
|
312
|
+
if result["diagnostics"][arm]["n"] == 0:
|
|
313
|
+
print(f"{arm:<12}{0:>6} never chosen: no estimate")
|
|
314
|
+
continue
|
|
315
|
+
diff = "" if arm == result["baseline"] else f"{vs['value'] * 100:+.1f}pp [{vs['lo'] * 100:+.1f}, {vs['hi'] * 100:+.1f}]"
|
|
316
|
+
print(f"{arm:<12}{result['diagnostics'][arm]['n']:>6}"
|
|
317
|
+
+ "".join(f"{est[k]['value'] * 100:>8.1f}%" for k in ("naive", "ipw", "snipw", primary))
|
|
318
|
+
+ f" [{est[primary]['lo'] * 100:4.1f}, {est[primary]['hi'] * 100:4.1f}] {diff}")
|
|
319
|
+
early = result.get("early")
|
|
320
|
+
if early:
|
|
321
|
+
print(f"\nearly readout ({early['n']} decisions past the early window), dr vs {early['baseline']}: "
|
|
322
|
+
+ ", ".join(f"{a} {v['diff'] * 100:+.1f}pp [{v['diff_lo'] * 100:+.1f}, {v['diff_hi'] * 100:+.1f}]"
|
|
323
|
+
for a, v in early["arms"].items() if a != early["baseline"]))
|
|
324
|
+
cal = early.get("calibration")
|
|
325
|
+
if cal:
|
|
326
|
+
print(f" on {cal['n']} decisions with both windows closed, {cal['surfaced_early'] * 100:.0f}% of final bad "
|
|
327
|
+
"outcomes had surfaced early; early vs final there: "
|
|
328
|
+
+ ", ".join(f"{a} {v['early_diff'] * 100:+.1f} vs {v['final_diff'] * 100:+.1f}pp" for a, v in cal["pairs"].items()))
|
|
329
|
+
cs = [(a, v["dr"]) for a, v in result.get("analytic", {}).items() if a != result["baseline"]]
|
|
330
|
+
if cs:
|
|
331
|
+
print("\nanytime-valid (safe if you check repeatedly), dr vs " + result["baseline"] + ": "
|
|
332
|
+
+ ", ".join(f"{a} [{d['cs_lo'] * 100:+.1f}, {d['cs_hi'] * 100:+.1f}]pp" for a, d in cs))
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def segment_rows(args):
|
|
336
|
+
"""Finished decisions with the arms and baseline to compare, and the folds to use."""
|
|
337
|
+
need_ledger(args.repo)
|
|
338
|
+
with closing(connect(db_path(args.repo))) as con:
|
|
339
|
+
rows = load_rows(con, args.stratum, args.cluster)
|
|
340
|
+
if not rows:
|
|
341
|
+
raise CliError("no finished decisions yet. Decisions count once their window has passed and "
|
|
342
|
+
"`ledger label` has run; `ledger status` shows what is missing")
|
|
343
|
+
arms = sorted({r["arm"] for r in rows} | {a for r in rows for a in r["p"]})
|
|
344
|
+
baseline = args.baseline or arms[0]
|
|
345
|
+
if baseline not in arms:
|
|
346
|
+
raise CliError(f"baseline {baseline!r} is not one of {arms}")
|
|
347
|
+
folds = args.folds if len(rows) >= MIN_CROSSFIT else 0
|
|
348
|
+
return rows, arms, baseline, folds
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def cmd_breakdown(args):
|
|
352
|
+
rows, arms, baseline, folds = segment_rows(args)
|
|
353
|
+
result = {"baseline": baseline, "stratum": args.stratum, "n": len(rows),
|
|
354
|
+
"arms": breakdown(rows, arms, baseline, args.clip, folds)}
|
|
355
|
+
write_json(ledger_dir(args.repo) / "breakdown.json", result)
|
|
356
|
+
if args.json:
|
|
357
|
+
print(json.dumps(result, indent=2))
|
|
358
|
+
return
|
|
359
|
+
if not result["arms"]:
|
|
360
|
+
print("not enough data per stratum to compare anything yet")
|
|
361
|
+
return
|
|
362
|
+
for arm, d in result["arms"].items():
|
|
363
|
+
print(f"\n{arm} vs {baseline}, by '{args.stratum}' (overall {d['overall'] * 100:+.1f}pp, "
|
|
364
|
+
f"spread between strata {d['tau'] * 100:.1f}pp)")
|
|
365
|
+
print(f"{'stratum':<22}{'n':>6}{'raw effect':>14}{'shrunk':>9} shrunk 95% CI adj. p")
|
|
366
|
+
for x in sorted(d["strata"], key=lambda x: x["shrunk"]):
|
|
367
|
+
print(f"{x['stratum']:<22}{x['n']:>6}{x['effect'] * 100:>+13.1f}p{x['shrunk'] * 100:>+8.1f}p"
|
|
368
|
+
f" [{x['shrunk_lo'] * 100:+5.1f}, {x['shrunk_hi'] * 100:+5.1f}] {x['p_adjusted']:.3f}")
|
|
369
|
+
verdict = ("at least one stratum differs from the rest after adjusting for looking at all of them"
|
|
370
|
+
if d["heterogeneous"] else "no stratum stands out once you account for how many were examined")
|
|
371
|
+
print(f"-> {verdict}.")
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def cmd_recommend(args):
|
|
375
|
+
rows, arms, baseline, folds = segment_rows(args)
|
|
376
|
+
try:
|
|
377
|
+
result = recommend(rows, arms, baseline, args.clip, folds)
|
|
378
|
+
except ValueError as exc:
|
|
379
|
+
raise CliError(str(exc))
|
|
380
|
+
result.update(baseline=baseline, stratum=args.stratum)
|
|
381
|
+
write_json(ledger_dir(args.repo) / "recommend.json", result)
|
|
382
|
+
if args.json:
|
|
383
|
+
print(json.dumps(result, indent=2))
|
|
384
|
+
return
|
|
385
|
+
pp = lambda v: f"{v * 100:+.2f}pp"
|
|
386
|
+
print(f"\nwhat happened: {result['current_rate'] * 100:.1f}% of {len(rows)} decisions went bad")
|
|
387
|
+
print(f"routing each '{args.stratum}' to its best tool (chosen on one half, judged on the other):")
|
|
388
|
+
print(f" {pp(result['gain'])} fewer bad outcomes [{pp(result['gain_lo'])}, {pp(result['gain_hi'])}]")
|
|
389
|
+
print(f"always using the single best tool: {pp(result['best_single_gain'])} "
|
|
390
|
+
f"[{pp(result['best_single_lo'])}, {pp(result['best_single_hi'])}]")
|
|
391
|
+
if result["stable_policy"]:
|
|
392
|
+
print("choices that did not change between halves: "
|
|
393
|
+
+ ", ".join(f"{s} -> {a}" for s, a in sorted(result["stable_policy"].items())))
|
|
394
|
+
if result["unstable_strata"]:
|
|
395
|
+
print("changed with the data (no real preference): " + ", ".join(result["unstable_strata"]))
|
|
396
|
+
if result["worth_it"]:
|
|
397
|
+
print("-> the interval is above zero. Shift `--scores` toward these choices, and keep epsilon at 0.2 or more "
|
|
398
|
+
"so the next estimate stays valid.")
|
|
399
|
+
elif result["best_single_lo"] > 0:
|
|
400
|
+
print("-> any gain comes from one tool being better overall, not from matching tools to task types. "
|
|
401
|
+
"`ledger estimate` shows which.")
|
|
402
|
+
else:
|
|
403
|
+
print("-> the interval includes zero: no evidence yet that routing by task type beats what you did. "
|
|
404
|
+
"Keep randomizing.")
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def cmd_label(args):
|
|
408
|
+
need_ledger(args.repo)
|
|
409
|
+
print(json.dumps(run_label(args.repo, args.horizon, parse_date(args.as_of), args.rev, args.ignore_revs,
|
|
410
|
+
fix_pattern(args.fix_style, args.fix_regex), args.early_days,
|
|
411
|
+
args.incident_linking, args.max_blamed), indent=2))
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def cmd_estimate(args):
|
|
415
|
+
need_ledger(args.repo)
|
|
416
|
+
result = run_estimate(args.repo, args.baseline, args.boot, args.clip, args.stratum, cluster=args.cluster,
|
|
417
|
+
planned_n=args.planned_n, folds=args.folds)
|
|
418
|
+
if args.json:
|
|
419
|
+
print(json.dumps(result, indent=2))
|
|
420
|
+
else:
|
|
421
|
+
print_estimate(result, args.primary)
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def cmd_report(args):
|
|
425
|
+
need_ledger(args.repo)
|
|
426
|
+
print(run_report(args.repo, args.out, args.primary))
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def cmd_audit(args):
|
|
430
|
+
from .audit import audit, compile_rules
|
|
431
|
+
ignore = []
|
|
432
|
+
path = Path(args.repo) / args.ignore_revs
|
|
433
|
+
if path.exists():
|
|
434
|
+
ignore = [ln.split()[0] for ln in path.read_text().splitlines() if ln.strip() and not ln.startswith("#")]
|
|
435
|
+
result = audit(args.repo, args.horizon, parse_date(args.as_of), parse_date(args.since), args.rev,
|
|
436
|
+
compile_rules(args.rule), ignore, fix_pattern(args.fix_style, args.fix_regex), args.max_blamed)
|
|
437
|
+
if args.json:
|
|
438
|
+
Path(args.json).write_text(json.dumps(result, indent=2))
|
|
439
|
+
if args.html:
|
|
440
|
+
Path(args.html).write_text(report_mod.render_audit(result, Path(args.repo).resolve().name), encoding="utf-8")
|
|
441
|
+
diffuse = result.get("fixes_too_diffuse", 0)
|
|
442
|
+
print(f"{result['mature_commits']} mature commits of {result['commits_scanned']}, {result['fix_commits']} fix commits"
|
|
443
|
+
+ (f" ({diffuse} ignored as too diffuse)" if diffuse else "")
|
|
444
|
+
+ f" (window {result['horizon_days']} days). Correlational: tools are used on different work.\n")
|
|
445
|
+
if not result["tools"]:
|
|
446
|
+
print(f"Nothing to report: no commit is old enough for its {result['horizon_days']}-day window to have closed "
|
|
447
|
+
"(or the history is empty after --since). Try a shorter --horizon, or come back later.")
|
|
448
|
+
return
|
|
449
|
+
print(f"{'tool':<14}{'commits':>8}{'reverted':>10}{'fixed':>7}{'bad rate':>10} 95% CI size-adjusted")
|
|
450
|
+
for tool, t in result["tools"].items():
|
|
451
|
+
print(f"{tool:<14}{t['commits']:>8}{t['reverted']:>10}{t['fixed']:>7}{t['bad_rate'] * 100:>9.1f}%"
|
|
452
|
+
f" [{t['ci_lo'] * 100:4.1f}, {t['ci_hi'] * 100:4.1f}] {t['size_adjusted_rate'] * 100:5.1f}%")
|
|
453
|
+
for note in result.get("detection_notes", ()):
|
|
454
|
+
print(f"\nnote: {note}")
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def cmd_demo(args):
|
|
458
|
+
from .demo import build, validate
|
|
459
|
+
out = Path(args.out or Path(tempfile.gettempdir()) / "consequence-ledger-demo")
|
|
460
|
+
print(f"building synthetic repo in {out} ...")
|
|
461
|
+
repo, as_of, truth = build(out, n=args.n, seed=args.seed, horizon_days=args.horizon)
|
|
462
|
+
stats = run_label(repo, args.horizon, as_of)
|
|
463
|
+
print(json.dumps(stats, indent=2))
|
|
464
|
+
result = run_estimate(repo, boot=args.boot, seed=args.seed)
|
|
465
|
+
print_estimate(result)
|
|
466
|
+
print("\ntrue rates: " + ", ".join(f"{a} {v * 100:.1f}%" for a, v in truth.items()))
|
|
467
|
+
v = validate(repo, boot=args.boot, seed=args.seed)
|
|
468
|
+
print(f"labeler vs oracle: precision {v['precision']:.2f}, recall {v['recall']:.2f} "
|
|
469
|
+
f"(tp {v['tp']}, fp {v['fp']}, fn {v['fn']})")
|
|
470
|
+
print("DR on oracle labels: " + ", ".join(f"{a} {e['value'] * 100:.1f}%" for a, e in v["oracle_dr"].items()))
|
|
471
|
+
print(f"\nreport: {run_report(repo, title='Consequence Ledger demo')}")
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def add_fix_args(parser):
|
|
475
|
+
parser.add_argument("--fix-style", default="conventional", choices=["conventional", "keywords"],
|
|
476
|
+
help="conventional: fix:/hotfix: prefixes; keywords: 'Fix crash...' style messages")
|
|
477
|
+
parser.add_argument("--fix-regex", help="custom regex on the commit subject (overrides --fix-style)")
|
|
478
|
+
parser.add_argument("--max-blamed", type=int, default=DEFAULT_MAX_BLAMED,
|
|
479
|
+
help="ignore a fix that blames more than this many commits as too diffuse to attribute "
|
|
480
|
+
f"(default {DEFAULT_MAX_BLAMED}; 0 keeps every fix)")
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def main(argv=None):
|
|
484
|
+
p = argparse.ArgumentParser(prog="ledger", description="Consequence Ledger: causal outcomes of AI coding decisions")
|
|
485
|
+
p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
486
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
487
|
+
|
|
488
|
+
s = sub.add_parser("init", help="create the ledger and install the commit hook")
|
|
489
|
+
s.add_argument("--repo", default=".")
|
|
490
|
+
s.set_defaults(fn=cmd_init)
|
|
491
|
+
|
|
492
|
+
s = sub.add_parser("decide", help="randomly choose an arm and log its propensity")
|
|
493
|
+
s.add_argument("--repo", default=".")
|
|
494
|
+
s.add_argument("--arms", required=True, help="comma-separated, e.g. claude-code,codex,cursor")
|
|
495
|
+
s.add_argument("--scores", help="comma-separated preference scores (default: uniform)")
|
|
496
|
+
s.add_argument("--context", default="{}", help='JSON, e.g. {"stratum": "hard"}')
|
|
497
|
+
s.add_argument("--epsilon", type=float, default=0.1)
|
|
498
|
+
s.add_argument("--temperature", type=float, default=1.0)
|
|
499
|
+
s.add_argument("--policy-version", default="v1")
|
|
500
|
+
s.set_defaults(fn=cmd_decide)
|
|
501
|
+
|
|
502
|
+
s = sub.add_parser("done", help="stop tagging commits on the current branch with its decision")
|
|
503
|
+
s.add_argument("--repo", default=".")
|
|
504
|
+
s.add_argument("--all", action="store_true", help="clear the active decision on every branch")
|
|
505
|
+
s.set_defaults(fn=cmd_done)
|
|
506
|
+
|
|
507
|
+
s = sub.add_parser("attach", help="map a commit (e.g. a squash merge) to a decision by hand")
|
|
508
|
+
s.add_argument("--repo", default=".")
|
|
509
|
+
s.add_argument("--decision", required=True)
|
|
510
|
+
s.add_argument("--commit", required=True)
|
|
511
|
+
s.set_defaults(fn=cmd_attach)
|
|
512
|
+
|
|
513
|
+
s = sub.add_parser("sync", help="pull incidents live (reads SENTRY_AUTH_TOKEN or PAGERDUTY_API_KEY from the environment)")
|
|
514
|
+
s.add_argument("source", choices=["sentry", "pagerduty"])
|
|
515
|
+
s.add_argument("--org", help="Sentry organization slug (sentry only)")
|
|
516
|
+
s.add_argument("--days", type=int, default=90)
|
|
517
|
+
s.add_argument("--query", default="", help="Sentry issue search, e.g. 'is:unresolved environment:production'")
|
|
518
|
+
s.add_argument("--levels", default="error,fatal", help="Sentry issue levels to count; empty for all")
|
|
519
|
+
s.add_argument("--urgencies", default="high", help="PagerDuty urgencies to count, comma-separated; empty for all")
|
|
520
|
+
s.add_argument("--repo", default=".")
|
|
521
|
+
s.set_defaults(fn=cmd_sync)
|
|
522
|
+
|
|
523
|
+
s = sub.add_parser("status", help="check the ledger for gaps before you trust an estimate")
|
|
524
|
+
s.add_argument("--repo", default=".")
|
|
525
|
+
s.set_defaults(fn=cmd_status)
|
|
526
|
+
|
|
527
|
+
s = sub.add_parser("plan", help="how many finished decisions you need to detect a difference")
|
|
528
|
+
s.add_argument("--base-rate", type=float, required=True, help="expected bad-outcome rate of the baseline, e.g. 0.12")
|
|
529
|
+
s.add_argument("--effect", type=float, required=True, help="difference to detect, e.g. -0.03 for 3 points better")
|
|
530
|
+
s.add_argument("--arms", type=int, default=2)
|
|
531
|
+
s.add_argument("--epsilon", type=float, default=0.1)
|
|
532
|
+
s.add_argument("--prefer", type=float, default=0.0, help="score gap in favour of the tested arm in your routing")
|
|
533
|
+
s.add_argument("--power", type=float, default=0.8)
|
|
534
|
+
s.add_argument("--grid", help="comma-separated decision counts, default 500..16000")
|
|
535
|
+
s.add_argument("--reps", type=int, default=200)
|
|
536
|
+
s.add_argument("--seed", type=int, default=0)
|
|
537
|
+
s.set_defaults(fn=cmd_plan)
|
|
538
|
+
|
|
539
|
+
s = sub.add_parser("label", help="detect reverts and fixes and label each decision")
|
|
540
|
+
s.add_argument("--repo", default=".")
|
|
541
|
+
s.add_argument("--horizon", type=int, default=30, help="days")
|
|
542
|
+
s.add_argument("--as-of", help="ISO date or unix seconds (default: now)")
|
|
543
|
+
s.add_argument("--rev", default="HEAD")
|
|
544
|
+
s.add_argument("--ignore-revs", default=".git-blame-ignore-revs", help="commits SZZ should skip")
|
|
545
|
+
s.add_argument("--early-days", type=int, default=7, help="early-warning window for a first readout (0 = off)")
|
|
546
|
+
s.add_argument("--incident-linking", default="off", choices=["off", "latest", "window"],
|
|
547
|
+
help="for incidents without suspect commits: off (default, safest), latest deploy, or every "
|
|
548
|
+
"matching commit in the window")
|
|
549
|
+
add_fix_args(s)
|
|
550
|
+
s.set_defaults(fn=cmd_label)
|
|
551
|
+
|
|
552
|
+
s = sub.add_parser("import", help="import incidents or deploys exported from Sentry, PagerDuty, etc.")
|
|
553
|
+
s.add_argument("kind", choices=["incidents", "deploys"])
|
|
554
|
+
s.add_argument("--file", required=True, help="CSV or JSON export")
|
|
555
|
+
s.add_argument("--repo", default=".")
|
|
556
|
+
s.set_defaults(fn=cmd_import)
|
|
557
|
+
|
|
558
|
+
s = sub.add_parser("estimate", help="causal estimates per arm")
|
|
559
|
+
s.add_argument("--repo", default=".")
|
|
560
|
+
s.add_argument("--baseline")
|
|
561
|
+
s.add_argument("--boot", type=int, default=500)
|
|
562
|
+
s.add_argument("--clip", type=float, help="cap on importance weights")
|
|
563
|
+
s.add_argument("--stratum", default="stratum", help="context key used by the outcome model")
|
|
564
|
+
s.add_argument("--cluster", help="context key whose values share outcomes, e.g. developer or service")
|
|
565
|
+
s.add_argument("--planned-n", type=int, default=2000,
|
|
566
|
+
help="decisions you expect at the final readout; tunes the anytime-valid interval (fix it up front)")
|
|
567
|
+
s.add_argument("--primary", default="dr", choices=["ipw", "snipw", "dm", "dr"])
|
|
568
|
+
s.add_argument("--folds", type=int, default=DEFAULT_FOLDS,
|
|
569
|
+
help="cross-fitting folds for the outcome model (0 fits it on all decisions)")
|
|
570
|
+
s.add_argument("--json", action="store_true")
|
|
571
|
+
s.set_defaults(fn=cmd_estimate)
|
|
572
|
+
|
|
573
|
+
s = sub.add_parser("breakdown", help="which tool is better for which kind of task (shrunk, multiplicity-aware)")
|
|
574
|
+
s.add_argument("--repo", default=".")
|
|
575
|
+
s.add_argument("--baseline")
|
|
576
|
+
s.add_argument("--clip", type=float, help="cap on importance weights")
|
|
577
|
+
s.add_argument("--stratum", default="stratum", help="context key (or comma-separated keys) to break down by")
|
|
578
|
+
s.add_argument("--cluster", help="context key whose values share outcomes")
|
|
579
|
+
s.add_argument("--folds", type=int, default=DEFAULT_FOLDS)
|
|
580
|
+
s.add_argument("--json", action="store_true")
|
|
581
|
+
s.set_defaults(fn=cmd_breakdown)
|
|
582
|
+
|
|
583
|
+
s = sub.add_parser("recommend", help="what routing each kind of task to its best tool would be worth (honestly)")
|
|
584
|
+
s.add_argument("--repo", default=".")
|
|
585
|
+
s.add_argument("--baseline")
|
|
586
|
+
s.add_argument("--clip", type=float, help="cap on importance weights")
|
|
587
|
+
s.add_argument("--stratum", default="stratum", help="context key (or comma-separated keys) to route by")
|
|
588
|
+
s.add_argument("--cluster", help="context key whose values share outcomes")
|
|
589
|
+
s.add_argument("--folds", type=int, default=DEFAULT_FOLDS)
|
|
590
|
+
s.add_argument("--json", action="store_true")
|
|
591
|
+
s.set_defaults(fn=cmd_recommend)
|
|
592
|
+
|
|
593
|
+
s = sub.add_parser("report", help="write the HTML report")
|
|
594
|
+
s.add_argument("--repo", default=".")
|
|
595
|
+
s.add_argument("--out")
|
|
596
|
+
s.add_argument("--primary", default="dr", choices=["ipw", "snipw", "dm", "dr"])
|
|
597
|
+
s.set_defaults(fn=cmd_report)
|
|
598
|
+
|
|
599
|
+
s = sub.add_parser("audit", help="baseline revert/fix rates per AI tool from history (correlational)")
|
|
600
|
+
s.add_argument("--repo", default=".")
|
|
601
|
+
s.add_argument("--horizon", type=int, default=30, help="days")
|
|
602
|
+
s.add_argument("--as-of", help="ISO date or unix seconds (default: now)")
|
|
603
|
+
s.add_argument("--since", help="only commits after this date")
|
|
604
|
+
s.add_argument("--rev", default="HEAD")
|
|
605
|
+
s.add_argument("--rule", action="append", default=[], help="extra tool=regex on the commit message")
|
|
606
|
+
s.add_argument("--ignore-revs", default=".git-blame-ignore-revs")
|
|
607
|
+
s.add_argument("--json", help="write full results here")
|
|
608
|
+
s.add_argument("--html", help="write an HTML summary here")
|
|
609
|
+
add_fix_args(s)
|
|
610
|
+
s.set_defaults(fn=cmd_audit)
|
|
611
|
+
|
|
612
|
+
s = sub.add_parser("demo", help="synthetic end-to-end run with known true effects")
|
|
613
|
+
s.add_argument("--out")
|
|
614
|
+
s.add_argument("--n", type=int, default=2400)
|
|
615
|
+
s.add_argument("--seed", type=int, default=7)
|
|
616
|
+
s.add_argument("--horizon", type=int, default=30)
|
|
617
|
+
s.add_argument("--boot", type=int, default=300)
|
|
618
|
+
s.set_defaults(fn=cmd_demo)
|
|
619
|
+
|
|
620
|
+
args = p.parse_args(argv)
|
|
621
|
+
try:
|
|
622
|
+
if getattr(args, "repo", None) is not None and args.cmd not in ("demo", "plan"):
|
|
623
|
+
# Work from the repository's main working tree whatever folder or worktree we were run in,
|
|
624
|
+
# but remember where we were: its checked-out branch decides which decision applies.
|
|
625
|
+
args.workdir = args.repo
|
|
626
|
+
args.repo = str(repo_root(args.repo))
|
|
627
|
+
args.fn(args)
|
|
628
|
+
except CliError as e:
|
|
629
|
+
sys.exit(f"error: {e}")
|
|
630
|
+
except GitError as e:
|
|
631
|
+
sys.exit(f"error: {friendly_git_error(e, getattr(args, 'repo', '.'))}")
|
|
632
|
+
except (ValueError, OSError) as e:
|
|
633
|
+
# ValueError covers invalid options deep in the library (epsilon, horizon, base rate ...);
|
|
634
|
+
# OSError covers missing files, permissions, and the demo refusing to overwrite a folder.
|
|
635
|
+
sys.exit(f"error: {e}")
|
|
636
|
+
except KeyboardInterrupt:
|
|
637
|
+
sys.exit(130)
|