consequence-ledger 0.3.0__py3-none-any.whl

ledger/db.py

@@ -0,0 +1,70 @@
+"""SQLite storage for decisions, commit attribution and outcome labels."""
+import sqlite3
+from pathlib import Path
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS decisions(
+    id             TEXT PRIMARY KEY,
+    ts             INTEGER NOT NULL,     -- unix seconds when the arm was chosen
+    policy_version TEXT NOT NULL,
+    context        TEXT NOT NULL,        -- JSON object; "stratum" is used by the estimators
+    arms           TEXT NOT NULL,        -- JSON list of candidate arms
+    propensities   TEXT NOT NULL,        -- JSON {arm: probability} for every candidate
+    chosen         TEXT NOT NULL,
+    propensity     REAL NOT NULL         -- probability of the chosen arm
+);
+CREATE TABLE IF NOT EXISTS commits(
+    sha         TEXT PRIMARY KEY,
+    decision_id TEXT NOT NULL REFERENCES decisions(id),
+    ts          INTEGER,
+    source      TEXT NOT NULL            -- 'trailer' or 'manual'
+);
+CREATE TABLE IF NOT EXISTS szz_cache(
+    fix_sha     TEXT NOT NULL,
+    key         TEXT NOT NULL,           -- algorithm version + skip rules + ignore list
+    introducers TEXT NOT NULL,           -- JSON list of commits whose lines the fix changed
+    PRIMARY KEY (fix_sha, key)
+);
+CREATE TABLE IF NOT EXISTS outcomes(
+    decision_id  TEXT PRIMARY KEY REFERENCES decisions(id),
+    bad          INTEGER NOT NULL,       -- 1 if reverted or fixed within the horizon
+    mature       INTEGER NOT NULL,       -- 1 if the full horizon has elapsed
+    kind         TEXT NOT NULL,          -- 'revert' | 'fix' | 'clean' | 'no_commit'
+    event_sha    TEXT,
+    delay_days   REAL,
+    horizon_days INTEGER NOT NULL,
+    as_of        INTEGER NOT NULL,
+    early_bad    INTEGER NOT NULL DEFAULT 0,  -- bad within the early window (e.g. 7 days)
+    early_mature INTEGER NOT NULL DEFAULT 0,  -- 1 if the early window has elapsed
+    early_days   INTEGER
+);
+"""
+
+
+def db_path(repo):
+    return Path(repo) / ".ledger" / "ledger.db"
+
+
+def connect(path):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    ignore = path.parent / ".gitignore"
+    if path.parent.name == ".ledger" and not ignore.exists():
+        # The ledger must never be committed: it holds every decision and would churn on each run.
+        ignore.write_text("*\n")
+    con = sqlite3.connect(path)
+    con.row_factory = sqlite3.Row
+    con.executescript(SCHEMA)
+    migrate(con)
+    from .incidents import ensure_schema
+    ensure_schema(con)
+    return con
+
+
+def migrate(con):
+    """Add columns introduced after a ledger was first created."""
+    have = {r["name"] for r in con.execute("PRAGMA table_info(outcomes)")}
+    for name, ddl in (("early_bad", "INTEGER NOT NULL DEFAULT 0"), ("early_mature", "INTEGER NOT NULL DEFAULT 0"),
+                      ("early_days", "INTEGER")):
+        if name not in have:
+            con.execute(f"ALTER TABLE outcomes ADD COLUMN {name} {ddl}")

ledger/demo.py

@@ -0,0 +1,306 @@
+"""Build a synthetic repository with a known ground truth, then run the pipeline on it.
+
+The world: three agents handle easy and hard tasks. The logging policy routes
+hard tasks mostly to agent-B (the "strong" one), so naive comparisons blame B
+for the hard tasks it was handed. The true effects are fixed in advance, so
+the report shows whether the causal estimators recover them.
+
+Git objects are written directly (zlib-compressed loose objects) so thousands
+of commits with back-dated timestamps take seconds, not minutes.
+"""
+import hashlib
+import heapq
+import json
+import os
+import random
+import shutil
+import stat
+import sys
+import zlib
+from pathlib import Path
+
+from .db import connect
+from .gitio import git
+from .policy import record_decision
+
+DAY = 86400
+ARMS = ["agent-A", "agent-B", "agent-C"]
+STRATA = {"easy": 0.65, "hard": 0.35}
+# Logging-policy scores: hard work is steered to agent-B.
+SCORES = {"easy": [1.0, 0.2, 0.8], "hard": [0.2, 1.6, 0.1]}
+# True probability that a decision's change goes bad (revert or bug fix) at some point.
+P_BAD = {
+    ("easy", "agent-A"): 0.06, ("easy", "agent-B"): 0.03, ("easy", "agent-C"): 0.12,
+    ("hard", "agent-A"): 0.24, ("hard", "agent-B"): 0.20, ("hard", "agent-C"): 0.30,
+}
+P_IN_HORIZON = 0.85   # share of bad outcomes that surface within 30 days
+P_REVERT = 0.45       # bad outcomes that are reverts (the rest are forward fixes)
+P_UNMERGED = 0.06     # decisions whose change never merges
+P_INCIDENT = 0.4      # forward-fix outcomes that instead surface as a production incident (no git trace)
+DEVS = [("Asha Rao", "asha"), ("Ben Ode", "ben"), ("Chen Wu", "chen"), ("Dara Kim", "dara"),
+        ("Eli Moss", "eli"), ("Femi Ade", "femi")]
+
+
+def rmtree(path):
+    # onexc replaced onerror in Python 3.12; both receive (func, path, error-ish).
+    if sys.version_info >= (3, 12):
+        shutil.rmtree(path, onexc=_force_remove)
+    else:
+        shutil.rmtree(path, onerror=_force_remove)
+
+
+def _force_remove(func, path, _exc):
+    # Git marks pack files read-only, which blocks deletion on Windows.
+    os.chmod(path, stat.S_IWRITE)
+    func(path)
+
+
+def true_rates(horizon_share=P_IN_HORIZON):
+    """E[Y(a)] over the task population: P(bad within the horizon | every task goes to a)."""
+    return {a: sum(w * P_BAD[(s, a)] for s, w in STRATA.items()) * horizon_share for a in ARMS}
+
+
+class ObjectWriter:
+    """Minimal git object store writer for a flat tree of text files."""
+
+    def __init__(self, path):
+        self.path = Path(path)
+        git(path.parent, "init", "-q", "-b", "main", path.name)
+        self.objects = self.path / ".git" / "objects"
+        self.files = {}
+        self.blobs = {}
+        self.dirty = set()
+        self.head = None
+
+    def _write(self, kind, data):
+        raw = f"{kind} {len(data)}\0".encode() + data
+        sha = hashlib.sha1(raw).hexdigest()
+        target = self.objects / sha[:2] / sha[2:]
+        if not target.exists():
+            target.parent.mkdir(exist_ok=True)
+            target.write_bytes(zlib.compress(raw, 1))
+        return sha
+
+    def commit(self, message, ts, author):
+        for name in self.dirty:
+            self.blobs[name] = self._write("blob", ("\n".join(self.files[name]) + "\n").encode())
+        self.dirty.clear()
+        tree = b"".join(b"100644 " + n.encode() + b"\0" + bytes.fromhex(self.blobs[n])
+                        for n in sorted(self.blobs))
+        tree_sha = self._write("tree", tree)
+        name, handle = author
+        ident = f"{name} <{handle}@example.test> {ts} +0000"
+        body = f"tree {tree_sha}\n" + (f"parent {self.head}\n" if self.head else "")
+        body += f"author {ident}\ncommitter {ident}\n\n{message.rstrip()}\n"
+        self.head = self._write("commit", body.encode())
+        return self.head
+
+    def finish(self):
+        (self.path / ".git" / "refs" / "heads" / "main").write_text(self.head + "\n")
+        git(self.path, "repack", "-a", "-d", "-q")
+        git(self.path, "prune")
+        git(self.path, "reset", "-q", "--hard")
+
+
+def build(out, n=2400, days=240, seed=7, horizon_days=30, incidents=True):
+    """Create the repo and ledger. Returns (repo_path, as_of, truth).
+
+    With incidents on, the repo deploys daily and some bad changes show up only as production
+    incidents: half exported with a suspect commit (Sentry style), half with just the files
+    involved, so the ledger must find them through the deploy window.
+    """
+    rng = random.Random(seed)
+    # Incident choices use their own stream so the git history matches a run without incidents.
+    inc_rng = random.Random(seed + 1)
+    out = Path(out)
+    if out.exists() and any(out.iterdir()):
+        if not (out / ".ledger" / "truth.json").exists():
+            raise FileExistsError(f"{out} is not empty and is not a previous demo; pick another --out")
+        rmtree(out)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    repo = ObjectWriter(out)
+    con = connect(out / ".ledger" / "ledger.db")
+    (out / ".git" / "info" / "exclude").write_text(".ledger/\n")
+
+    start = 1_746_057_600  # 2025-05-01 00:00 UTC
+    as_of = start + days * DAY
+
+    files = [f"src/mod_{i:02d}.py".replace("/", "_") for i in range(30)]
+    for f in files:
+        repo.files[f] = [f"# {f}", "def run(ctx):"] + [f"    base_{f[:-3]}_{j}(ctx)" for j in range(30)]
+        repo.dirty.add(f)
+    repo.commit("chore: initial import", start - DAY, DEVS[0])
+
+    queue = []  # (ts, seq, kind, payload)
+    seq = 0
+
+    def push(ts, kind, payload):
+        nonlocal seq
+        heapq.heappush(queue, (int(ts), seq, kind, payload))
+        seq += 1
+
+    for i in range(n):
+        ts = start + rng.random() * (days - 2) * DAY
+        stratum = "easy" if rng.random() < STRATA["easy"] else "hard"
+        d = record_decision(con, ARMS, SCORES[stratum], {"stratum": stratum, "task": f"T-{i:05d}"},
+                            epsilon=0.1, temperature=0.5, policy_version="demo-v1", ts=ts, rng=rng)
+        if rng.random() < P_UNMERGED:
+            continue
+        merge_ts = ts + rng.uniform(0.1, 2.0) * DAY
+        push(merge_ts, "decision", d)
+        if rng.random() < P_BAD[(stratum, d["chosen"])]:
+            delay = rng.uniform(1, horizon_days - 2) if rng.random() < P_IN_HORIZON else rng.uniform(horizon_days + 3, 90)
+            kind = "revert" if rng.random() < P_REVERT else "fix"
+            if incidents and kind == "fix" and inc_rng.random() < P_INCIDENT:
+                kind = "incident"
+            push(merge_ts + delay * DAY, kind, d["id"])
+    for i in range(n // 2):
+        push(start + rng.random() * days * DAY, "human", i)
+    if incidents:
+        for k in range(days + 1):
+            push(start + k * DAY + 18 * 3600, "deploy", k)  # daily deploy at 18:00
+    con.commit()
+
+    placed = {}  # decision id -> (file, lines, sha, subject, merge_ts)
+    first_event = {}  # decision id -> seconds from merge to the first revert/fix/incident that happened
+    deploys, incident_rows = [], []
+    while queue:
+        ts, _, kind, payload = heapq.heappop(queue)
+        if ts > as_of:
+            continue
+        if kind == "deploy":
+            deploys.append((repo.head, ts))
+            continue
+        if kind == "incident":
+            if payload in placed:
+                f, lines, sha, subject, merged = placed[payload]
+                shipped = next((t for _, t in deploys if t >= merged), None)
+                if shipped is not None:
+                    # An incident can only start once the change is in production.
+                    inc_ts = max(ts, shipped + 3600)
+                    by_suspect = inc_rng.random() < 0.5
+                    incident_rows.append((f"INC-{len(incident_rows) + 1:04d}", inc_ts,
+                                          sha if by_suspect else "", "" if by_suspect else f, sha))
+                    first_event.setdefault(payload, inc_ts - merged)
+            continue
+        dev = rng.choice(DEVS)
+        if kind == "decision":
+            d = payload
+            f = rng.choice(files)
+            lines = [f"    step_{d['id'][:10]}_{j}(ctx)" for j in range(rng.randint(3, 9))]
+            pos = rng.randint(2, len(repo.files[f]))
+            repo.files[f][pos:pos] = lines
+            repo.dirty.add(f)
+            subject = f"feat: {d['context']['task']} {rng.choice(['add', 'extend', 'wire up', 'handle'])} {f[:-3]}"
+            sha = repo.commit(f"{subject}\n\nLedger-Decision-Id: {d['id']}", ts, dev)
+            placed[d["id"]] = (f, lines, sha, subject, ts)
+        elif kind in ("revert", "fix"):
+            if payload not in placed:
+                continue
+            f, lines, sha, subject, merged = placed[payload]
+            present = [ln for ln in lines if ln in repo.files[f]]
+            if not present:
+                continue
+            if kind == "revert":
+                repo.files[f] = [ln for ln in repo.files[f] if ln not in set(lines)]
+                msg = f'Revert "{subject}"\n\nThis reverts commit {sha}.'
+            else:
+                for ln in rng.sample(present, min(2, len(present))):
+                    i = repo.files[f].index(ln)
+                    repo.files[f][i] = ln + "  # guard None ctx"
+                msg = f"fix: handle missing ctx in {f[:-3]}"
+            repo.dirty.add(f)
+            repo.commit(msg, ts, dev)
+            first_event.setdefault(payload, ts - merged)
+        else:
+            f = rng.choice(files)
+            i = rng.randint(2, len(repo.files[f]) - 1)
+            repo.files[f][i] = f"    tidy_{payload}(ctx)"
+            repo.dirty.add(f)
+            noisy_fix = rng.random() < 0.12
+            repo.commit(f"{'fix: off-by-one' if noisy_fix else 'refactor: tidy'} in {f[:-3]}", ts, dev)
+    repo.finish()
+    if incidents:
+        from .incidents import import_deploys, import_incidents
+        led = out / ".ledger"
+        (led / "deploys.csv").write_text("sha,deployed_at\n" + "".join(f"{s},{t}\n" for s, t in deploys))
+        (led / "incidents.csv").write_text("id,started_at,commits,files\n"
+                                           + "".join(f"{i},{t},{c},{f}\n" for i, t, c, f, _ in incident_rows))
+        # The true culprit of each incident, kept apart from the export so the ledger never sees it.
+        (led / "oracle_incidents.json").write_text(json.dumps(
+            {i: {"culprit": culprit, "method": "suspect" if c else "deploy"} for i, t, c, f, culprit in incident_rows}))
+        import_deploys(con, led / "deploys.csv")
+        import_incidents(con, led / "incidents.csv")
+
+    truth = true_rates()
+    (out / ".ledger" / "truth.json").write_text(json.dumps(truth, indent=2))
+    # Oracle label per merged decision: did a revert/fix really land within the horizon?
+    oracle = {d: int(first_event.get(d, horizon_days * DAY + 1) <= horizon_days * DAY) for d in placed}
+    (out / ".ledger" / "oracle.json").write_text(json.dumps(oracle))
+    con.close()
+    return out, as_of, truth
+
+
+def incident_accuracy(repo):
+    """Per linking method: how often the true culprit was blamed, and how many commits were blamed."""
+    from .gitio import log_commits
+    from .incidents import incident_events
+
+    path = Path(repo) / ".ledger" / "oracle_incidents.json"
+    if not path.exists():
+        return None
+    truth = json.loads(path.read_text())
+    commits = log_commits(repo)
+    con = connect(Path(repo) / ".ledger" / "ledger.db")
+    out = {}
+    for mode in ("off", "latest", "window"):
+        events, _ = incident_events(repo, con, commits, {c["sha"] for c in commits}, mode)
+        blamed = {}
+        for sha, evs in events.items():
+            for _, _, incident_id in evs:
+                blamed.setdefault(incident_id, set()).add(sha)
+        for method in ("suspect", "deploy"):
+            ids = [i for i, t in truth.items() if t["method"] == method]
+            if not ids or (method == "suspect" and mode != "off"):
+                continue  # suspect links don't depend on the deploy mode
+            hits = sum(truth[i]["culprit"] in blamed.get(i, set()) for i in ids)
+            sizes = [len(blamed.get(i, ())) for i in ids]
+            key = method if method == "suspect" else f"files_{mode}"
+            out[key] = {"incidents": len(ids), "culprit_found": hits / len(ids),
+                        "avg_commits_blamed": sum(sizes) / len(sizes)}
+    con.close()
+    return out
+
+
+def validate(repo, stratum="stratum", boot=200, seed=0):
+    """Compare the labeler with the oracle and re-estimate on oracle labels."""
+    from .estimate import estimate, load_rows
+
+    oracle = json.loads((Path(repo) / ".ledger" / "oracle.json").read_text())
+    con = connect(Path(repo) / ".ledger" / "ledger.db")
+    tp = fp = fn = tn = 0
+    for r in con.execute("SELECT decision_id, bad FROM outcomes WHERE mature=1 AND kind!='no_commit'"):
+        truth = oracle.get(r["decision_id"], 0)
+        if r["bad"] and truth:
+            tp += 1
+        elif r["bad"]:
+            fp += 1
+        elif truth:
+            fn += 1
+        else:
+            tn += 1
+    # Same rows, but Y replaced by the oracle: isolates estimator error from label noise.
+    rows = load_rows(con, stratum)
+    con.close()
+    for row in rows:
+        row["y"] = oracle.get(row["id"], 0)
+    oracle_est = estimate(rows, boot=boot, seed=seed)
+    result = {
+        "incident_linking": incident_accuracy(repo),
+        "tp": tp, "fp": fp, "fn": fn, "tn": tn,
+        "precision": tp / (tp + fp) if tp + fp else None,
+        "recall": tp / (tp + fn) if tp + fn else None,
+        "oracle_dr": {a: e["estimates"]["dr"] for a, e in oracle_est["arms"].items()},
+    }
+    (Path(repo) / ".ledger" / "validation.json").write_text(json.dumps(result, indent=2))
+    return result