consequence-ledger 0.3.0__py3-none-any.whl

ledger/incidents.py

@@ -0,0 +1,202 @@
+"""Production incidents as outcome events.
+
+A change can be bad without ever being reverted or "fixed" in git: it causes an
+incident that is mitigated elsewhere (a flag flip, a config rollback). Incident
+tools export the link to code in one of two ways, and both are supported:
+
+  suspect commits -- the incident names commits (Sentry "suspect commits",
+                     a postmortem's root-cause SHA)
+  deploy window   -- the incident started after a deploy; every commit shipped
+                     in that deploy (since the previous one) is a suspect,
+                     narrowed to commits touching the incident's files if given
+
+Imports are CSV or JSON. Incident fields: id, started_at (ISO 8601 or unix
+seconds), title, commits (SHAs or prefixes, separated by spaces, commas or
+semicolons), files (paths, same separators). Deploy fields: sha, deployed_at.
+"""
+import csv
+import datetime as dt
+import json
+import re
+from pathlib import Path
+
+from .gitio import git
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS incidents(
+    id      TEXT PRIMARY KEY,
+    ts      INTEGER NOT NULL,
+    title   TEXT,
+    commits TEXT NOT NULL DEFAULT '[]',   -- JSON list of SHAs/prefixes as exported
+    files   TEXT NOT NULL DEFAULT '[]'    -- JSON list of paths as exported
+);
+CREATE TABLE IF NOT EXISTS deploys(
+    sha TEXT NOT NULL,
+    ts  INTEGER NOT NULL,
+    PRIMARY KEY (sha, ts)
+);
+"""
+SPLIT_RE = re.compile(r"[\s,;]+")
+DEPLOY_WINDOW_DAYS = 14  # an incident more than this long after the last deploy is not blamed on it
+
+
+def ensure_schema(con):
+    con.executescript(SCHEMA)
+
+
+def parse_time(value):
+    value = str(value).strip()
+    if re.fullmatch(r"\d+(\.\d+)?", value):
+        return int(float(value))
+    d = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
+    if d.tzinfo is None:
+        d = d.replace(tzinfo=dt.timezone.utc)
+    return int(d.timestamp())
+
+
+def split_list(value):
+    if value is None:
+        return []
+    if isinstance(value, list):
+        return [str(v).strip() for v in value if str(v).strip()]
+    return [v for v in SPLIT_RE.split(str(value)) if v]
+
+
+def read_records(path):
+    path = Path(path)
+    text = path.read_text(encoding="utf-8-sig")
+    if path.suffix.lower() == ".json":
+        data = json.loads(text)
+        return data if isinstance(data, list) else data.get("incidents") or data.get("deploys") or []
+    return list(csv.DictReader(text.splitlines()))
+
+
+def first_field(record, *names):
+    for n in names:
+        if record.get(n) not in (None, ""):
+            return record[n]
+    return None
+
+
+def import_incidents(con, path):
+    return import_incidents_records(con, read_records(path))
+
+
+def import_incidents_records(con, records):
+    """Insert incident records. A record with no suspect commits or files never wipes ones already stored
+    for the same id, so a live sync can refresh timing without undoing links from an earlier export."""
+    ensure_schema(con)
+    count = 0
+    for r in records:
+        incident_id = first_field(r, "id", "incident_id", "number", "key")
+        started = first_field(r, "started_at", "created_at", "start", "triggered_at", "ts")
+        if incident_id is None or started is None:
+            raise ValueError(f"incident record needs an id and started_at: {r}")
+        commits = split_list(first_field(r, "commits", "suspect_commits", "sha"))
+        files = split_list(first_field(r, "files", "paths"))
+        old = con.execute("SELECT commits, files FROM incidents WHERE id = ?", (str(incident_id),)).fetchone()
+        if old:
+            commits = commits or json.loads(old["commits"])
+            files = files or json.loads(old["files"])
+        con.execute("INSERT OR REPLACE INTO incidents VALUES (?,?,?,?,?)",
+                    (str(incident_id), parse_time(started), first_field(r, "title", "summary", "name"),
+                     json.dumps(commits), json.dumps(files)))
+        count += 1
+    con.commit()
+    return count
+
+
+def import_deploys(con, path):
+    ensure_schema(con)
+    count = 0
+    for r in read_records(path):
+        sha = first_field(r, "sha", "commit", "revision", "version")
+        when = first_field(r, "deployed_at", "finished_at", "created_at", "ts")
+        if sha is None or when is None:
+            raise ValueError(f"deploy record needs a sha and deployed_at: {r}")
+        con.execute("INSERT OR IGNORE INTO deploys VALUES (?,?)", (str(sha).strip(), parse_time(when)))
+        count += 1
+    con.commit()
+    return count
+
+
+def files_by_commit(repo):
+    """{sha: set of paths it touched}, for all of history in one git call."""
+    out = git(repo, "log", "--all", "--no-renames", "--name-only", "--format=%x1e%H")
+    touched = {}
+    for record in out.split("\x1e"):
+        parts = [ln.strip() for ln in record.strip().splitlines() if ln.strip()]
+        if parts:
+            touched[parts[0]] = set(parts[1:])
+    return touched
+
+
+def incident_events(repo, con, commits, targets, deploy_mode="off"):
+    """{sha: [(incident_ts, 'incident', incident_id)]} for target commits linked to an incident.
+
+    deploy_mode controls incidents that name no suspect commit:
+      off    -- leave them unlinked (default). A miss can only understate a difference between arms.
+      latest -- blame the latest deploy in the window (narrowed to the incident's files if given)
+      window -- blame every commit in the window that touched the incident's files
+    In a synthetic repo where each file changes every few days, file-level linking put the true culprit
+    among the blamed commits only 6-13% of the time (research/FINDINGS.md), so it is opt-in.
+    """
+    if deploy_mode not in ("off", "latest", "window"):
+        raise ValueError("deploy_mode must be 'off', 'latest' or 'window'")
+    ensure_schema(con)
+    from .label import resolve_prefix  # local import: label imports this module
+
+    sorted_shas = sorted(c["sha"] for c in commits)
+    known = set(sorted_shas)
+    deploys = []
+    for row in con.execute("SELECT sha, ts FROM deploys ORDER BY ts"):
+        full = resolve_prefix(sorted_shas, row["sha"].lower())
+        if full:
+            deploys.append((row["ts"], full))
+
+    events = {}
+    links = {"suspect": 0, "deploy": 0, "unlinked": 0}
+    touched = None
+    shipped_cache = {}
+    for inc in con.execute("SELECT * FROM incidents ORDER BY ts"):
+        suspects = {resolve_prefix(sorted_shas, s.lower()) for s in json.loads(inc["commits"])} - {None}
+        how = "suspect"
+        if not suspects and deploy_mode != "off":
+            how = "deploy"
+            window_start = inc["ts"] - DEPLOY_WINDOW_DAYS * 86400
+            before = [d for d in deploys if d[0] <= inc["ts"]]
+            files = set(json.loads(inc["files"]))
+            # Walk back from the latest deploy. Without files, only the latest deploy is suspect.
+            # With files: "latest" blames the most recent deploy in the window that changed one of
+            # them; "window" blames every commit in the window that did (an incident can surface
+            # days after the deploy that caused it).
+            for i in range(len(before) - 1, -1, -1):
+                shipped_ts, shipped = before[i]
+                if shipped_ts < window_start:
+                    break
+                earlier = [d for d in before[:i] if d[1] != shipped]
+                if earlier:
+                    args = (f"{earlier[-1][1]}..{shipped}",)
+                else:
+                    # First deploy on record: only commits from the window before it, not all history.
+                    args = (f"--since={window_start}", shipped)
+                if args not in shipped_cache:
+                    shipped_cache[args] = set(git(repo, "rev-list", "--no-merges", *args).split()) & known
+                shipped_commits = shipped_cache[args]
+                if not files:
+                    suspects = shipped_commits
+                    break
+                if touched is None:
+                    touched = files_by_commit(repo)
+                matched = {s for s in shipped_commits if touched.get(s, set()) & files}
+                suspects |= matched
+                if matched and deploy_mode == "latest":
+                    break
+        hit = suspects & targets
+        if hit:
+            links[how] += 1
+        else:
+            links["unlinked"] += 1
+        for sha in hit:
+            events.setdefault(sha, []).append((inc["ts"], "incident", inc["id"]))
+    return events, links

ledger/label.py

@@ -0,0 +1,386 @@
+"""Attribute commits to decisions and label each decision's delayed outcome.
+
+Outcome signals, most precise first:
+  revert -- a later commit says "This reverts commit <sha>" (git's default message)
+  fix    -- a later fix/hotfix commit edits lines this commit introduced (SZZ via git blame)
+
+A decision is "bad" if either event lands within the horizon after its first
+commit, and "mature" only once the whole horizon has elapsed. Estimators use
+mature decisions only, so recent decisions can't look clean just because
+their bugs haven't surfaced yet.
+"""
+import bisect
+import hashlib
+import json
+import os
+import re
+import time
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor
+
+from .gitio import GitError, git, log_commits
+from .squash import recover_squashed
+
+TRAILER_RE = re.compile(r"^Ledger-Decision-Id:\s*(\S+)\s*$", re.M)
+REVERT_RE = re.compile(r"This reverts commit ([0-9a-f]{7,40})")
+REVERT_SUBJECT_RE = re.compile(r'^Revert "(.+)"(?: \(#\d+\))?$')
+SQUASH_PR_RE = re.compile(r"\(#(\d+)\)\s*$")
+MERGE_PR_RE = re.compile(r"^Merge pull request #(\d+)\b")
+STRONG_REF_RE = re.compile(
+    r"\b(?:(?:introduced|caused|broken)\s+(?:in|by)|regress(?:ed|ion)\s+(?:in|from|by|of))\s+#(\d+)", re.I)
+FOLLOWUP_REF_RE = re.compile(r"\bfollow[- ]?up\s+(?:to|for|on)\s+#(\d+)", re.I)
+FIX_RE = re.compile(r"^(fix|hotfix)(\([^)]*\))?!?:", re.I)
+# How fix commits are spotted. "conventional" trusts fix:/hotfix: prefixes (precise, needs the convention);
+# "keywords" is the classic SZZ heuristic for repos that write "Fix crash in ..." (more recall, more noise).
+FIX_STYLES = {
+    "conventional": FIX_RE,
+    "keywords": re.compile(r"\b(fix(es|ed)?|bug(fix)?|hotfix|regression|crash(es)?)\b", re.I),
+}
+HUNK_RE = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@")
+# Files whose edits fix no product code: tests, docs, editor/CI config, lockfiles.
+SKIP_PATHS = re.compile(
+    r"(^|/)(tests?|__tests__|spec|specs|docs?|\.vscode|\.idea|\.github)/"
+    r"|(_test|\.test|_spec|\.spec|Test|Tests)\.[^/]+$|(^|/)test_[^/]+$"
+    r"|\.(md|rst|txt|lock)$|(^|/)(package-lock\.json|yarn\.lock|pnpm-lock\.yaml|go\.sum)$")
+BLAME_RE = re.compile(r"^([0-9a-f]{40}) \d+ \d+")
+
+DAY = 86400
+
+
+def attribute(con, commits):
+    """Map commits to decisions using their Ledger-Decision-Id trailer."""
+    known = {row["id"] for row in con.execute("SELECT id FROM decisions")}
+    found = unknown = 0
+    for c in commits:
+        for decision_id in TRAILER_RE.findall(c["body"]):
+            if decision_id not in known:
+                unknown += 1
+                continue
+            cur = con.execute("INSERT OR IGNORE INTO commits VALUES (?,?,?,'trailer')",
+                              (c["sha"], decision_id, c["ts"]))
+            found += cur.rowcount
+    return {"attributed": found, "unknown_trailers": unknown}
+
+
+def resolve_prefix(sorted_shas, prefix):
+    """The unique full SHA starting with prefix, or None if absent or ambiguous."""
+    i = bisect.bisect_left(sorted_shas, prefix)
+    hits = sorted_shas[i:i + 2]
+    if hits and hits[0].startswith(prefix) and not (len(hits) > 1 and hits[1].startswith(prefix)):
+        return hits[0]
+    return None
+
+
+def find_reverts(commits):
+    """{reverted_sha: (revert_ts, revert_sha)}, keeping the earliest revert.
+
+    Matches git's "This reverts commit <sha>" and, for GitHub revert PRs that were
+    squash-merged (no SHA, just 'Revert "<original subject>"'), the most recent
+    earlier commit with exactly that subject.
+    """
+    sorted_shas = sorted(c["sha"] for c in commits)
+    by_subject = defaultdict(list)  # subject -> [(ts, sha)] in history order
+    reverts = {}
+
+    def record(target, c):
+        if target and target != c["sha"] and (target not in reverts or c["ts"] < reverts[target][0]):
+            reverts[target] = (c["ts"], c["sha"])
+
+    for c in commits:
+        prefixes = REVERT_RE.findall(c["body"])
+        for prefix in prefixes:
+            record(resolve_prefix(sorted_shas, prefix), c)
+        m = REVERT_SUBJECT_RE.match(c["subject"])
+        if not prefixes and m:
+            earlier = [sha for ts, sha in by_subject.get(m.group(1), []) if ts <= c["ts"]]
+            if earlier:
+                record(earlier[-1], c)
+        by_subject[c["subject"]].append((c["ts"], c["sha"]))
+    return reverts
+
+
+def is_fix(commit, fix_re=FIX_RE):
+    """A bug-fix commit by its message. Merge commits never count (MA-SZZ): "Merge pull request #12
+    from me/fix-crash" would otherwise blame every line its whole branch touched; the branch's own
+    commits are judged on their own."""
+    return (bool(fix_re.search(commit["subject"])) and len(commit.get("parents", ())) < 2
+            and not REVERT_RE.search(commit["body"]) and not REVERT_SUBJECT_RE.match(commit["subject"]))
+
+
+def is_trivial(text):
+    """Lines whose removal says nothing about a bug: blank, punctuation-only, or comment-only."""
+    s = text.strip()
+    return (not any(ch.isalnum() for ch in s)
+            or s.startswith(("//", "/*", "*", "# ", "<!--")) or s == "#")
+
+
+def deleted_lines(diff):
+    """Parse a -U0 diff into ({path: [(old_lineno, text)]}, {stripped text of every added line})."""
+    deleted = defaultdict(list)
+    added = set()
+    path = None
+    old_no = old_left = new_left = 0
+    for line in diff.splitlines():
+        if old_left or new_left:
+            # Inside a hunk; counting lines keeps "--- x" content apart from file headers.
+            if line.startswith("-") and old_left:
+                if path:
+                    deleted[path].append((old_no, line[1:]))
+                old_no += 1
+                old_left -= 1
+            elif line.startswith("+") and new_left:
+                added.add(line[1:].strip())
+                new_left -= 1
+            continue
+        if line.startswith("--- "):
+            path = line[6:] if line.startswith("--- a/") else None
+        elif line.startswith("@@"):
+            m = HUNK_RE.match(line)
+            if not m:
+                continue   # a malformed header; git never writes one, but a bad diff must not crash labelling
+            old_no, old_left = int(m.group(1)), int(m.group(2) or 1)
+            new_left = int(m.group(4) or 1)
+    return deleted, added
+
+
+def to_ranges(numbers):
+    ranges = []
+    for n in sorted(numbers):
+        if ranges and n == ranges[-1][0] + ranges[-1][1]:
+            ranges[-1][1] += 1
+        else:
+            ranges.append([n, 1])
+    return ranges
+
+
+def szz(repo, commit, ignore=frozenset(), skip_paths=SKIP_PATHS):
+    """Commits whose lines this fix commit deleted or changed (SZZ, via blame).
+
+    Refinements over plain SZZ, each a documented source of false positives:
+    whitespace-only edits (diff -w), blank/comment-only lines, lines that were moved
+    rather than changed (deleted here but added elsewhere in the same commit), and
+    changes to tests, docs and editor config, which fix no product code.
+    """
+    if not commit["parents"]:
+        return set()
+    parent = commit["parents"][0]
+    diff = git(repo, "diff", "-U0", "-w", "--no-color", "--no-renames", "--no-ext-diff",
+               parent, commit["sha"])
+    deleted, added = deleted_lines(diff)
+    ranges = {}
+    for path, lines in deleted.items():
+        if skip_paths and skip_paths.search(path):
+            continue
+        keep = [n for n, text in lines if not is_trivial(text) and text.strip() not in added]
+        if keep:
+            ranges[path] = to_ranges(keep)
+    introducers = set()
+    for path, spans in ranges.items():
+        args = ["blame", "--porcelain", "-w"]
+        for rev in sorted(ignore):
+            # Blame looks through formatting/refactor commits to the real author of the line.
+            args += ["--ignore-rev", rev]
+        for start, count in spans:
+            args += ["-L", f"{start},+{count}"]
+        try:
+            out = git(repo, *args, parent, "--", path)
+        except GitError:
+            continue  # e.g. a path that only exists in a submodule
+        for line in out.splitlines():
+            m = BLAME_RE.match(line)
+            if m:
+                introducers.add(m.group(1))
+    return introducers
+
+
+SZZ_VERSION = 2  # bump whenever szz() would return something different for the same commit
+# A fix that blames more commits than this is too diffuse to attribute: in a study of SZZ variants, fixes
+# blaming over 20 commits were outliers (3 fixes produced 72 suspects, 3 correct) and a cutoff of 10 raised
+# precision further (Rosa et al., ICSE 2021, arXiv 2102.03300). 0 turns the filter off.
+DEFAULT_MAX_BLAMED = 10
+SZZ_WORKERS = min(8, (os.cpu_count() or 2) * 2)
+
+
+class SzzCache:
+    """Remembers, per fix commit, which commits its changed lines came from.
+
+    That answer depends only on the fix commit, the history behind it, and how szz() is configured, and
+    git history is immutable (a rewritten commit has a new sha), so it never goes stale. Relabelling
+    then costs one `git log` plus new fix commits, instead of a diff and blame for every fix every time.
+    The key covers the algorithm version, the skip-path rules and the ignore list; entries made under a
+    different configuration are dropped.
+    """
+
+    def __init__(self, con, ignore):
+        text = f"{SZZ_VERSION}|{SKIP_PATHS.pattern}|{' '.join(sorted(ignore))}"
+        self.con = con
+        self.key = hashlib.sha1(text.encode("utf-8")).hexdigest()
+        self.hits = self.misses = 0
+        con.execute("DELETE FROM szz_cache WHERE key != ?", (self.key,))
+
+    def get(self, fix_sha):
+        row = self.con.execute("SELECT introducers FROM szz_cache WHERE fix_sha = ? AND key = ?",
+                               (fix_sha, self.key)).fetchone()
+        if row is None:
+            self.misses += 1
+            return None
+        self.hits += 1
+        return set(json.loads(row[0]))
+
+    def put(self, fix_sha, introducers):
+        self.con.execute("INSERT OR REPLACE INTO szz_cache VALUES (?,?,?)",
+                         (fix_sha, self.key, json.dumps(sorted(introducers))))
+
+
+def outcome_events(repo, commits, targets, ignore_revs=(), fix_re=FIX_RE, cache_con=None, stats=None,
+                   max_blamed=DEFAULT_MAX_BLAMED):
+    """{sha: [(ts, 'revert'|'fix', event_sha)]} for each target commit that was later reverted or fixed.
+
+    With cache_con (a ledger connection) fix-commit analysis is remembered between runs; `stats`, if
+    given, receives how many were reused and how many computed.
+    """
+    by_sha = {c["sha"]: c for c in commits}
+    events = defaultdict(list)
+    for target, (ts, sha) in find_reverts(commits).items():
+        if target in targets:
+            events[target].append((ts, "revert", sha))
+    # Only full SHAs present in history; unknown revs would make git blame fail.
+    ignore = frozenset(s for r in ignore_revs for s in by_sha if s.startswith(r.lower())) if ignore_revs else frozenset()
+    cache = SzzCache(cache_con, ignore) if cache_con is not None else None
+    first_ts = min((by_sha[s]["ts"] for s in targets if s in by_sha), default=None)
+    diffuse = 0
+    fixes = [f for f in commits if first_ts is not None and f["ts"] >= first_ts and is_fix(f, fix_re)]
+    found = {f["sha"]: cache.get(f["sha"]) for f in fixes} if cache else {}
+    todo = [f for f in fixes if found.get(f["sha"]) is None]
+    # Each analysis is a few git subprocesses that spend their time outside Python, so threads overlap them.
+    with ThreadPoolExecutor(max_workers=min(SZZ_WORKERS, len(todo)) or 1) as pool:
+        for fix, introducers in zip(todo, pool.map(lambda f: szz(repo, f, ignore), todo)):
+            found[fix["sha"]] = introducers
+            if cache:
+                cache.put(fix["sha"], introducers)
+    for fix in fixes:
+        introducers = found[fix["sha"]]
+        if max_blamed and len(introducers) > max_blamed:
+            diffuse += 1   # applied after the cache so changing the threshold never needs a recompute
+            continue
+        for intro in introducers & targets:
+            events[intro].append((fix["ts"], "fix", fix["sha"]))
+    if stats is not None:
+        stats.update(szz_reused=cache.hits if cache else 0, szz_computed=cache.misses if cache else 0,
+                     fixes_too_diffuse=diffuse,
+                     # Fixes that blame nothing: addition-only, or touching only tests and docs. Mostly not product
+                     # bugs (5 of 10 on two local repos were notes or tests that mention a fix), but a high share
+                     # is also where recall is lost, so it is reported rather than hidden.
+                     fixes_untraceable=sum(1 for f in fixes if not found[f["sha"]]))
+    for target, (ts, sha) in find_pr_references(repo, commits, fix_re).items():
+        if target in targets:
+            events[target].append((ts, "fix", sha))
+    return events
+
+
+def find_pr_references(repo, commits, fix_re=FIX_RE):
+    """{target_sha: (ts, fixing_sha)} from messages that name the PR they repair.
+
+    "Bug introduced in #1507", "caused by #12", "regression from #88" count on any commit;
+    "follow-up to #12" only on a commit that is itself a fix. #N resolves to the squash-merged
+    commit whose subject ends in "(#N)", or to the commits a "Merge pull request #N" brought in.
+    """
+    squashed, merged = {}, {}
+    for c in commits:
+        m = SQUASH_PR_RE.search(c["subject"])
+        if m:
+            squashed.setdefault(m.group(1), c)
+        m = MERGE_PR_RE.match(c["subject"])
+        if m and len(c["parents"]) == 2:
+            merged.setdefault(m.group(1), c)
+    found = {}
+    for c in commits:
+        numbers = set(STRONG_REF_RE.findall(c["body"]))
+        if is_fix(c, fix_re):
+            numbers |= set(FOLLOWUP_REF_RE.findall(c["body"]))
+        for n in numbers:
+            if n in squashed:
+                targets = [squashed[n]["sha"]]
+            elif n in merged:
+                mc = merged[n]
+                targets = git(repo, "rev-list", "--no-merges", f"{mc['parents'][0]}..{mc['parents'][1]}").split()
+            else:
+                continue
+            for t in targets:
+                if t != c["sha"] and (t not in found or c["ts"] < found[t][0]):
+                    found[t] = (c["ts"], c["sha"])
+    return found
+
+
+def first_bad(events, shas, start, horizon):
+    """Earliest revert/fix of any of shas within horizon seconds of start, else None."""
+    hits = sorted(e for s in shas for e in events.get(s, []) if e[0] >= start)
+    return hits[0] if hits and hits[0][0] - start <= horizon else None
+
+
+OUTCOME_COLUMNS = ("decision_id", "bad", "mature", "kind", "event_sha", "delay_days", "horizon_days", "as_of",
+                   "early_bad", "early_mature", "early_days")
+
+
+def label(repo, con, horizon_days=30, as_of=None, rev="HEAD", ignore_revs=(), fix_re=FIX_RE, early_days=7,
+          incident_linking="off", max_blamed=DEFAULT_MAX_BLAMED):
+    """Attribute, detect reverts, fixes and incidents, and write one outcome row per decision.
+
+    Each decision also gets an early label (bad within early_days), so a first readout is
+    possible a week in; see estimate.early_readout for how far to trust it.
+    """
+    from .incidents import incident_events
+
+    as_of = int(as_of if as_of is not None else time.time())
+    if early_days and early_days >= horizon_days:
+        raise ValueError(f"the early window ({early_days} days) must be shorter than the horizon ({horizon_days} days): "
+                         "lower --early-days, or pass --early-days 0 to turn the early readout off")
+    commits = log_commits(repo, rev)
+    stats = attribute(con, commits)
+    stats["squash_recovered"] = recover_squashed(con, repo, rev, commits, TRAILER_RE)
+
+    by_sha = {c["sha"]: c for c in commits}
+    owned = defaultdict(list)  # decision_id -> [sha]
+    for row in con.execute("SELECT sha, decision_id FROM commits"):
+        if row["sha"] in by_sha:
+            owned[row["decision_id"]].append(row["sha"])
+    attributed = {sha for shas in owned.values() for sha in shas}
+
+    szz_stats = {}
+    events = outcome_events(repo, commits, attributed, ignore_revs, fix_re, cache_con=con, stats=szz_stats,
+                            max_blamed=max_blamed)
+    inc_events, inc_links = incident_events(repo, con, commits, attributed, incident_linking)
+    for sha, evs in inc_events.items():
+        events[sha].extend(evs)
+
+    horizon, early = horizon_days * DAY, (early_days or 0) * DAY
+    counts = {"mature": 0, "immature": 0, "revert": 0, "fix": 0, "incident": 0, "clean": 0, "no_commit": 0}
+    insert = f"INSERT INTO outcomes ({', '.join(OUTCOME_COLUMNS)}) VALUES ({', '.join('?' * len(OUTCOME_COLUMNS))})"
+    con.execute("DELETE FROM outcomes")
+    for row in con.execute("SELECT id, ts FROM decisions").fetchall():
+        shas = owned.get(row["id"])
+        if not shas:
+            con.execute(insert, (row["id"], 0, 0, "no_commit", None, None, horizon_days, as_of, 0, 0, early_days))
+            counts["no_commit"] += 1
+            continue
+        start = min(by_sha[s]["ts"] for s in shas)
+        mature = int(start + horizon <= as_of)
+        early_mature = int(bool(early) and start + early <= as_of)
+        hit = first_bad(events, shas, start, horizon)
+        early_bad = int(bool(hit) and bool(early) and hit[0] - start <= early)
+        if hit:
+            ts, kind, sha = hit
+            con.execute(insert, (row["id"], 1, mature, kind, sha, (ts - start) / DAY, horizon_days, as_of,
+                                 early_bad, early_mature, early_days))
+        else:
+            kind = "clean"
+            con.execute(insert, (row["id"], 0, mature, kind, None, None, horizon_days, as_of,
+                                 0, early_mature, early_days))
+        # Event counts cover mature decisions only, matching what the estimators see.
+        if mature:
+            counts[kind] += 1
+        counts["mature" if mature else "immature"] += 1
+    con.commit()
+    return {**stats, "commits_scanned": len(commits), "fix_commits": sum(is_fix(c, fix_re) for c in commits),
+            "incident_links": inc_links, **szz_stats, "horizon_days": horizon_days, "early_days": early_days,
+            "as_of": as_of, **counts}