consequence-ledger 0.3.0__py3-none-any.whl

ledger/segments.py

@@ -0,0 +1,142 @@
+"""Which tool is best for which kind of task, and what routing by that would be worth.
+
+Both questions invite self-deception, so both are answered conservatively:
+
+  breakdown  per-stratum effects are noisy, and the largest of several noisy numbers is biased upwards. Raw
+             effects are shown beside empirical-Bayes shrunk ones (each pulled toward the overall effect in
+             proportion to its noise) and a Bonferroni-adjusted p-value. In simulation, shrinkage cut the
+             error of per-stratum effects by 33% to 82%, and raw tests raised about 0.45 false alarms per
+             8-stratum report.
+  recommend  "send each task type to its best tool" always looks good on the data that picked the tool. In a
+             world where the tools were identical it claimed gains of 1.4 to 3.0 percentage points on the same
+             data and 0.0 when the tool was chosen on one half and judged on the other. Here the choice and the
+             judging never share a decision.
+"""
+import math
+from collections import defaultdict
+
+from .estimate import _counts, _mean_se, contributions, fold_of, floor_se
+
+Z = 1.959964
+MIN_CELL = 10   # decisions an arm needs within a stratum before it can win that stratum
+
+
+def _p_two_sided(z):
+    return math.erfc(abs(z) / math.sqrt(2))
+
+
+def _by_stratum(rows):
+    groups = defaultdict(list)
+    for i, r in enumerate(rows):
+        groups[r["s"]].append(i)
+    return groups
+
+
+def shrink(effects, ses):
+    """Empirical-Bayes shrinkage (DerSimonian-Laird between-stratum variance). Returns (mean, tau2, shrunk, post_sd)."""
+    k = len(effects)
+    w = [1 / s ** 2 for s in ses]
+    mu = sum(wi * e for wi, e in zip(w, effects)) / sum(w)
+    if k < 2:
+        return mu, 0.0, list(effects), list(ses)   # one stratum: nothing to borrow from
+    q = sum(wi * (e - mu) ** 2 for wi, e in zip(w, effects))
+    c = sum(w) - sum(wi * wi for wi in w) / sum(w)
+    tau2 = max(0.0, (q - (k - 1)) / c) if c > 0 else 0.0
+    shrunk, post = [], []
+    for e, s in zip(effects, ses):
+        b = tau2 / (tau2 + s * s) if tau2 + s * s > 0 else 0.0
+        shrunk.append(mu + b * (e - mu))
+        # Posterior spread: noise left in the shrunk part, plus uncertainty about the overall mean it shrinks toward.
+        post.append(math.sqrt(b * s * s + (1 - b) ** 2 / sum(w)))
+    return mu, tau2, shrunk, post
+
+
+def breakdown(rows, arms, baseline, clip=None, folds=None):
+    """Per-stratum effect of every non-baseline arm against the baseline, raw and shrunk."""
+    c = contributions(rows, arms, clip, folds)
+    groups = _by_stratum(rows)
+    out = {}
+    for arm in arms:
+        if arm == baseline:
+            continue
+        cells = []
+        for stratum, idx in sorted(groups.items()):
+            diff = [c[arm]["dr"][i] - c[baseline]["dr"][i] for i in idx]
+            effect, se = _mean_se(diff) if len(idx) > 1 else (diff[0], float("nan"))
+            counts = _counts([rows[i] for i in idx], [arm, baseline])
+            se = max(se if se == se else 0.0, floor_se(counts[arm], counts[baseline]))
+            if se <= 0:
+                continue
+            cells.append({"stratum": stratum, "n": len(idx), "effect": effect, "se": se,
+                          "arm_rate": sum(c[arm]["dr"][i] for i in idx) / len(idx),
+                          "baseline_rate": sum(c[baseline]["dr"][i] for i in idx) / len(idx)})
+        if not cells:
+            continue
+        mu, tau2, shrunk, post = shrink([x["effect"] for x in cells], [x["se"] for x in cells])
+        m = len(cells)
+        for cell, sh, ps in zip(cells, shrunk, post):
+            cell.update(shrunk=sh, shrunk_lo=sh - Z * ps, shrunk_hi=sh + Z * ps,
+                        raw_lo=cell["effect"] - Z * cell["se"], raw_hi=cell["effect"] + Z * cell["se"],
+                        p=_p_two_sided(cell["effect"] / cell["se"]))
+            cell["p_adjusted"] = min(1.0, cell["p"] * m)
+        out[arm] = {"overall": mu, "tau": math.sqrt(tau2), "strata": cells,
+                    "heterogeneous": bool(tau2 > 0 and any(x["p_adjusted"] < 0.05 for x in cells))}
+    return out
+
+
+def _policy_from(rows, idx, c, arms, baseline):
+    """Arm per stratum chosen from the rows in idx only: lowest estimated bad rate among arms with enough data."""
+    groups = defaultdict(list)
+    for i in idx:
+        groups[rows[i]["s"]].append(i)
+    policy = {}
+    for stratum, members in groups.items():
+        best, best_rate = baseline, None
+        for arm in arms:
+            taken = sum(1 for i in members if rows[i]["arm"] == arm)
+            if taken < MIN_CELL:
+                continue
+            rate = sum(c[arm]["dr"][i] for i in members) / len(members)
+            if best_rate is None or rate < best_rate:
+                best, best_rate = arm, rate
+        policy[stratum] = best
+    return policy
+
+
+def recommend(rows, arms, baseline, clip=None, folds=None, rounds=2):
+    """Cross-fitted value of routing each stratum to its best arm, against what actually happened.
+
+    Rows are split by a stable hash of their cluster (or id). The policy for one half is chosen from the
+    other half, so no decision both picks the winner and judges it. gain > 0 means fewer bad outcomes.
+    """
+    c = contributions(rows, arms, clip, folds)
+    side = [fold_of(r, rounds, i) for i, r in enumerate(rows)]
+    gains, single_gains = [], []
+    policies = {}
+    for k in range(rounds):
+        train = [i for i, s in enumerate(side) if s != k]
+        test = [i for i, s in enumerate(side) if s == k]
+        if not train or not test:
+            continue
+        policy = _policy_from(rows, train, c, arms, baseline)
+        best_single = min(arms, key=lambda a: (sum(c[a]["dr"][i] for i in train) / len(train)))
+        for i in test:
+            chosen = policy.get(rows[i]["s"], baseline)
+            gains.append(rows[i]["y"] - c[chosen]["dr"][i])
+            single_gains.append(rows[i]["y"] - c[best_single]["dr"][i])
+        for stratum, arm in policy.items():
+            policies.setdefault(stratum, []).append(arm)
+    if not gains:
+        raise ValueError("not enough decisions to split into a choosing half and a judging half")
+    gain, gain_se = _mean_se(gains)
+    single, single_se = _mean_se(single_gains)
+    agree = {s: arms_[0] for s, arms_ in policies.items() if len(set(arms_)) == 1}
+    return {
+        "n": len(gains),
+        "current_rate": sum(r["y"] for r in rows) / len(rows),
+        "gain": gain, "gain_lo": gain - Z * gain_se, "gain_hi": gain + Z * gain_se,
+        "best_single_gain": single, "best_single_lo": single - Z * single_se, "best_single_hi": single + Z * single_se,
+        "stable_policy": agree,
+        "unstable_strata": sorted(set(policies) - set(agree)),
+        "worth_it": gain - Z * gain_se > 0,
+    }

ledger/sentry.py

@@ -0,0 +1,91 @@
+"""Pull incidents from Sentry's REST API into the ledger.
+
+Reads the token from the SENTRY_AUTH_TOKEN environment variable only; it is never accepted as an
+argument, written to disk, or printed. Needs a read-only token (Issue & Event: Read).
+
+Sentry's documented API has no suspect-commit endpoint, so this brings in incident timing and titles
+but no commit links. Link them with `ledger import incidents` from an export that names suspect
+commits, or with --incident-linking on `ledger label`.
+"""
+import json
+import os
+import re
+import urllib.error
+import urllib.parse
+import urllib.request
+
+from .incidents import import_incidents_records
+
+BASE_URL = "https://sentry.io"
+LINK_NEXT_RE = re.compile(r'<([^>]+)>;\s*rel="next";\s*results="true"')
+MAX_PAGES = 50  # 100 issues each: a hard stop against runaway pagination
+
+
+class SentryError(RuntimeError):
+    pass
+
+
+def token_from_env():
+    token = os.environ.get("SENTRY_AUTH_TOKEN", "").strip()
+    if not token:
+        raise SentryError("SENTRY_AUTH_TOKEN is not set. Create a read-only token and store it with: "
+                          'setx SENTRY_AUTH_TOKEN "<token>" (then open a new terminal).')
+    return token
+
+
+def _get(url, token, opener=urllib.request.urlopen):
+    req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}", "Accept": "application/json"})
+    try:
+        with opener(req, timeout=30) as resp:
+            return json.load(resp), resp.headers.get("Link", "")
+    except urllib.error.HTTPError as e:
+        # Never echo request headers: the message carries the status and Sentry's own detail only.
+        detail = ""
+        try:
+            detail = json.load(e).get("detail", "")
+        except Exception:
+            pass
+        finally:
+            e.close()
+        hint = {401: "the token was rejected (wrong, expired or revoked)",
+                403: "the token lacks permission (needs Issue & Event: Read)",
+                404: "organization not found (check the slug and SENTRY_URL region)",
+                429: "rate limited by Sentry; try again later"}.get(e.code, "")
+        raise SentryError(f"Sentry returned HTTP {e.code}: {hint or detail or e.reason}") from None
+    except urllib.error.URLError as e:
+        raise SentryError(f"could not reach Sentry: {e.reason}") from None
+
+
+def fetch_issues(org, days=90, query="", token=None, base_url=None, opener=urllib.request.urlopen):
+    """All issues first seen in the last `days` days, following Sentry's cursor pagination."""
+    token = token or token_from_env()
+    base = (base_url or os.environ.get("SENTRY_URL") or BASE_URL).rstrip("/")
+    params = urllib.parse.urlencode({"statsPeriod": f"{int(days)}d", "query": query, "limit": 100})
+    url = f"{base}/api/0/organizations/{urllib.parse.quote(org, safe='')}/issues/?{params}"
+    issues = []
+    for _ in range(MAX_PAGES):
+        page, link = _get(url, token, opener)
+        issues.extend(page)
+        nxt = LINK_NEXT_RE.search(link or "")
+        if not nxt:
+            return issues
+        url = nxt.group(1)
+    raise SentryError(f"stopped after {MAX_PAGES} pages; narrow --days or --query")
+
+
+def issues_to_incidents(issues, levels=("error", "fatal")):
+    """Incident records for import. Only error-level issues count; info and warnings are noise."""
+    out = []
+    for i in issues:
+        if levels and str(i.get("level", "")).lower() not in levels:
+            continue
+        out.append({"id": i.get("shortId") or i.get("id"), "started_at": i["firstSeen"],
+                    "title": i.get("title"), "commits": "", "files": ""})
+    return out
+
+
+def sync(con, org, days=90, query="", levels=("error", "fatal"), **kwargs):
+    issues = fetch_issues(org, days, query, **kwargs)
+    records = issues_to_incidents(issues, levels)
+    import_incidents_records(con, records)
+    return {"fetched": len(issues), "imported": len(records)}

ledger/squash.py

@@ -0,0 +1,92 @@
+"""Recover decisions whose commits were squash-merged.
+
+A squash merge replaces a branch's commits with one new commit that has no `Ledger-Decision-Id`
+trailer, so the decision looks as if it was never merged. That wrongly counts merged work as
+abandoned and biases the comparison. The branch's commits usually still exist (as a branch, a
+remote-tracking ref, or in the reflog), and a squash commit makes exactly the change the branch made.
+So: fingerprint the branch's combined diff with `git patch-id --stable` and look for a commit on the
+target branch with the same fingerprint.
+
+Only a *unique* match is accepted, and never one that predates the branch's work: two commits can share
+a fingerprint (the same one-line change made twice), and guessing would attach outcomes to the wrong
+decision. Leftovers are reported, and `ledger attach` links them by hand.
+"""
+from collections import defaultdict
+
+from .gitio import GitError, git, log_commits
+
+DIFF_ARGS = ("--no-renames", "--no-color", "--no-ext-diff")
+
+
+def patch_id(repo, diff_text):
+    """git's stable fingerprint of a diff, or None for an empty one."""
+    out = git(repo, "patch-id", "--stable", input=diff_text).split()
+    return out[0] if out else None
+
+
+def patch_ids(repo, rev_range):
+    """{commit sha: patch-id} for every non-merge commit in the range."""
+    log = git(repo, "log", "-p", "--no-merges", *DIFF_ARGS, rev_range)
+    mapping = {}
+    for line in git(repo, "patch-id", "--stable", input=log).splitlines():
+        parts = line.split()
+        if len(parts) == 2:
+            mapping[parts[1]] = parts[0]
+    return mapping
+
+
+def recover_squashed(con, repo, rev, reachable, trailer_re):
+    """Attribute squash commits to decisions that have no reachable commit. Returns how many were recovered."""
+    reachable_shas = {c["sha"] for c in reachable}
+    owned = {r[0] for r in con.execute("SELECT decision_id, sha FROM commits") if r[1] in reachable_shas}
+    pending = {r[0] for r in con.execute("SELECT id FROM decisions")} - owned
+    if not pending:
+        return 0
+
+    branch_commits = defaultdict(list)
+    for c in log_commits(repo, all_refs=True):
+        if c["sha"] in reachable_shas:
+            continue
+        for decision_id in trailer_re.findall(c["body"]):
+            if decision_id in pending:
+                branch_commits[decision_id].append(c)
+    if not branch_commits:
+        return 0
+
+    head = git(repo, "rev-parse", "--verify", rev + "^{commit}").strip()
+    wanted = {}   # decision id -> (fingerprint of its branch's combined diff, branch start time, merge base)
+    for decision_id, commits in branch_commits.items():
+        tip = max(commits, key=lambda c: c["ts"])
+        try:
+            base = git(repo, "merge-base", head, tip["sha"]).strip()
+            fingerprint = patch_id(repo, git(repo, "diff", *DIFF_ARGS, base, tip["sha"]))
+        except GitError:
+            continue                      # unrelated history, or an object git no longer has
+        if fingerprint:
+            wanted[decision_id] = (fingerprint, min(c["ts"] for c in commits), base)
+    if not wanted:
+        return 0
+
+    # One pass over everything merged since the oldest branch point; --no-merges keeps merge commits out.
+    oldest = min((int(git(repo, "log", "-1", "--format=%ct", base).strip()), base)
+                 for _, _, base in wanted.values())[1]
+    try:
+        merged = patch_ids(repo, f"{oldest}..{head}")
+    except GitError:
+        return 0
+    when = {c["sha"]: c["ts"] for c in reachable}
+    by_fingerprint = defaultdict(list)
+    for sha, fingerprint in merged.items():
+        by_fingerprint[fingerprint].append(sha)
+
+    taken = {r[0] for r in con.execute("SELECT sha FROM commits")}
+    recovered = 0
+    for decision_id, (fingerprint, started, _) in sorted(wanted.items()):
+        matches = [s for s in by_fingerprint.get(fingerprint, []) if s not in taken and when.get(s, 0) >= started]
+        if len(matches) != 1:
+            continue                      # none, or ambiguous: never guess
+        sha = matches[0]
+        con.execute("INSERT OR IGNORE INTO commits VALUES (?,?,?,'squash')", (sha, decision_id, when[sha]))
+        taken.add(sha)
+        recovered += 1
+    return recovered

ledger/status.py

@@ -0,0 +1,56 @@
+"""A plain-language health check of a ledger: is anything stuck, missing, or about to mislead you?"""
+import json
+import time
+
+DAY = 86400
+
+
+def status(con, now=None, horizon_days=30):
+    now = int(now if now is not None else time.time())
+    one = lambda sql, *a: con.execute(sql, a).fetchone()[0]
+    out = {
+        "decisions": one("SELECT COUNT(*) FROM decisions"),
+        "attributed": one("SELECT COUNT(DISTINCT decision_id) FROM commits"),
+        "incidents": one("SELECT COUNT(*) FROM incidents"),
+        "deploys": one("SELECT COUNT(*) FROM deploys"),
+        "labeled": one("SELECT COUNT(*) FROM outcomes"),
+        "mature": one("SELECT COUNT(*) FROM outcomes WHERE mature = 1 AND kind != 'no_commit'"),
+        "last_label": one("SELECT MAX(as_of) FROM outcomes"),
+        "arms": {}, "notes": []}
+    for r in con.execute("SELECT chosen arm, COUNT(*) n, MIN(propensity) lo FROM decisions GROUP BY chosen"):
+        out["arms"][r["arm"]] = {"decisions": r["n"], "lowest_propensity": r["lo"]}
+    versions = [r[0] for r in con.execute("SELECT DISTINCT policy_version FROM decisions")]
+    out["policy_versions"] = versions
+
+    notes = out["notes"]
+    if not out["decisions"]:
+        notes.append("No decisions yet. Run `ledger decide` before starting each task.")
+        return out
+    gap = out["decisions"] - out["attributed"]
+    if gap > 0.2 * out["decisions"]:
+        notes.append(f"{gap} of {out['decisions']} decisions have no commit. Is the commit hook installed (`ledger init`), "
+                     "and was `ledger done` run before unrelated work? Squash merges drop trailers: `ledger label` recovers most of "
+                     "them by matching diffs; `ledger attach` links the rest by hand.")
+    if out["labeled"] == 0:
+        notes.append("Nothing labeled yet. Run `ledger label` once commits exist.")
+    elif out["last_label"] and now - out["last_label"] > 14 * DAY:
+        notes.append(f"Labels are {(now - out['last_label']) // DAY} days old. Re-run `ledger label`.")
+    if out["labeled"] and out["mature"] == 0:
+        oldest = one("SELECT MIN(ts) FROM decisions")
+        wait = max(0, (oldest + horizon_days * DAY - now) // DAY)
+        notes.append(f"No decision is {horizon_days} days old yet; the first estimate is about {wait} days away.")
+    if len(versions) > 1:
+        notes.append(f"{len(versions)} routing policy versions were used ({', '.join(sorted(versions))}). "
+                     "Propensities are logged per decision so this is fine, but keep them stable if you can.")
+    for arm, a in out["arms"].items():
+        if a["lowest_propensity"] < 0.02:
+            notes.append(f"{arm} was sometimes chosen with under 2% probability. Those decisions carry huge weights; "
+                         "raise --epsilon on `ledger decide`.")
+    if len(out["arms"]) == 1:
+        notes.append("Every decision went to one arm, so there is nothing to compare. Offer at least two arms to `decide`.")
+    if out["incidents"] and not out["deploys"]:
+        incidents_with_commits = sum(1 for r in con.execute("SELECT commits FROM incidents") if json.loads(r[0]))
+        if not incidents_with_commits:
+            notes.append("Incidents are imported but none names a suspect commit and no deploys are loaded, "
+                         "so they cannot be linked to changes.")
+    return out