pushback 0.1.0__py3-none-any.whl

pushback/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

pushback/audit.py

@@ -0,0 +1,63 @@
+"""Hand-check a stratified sample so the report can say how far to trust the labels.
+
+Half the sample comes from messages the model called corrections, half from
+the rest. Your answers never leave your machine.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import random
+
+PROMPT = "Is this a correction of the agent? [y]es / [n]o / [s]kip / [q]uit: "
+
+
+def load_audit(path: str) -> dict[str, dict]:
+    audit: dict[str, dict] = {}
+    if os.path.exists(path):
+        with open(path, encoding="utf-8") as fh:
+            for line in fh:
+                if line.strip():
+                    d = json.loads(line)
+                    audit[d["id"]] = d
+    return audit
+
+
+def sample(messages: dict[str, dict], labels: dict[str, dict], done: set[str], n: int, seed: int) -> list[tuple[str, str]]:
+    rng = random.Random(seed)
+    pos = sorted(i for i, d in labels.items() if d["correction"] and i not in done and i in messages)
+    neg = sorted(i for i, d in labels.items() if not d["correction"] and i not in done and i in messages)
+    rng.shuffle(pos)
+    rng.shuffle(neg)
+    half = n // 2
+    picked = [(i, "flagged") for i in pos[:half]] + [(i, "unflagged") for i in neg[: n - half]]
+    rng.shuffle(picked)  # don't let the order hint at the model's answer
+    return picked
+
+
+def run(messages, labels, audit_path, n=40, seed=0, ask=input, show=print) -> int:
+    done = load_audit(audit_path)
+    todo = sample(messages, labels, set(done), n, seed)
+    if not todo:
+        show("nothing left to audit")
+        return 0
+    answered = 0
+    with open(audit_path, "a", encoding="utf-8") as out:
+        for k, (i, stratum) in enumerate(todo, 1):
+            m = messages[i]
+            show(f"\n--- {k}/{len(todo)} ---")
+            show(f"AGENT: ...{m['prev_assistant'][-300:]}")
+            show(f"YOU:   {m['user']}")
+            while True:
+                a = ask(PROMPT).strip().lower()[:1]
+                if a in ("y", "n", "s", "q"):
+                    break
+            if a == "q":
+                break
+            if a == "s":
+                continue
+            out.write(json.dumps({"id": i, "stratum": stratum, "human": a == "y"}) + "\n")
+            out.flush()
+            answered += 1
+    return answered

pushback/cli.py

@@ -0,0 +1,231 @@
+"""pushback: how often do you correct your coding agent, and at what kind of work?"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+from . import audit as audit_mod
+from . import export as export_mod
+from . import extract as extract_mod
+from . import label as label_mod
+from . import report as report_mod
+from . import rules as rules_mod
+
+DATA = "pushback-data"
+
+
+def _paths(data_dir: str) -> dict[str, str]:
+    return {k: os.path.join(data_dir, f) for k, f in {
+        "messages": "messages.jsonl", "labels": "labels.jsonl",
+        "audit": "audit.jsonl", "silent": "silent.json", "report": "report.md",
+        "turns": "turns.jsonl", "dpo": "dpo.jsonl",
+        "rules": "rules.md", "rules_items": "rules-items.json", "rules_prompt": "rules-prompt.md",
+    }.items()}
+
+
+def _load_messages(path: str) -> dict[str, dict]:
+    if not os.path.exists(path):
+        sys.exit(f"{path} not found. Run `pushback extract` first.")
+    with open(path, encoding="utf-8") as fh:
+        return {m["id"]: m for m in map(json.loads, fh) if m}
+
+
+def cmd_extract(a):
+    p = _paths(a.data)
+    os.makedirs(a.data, exist_ok=True)
+    ex = extract_mod.extract(a.root, tuple(a.exclude or ()))
+    with open(p["messages"], "w", encoding="utf-8") as fh:
+        for m in ex.messages:
+            fh.write(json.dumps(m, ensure_ascii=False) + "\n")
+    with open(p["silent"], "w", encoding="utf-8") as fh:
+        json.dump({"rejections": ex.rejections, "interrupts": ex.interrupts}, fh)
+    with open(p["turns"], "w", encoding="utf-8") as fh:
+        for mid, t in ex.turns.items():
+            fh.write(json.dumps({"id": mid, **t}, ensure_ascii=False) + "\n")
+    print(f"{ex.sessions} sessions -> {len(ex.messages)} messages, "
+          f"{sum(ex.rejections.values())} rejected tool calls, {sum(ex.interrupts.values())} interrupts")
+    print(f"written to {a.data}/ (this folder holds your raw messages; keep it out of git)")
+
+
+def cmd_label(a):
+    p = _paths(a.data)
+    messages = list(_load_messages(p["messages"]).values())
+    base = os.environ.get("ANTHROPIC_BASE_URL")
+    print(f"Sending {len(messages)} messages to {base or 'the Anthropic API'} with model {a.model}.")
+    print("Your messages leave this machine for that provider. Check its data policy before using client logs.")
+    if not a.yes and input("Continue? [y/N] ").strip().lower() != "y":
+        sys.exit("aborted")
+    written, failed = label_mod.run(messages, p["labels"], a.model, a.batch_size, a.workers, a.effort)
+    print(f"done: {written} new labels, {failed} failed batches" + (" (re-run to retry them)" if failed else ""))
+
+
+def cmd_audit(a):
+    p = _paths(a.data)
+    messages = _load_messages(p["messages"])
+    labels = label_mod.load_labels(p["labels"])
+    if not labels:
+        sys.exit("no labels yet. Run `pushback label` first.")
+    n = audit_mod.run(messages, labels, p["audit"], a.n, a.seed)
+    print(f"\nsaved {n} answers to {p['audit']}")
+
+
+def cmd_report(a):
+    p = _paths(a.data)
+    labels = label_mod.load_labels(p["labels"])
+    if not labels:
+        sys.exit("no labels yet. Run `pushback label` first.")
+    silent = json.load(open(p["silent"], encoding="utf-8")) if os.path.exists(p["silent"]) else None
+    text = report_mod.render(report_mod.build(labels, audit_mod.load_audit(p["audit"]), silent))
+    print(text)
+    if a.markdown:
+        with open(p["report"], "w", encoding="utf-8") as fh:
+            fh.write(text + "\n")
+        print(f"\nwritten to {p['report']} (counts only, safe to share)")
+
+
+def cmd_export(a):
+    p = _paths(a.data)
+    messages = list(_load_messages(p["messages"]).values())
+    labels = label_mod.load_labels(p["labels"])
+    if not labels:
+        sys.exit("no labels yet. Run `pushback label` first.")
+    if not os.path.exists(p["turns"]):
+        sys.exit(f"{p['turns']} not found. Re-run `pushback extract` (it now saves full agent turns).")
+    with open(p["turns"], encoding="utf-8") as fh:
+        turns = {t["id"]: t for t in map(json.loads, fh)}
+    pairs, skipped = export_mod.build_pairs(
+        messages, labels, turns,
+        tasks=set(a.task) if a.task else None, ctypes=set(a.ctype) if a.ctype else None,
+        high_only=a.high_only, max_tools=a.max_tools,
+    )
+    redacted = export_mod.scrub(pairs)
+    if a.minimal:
+        pairs = [{k: x[k] for k in ("prompt", "chosen", "rejected")} for x in pairs]
+    out = a.out or p["dpo"]
+    with open(out, "w", encoding="utf-8") as fh:
+        for x in pairs:
+            fh.write(json.dumps(x, ensure_ascii=False) + "\n")
+    corrections = sum(1 for d in labels.values() if d["correction"])
+    print(f"{len(pairs)} pairs from {corrections} corrections -> {out}")
+    for reason, n in skipped.most_common():
+        print(f"  skipped {n}: {reason}")
+    print(f"  {redacted} likely secrets replaced with [REDACTED] (best effort: read the file before using it)")
+    print("This file holds raw agent and user text. It stays on your machine unless you move it.")
+
+
+def _default_existing() -> list[str]:
+    path = os.path.join(os.path.expanduser("~"), ".claude", "CLAUDE.md")
+    return [path] if os.path.exists(path) else []
+
+
+def cmd_rules(a):
+    p = _paths(a.data)
+    labels = label_mod.load_labels(p["labels"])
+    if not labels:
+        sys.exit("no labels yet. Run `pushback label` first.")
+
+    if a.from_response:
+        with open(p["rules_items"], encoding="utf-8") as fh:
+            items = json.load(fh)
+        with open(a.from_response, encoding="utf-8") as fh:
+            text = fh.read()
+        raw = json.loads(text[text.find("{"): text.rfind("}") + 1])  # tolerate prose or fences around the JSON
+    else:
+        if not os.path.exists(p["turns"]):
+            sys.exit(f"{p['turns']} not found. Re-run `pushback extract`.")
+        with open(p["turns"], encoding="utf-8") as fh:
+            turns = {t["id"]: t for t in map(json.loads, fh)}
+        items = rules_mod.collect(labels, turns, set(a.task) if a.task else None,
+                                  set(a.ctype) if a.ctype else None, a.limit)
+        if not items:
+            sys.exit("no corrections match those filters.")
+        existing_paths = a.existing if a.existing is not None else _default_existing()
+        existing = "\n".join(open(x, encoding="utf-8", errors="ignore").read() for x in existing_paths)
+        request = rules_mod.build_request(items, existing, a.model, a.effort)
+        with open(p["rules_items"], "w", encoding="utf-8") as fh:
+            json.dump(items, fh, ensure_ascii=False)
+        print(f"{len(items)} corrections, checked against {len(existing_paths)} existing rule file(s)")
+
+        if a.prompt_file:
+            with open(p["rules_prompt"], "w", encoding="utf-8") as fh:
+                fh.write(request["system"] + "\n\n" + request["messages"][0]["content"] + "\n\n"
+                         + "Reply with only a JSON object matching this schema:\n"
+                         + json.dumps(rules_mod.SCHEMA, indent=2) + "\n")
+            print(f"prompt written to {p['rules_prompt']}")
+            print("Give it to Claude (for example: ask Claude Code to answer the file), save the JSON reply,")
+            print("then run: pushback rules --from-response <reply file>")
+            return
+
+        base = os.environ.get("ANTHROPIC_BASE_URL")
+        print(f"Sending them to {base or 'the Anthropic API'} with model {a.model}.")
+        if not a.yes and input("Continue? [y/N] ").strip().lower() != "y":
+            sys.exit("aborted")
+        import anthropic
+        raw = rules_mod.ask(anthropic.Anthropic(), request)
+
+    found = rules_mod.parse(raw, items, a.min_support)
+    with open(p["rules"], "w", encoding="utf-8") as fh:
+        fh.write(rules_mod.render(found, len(items)))
+    broken = sum(1 for r in found if r["covered_by"])
+    print(f"{len(found)} rules ({broken} you already have but keep breaking) -> {p['rules']}")
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="pushback", description=__doc__)
+    ap.add_argument("--data", default=DATA, help=f"working folder (default: ./{DATA})")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    e = sub.add_parser("extract", help="pull your messages out of Claude Code transcripts")
+    e.add_argument("--root", default=extract_mod.DEFAULT_ROOT, help="transcripts folder")
+    e.add_argument("--exclude", nargs="*", help="session id prefixes to skip")
+    e.set_defaults(func=cmd_extract)
+
+    lb = sub.add_parser("label", help="label each message with Claude (resumable)")
+    lb.add_argument("--model", default=label_mod.DEFAULT_MODEL)
+    lb.add_argument("--effort", default="low", choices=["low", "medium", "high"])
+    lb.add_argument("--batch-size", type=int, default=40)
+    lb.add_argument("--workers", type=int, default=4)
+    lb.add_argument("-y", "--yes", action="store_true", help="skip the data-leaves-your-machine prompt")
+    lb.set_defaults(func=cmd_label)
+
+    au = sub.add_parser("audit", help="hand-check a sample so the report can bound the error")
+    au.add_argument("-n", type=int, default=40)
+    au.add_argument("--seed", type=int, default=0)
+    au.set_defaults(func=cmd_audit)
+
+    ex = sub.add_parser("export", help="write correction pairs as prompt/chosen/rejected JSONL")
+    ex.add_argument("--task", nargs="*", help="only these tasks, e.g. --task writing media")
+    ex.add_argument("--ctype", nargs="*",
+                    help="only these correction types; writing_content and tone_style make the cleanest pairs")
+    ex.add_argument("--high-only", action="store_true", help="only high-confidence labels")
+    ex.add_argument("--max-tools", type=int, help="drop pairs where either turn made more tool calls than this")
+    ex.add_argument("--minimal", action="store_true", help="only prompt/chosen/rejected (TRL DPO columns)")
+    ex.add_argument("--out", help="output path (default: <data>/dpo.jsonl)")
+    ex.set_defaults(func=cmd_export)
+
+    ru = sub.add_parser("rules", help="draft CLAUDE.md rules from your recurring corrections")
+    ru.add_argument("--task", nargs="*")
+    ru.add_argument("--ctype", nargs="*")
+    ru.add_argument("--limit", type=int, default=400, help="most recent N corrections (default 400)")
+    ru.add_argument("--min-support", type=int, default=3, help="corrections needed per rule (default 3)")
+    ru.add_argument("--existing", nargs="*", help="rule files to check against (default: ~/.claude/CLAUDE.md)")
+    ru.add_argument("--model", default=label_mod.DEFAULT_MODEL)
+    ru.add_argument("--effort", default="high", choices=["low", "medium", "high"])
+    ru.add_argument("--prompt-file", action="store_true", help="write the request to a file instead of calling the API")
+    ru.add_argument("--from-response", help="read the model's JSON reply from this file")
+    ru.add_argument("-y", "--yes", action="store_true")
+    ru.set_defaults(func=cmd_rules)
+
+    rp = sub.add_parser("report", help="print the correction-rate table")
+    rp.add_argument("--markdown", action="store_true", help="also write report.md")
+    rp.set_defaults(func=cmd_report)
+
+    args = ap.parse_args(argv)
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

pushback/export.py

@@ -0,0 +1,133 @@
+"""Turn labelled corrections into preference pairs (prompt / chosen / rejected).
+
+For a correction C in a session ... P, [agent turn R], C, [agent turn A], N ...
+
+    prompt   = P, the message the agent was answering
+    rejected = R, the agent turn you corrected
+    chosen   = A, the agent's reply to C, kept only if your next message N exists and is NOT a correction
+    feedback = C, your correction itself
+
+"You didn't correct it again" is weak evidence that A was good, and A was
+written after seeing your feedback. Treat these pairs as a personal dataset
+to inspect and filter, not as clean ground truth.
+
+Only agent *text* is exported. Edits and commands the agent ran are tool
+calls, so a code correction's pair may carry the explanation but not the
+diff. The tool-call counts are kept so you can filter those out.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+
+# Best-effort scrub of common credential shapes. It will miss things; read your export.
+SECRET_PATTERNS = [
+    re.compile(r"sk-ant-[A-Za-z0-9_\-]{20,}"),
+    re.compile(r"sk-[A-Za-z0-9_\-]{20,}"),
+    re.compile(r"gh[pousr]_[A-Za-z0-9]{30,}"),
+    re.compile(r"github_pat_[A-Za-z0-9_]{30,}"),
+    re.compile(r"AKIA[0-9A-Z]{16}"),
+    re.compile(r"xox[baprs]-[A-Za-z0-9\-]{10,}"),
+    re.compile(r"AIza[0-9A-Za-z_\-]{35}"),
+    re.compile(r"(?i)bearer\s+[A-Za-z0-9._\-]{20,}"),
+    re.compile(r"eyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}"),  # JWT
+    re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----"),
+]
+# NAME_API_KEY=value / SECRET: value with a long value. The name is kept so the text still reads;
+# placeholders like "your-key-here" are shorter than 24 characters and pass through.
+ASSIGNMENT = re.compile(
+    r"(?i)((?:api[_-]?key|secret|token|password|passwd)\w*\s*[:=]\s*['\"]?)[A-Za-z0-9_\-./+]{24,}")
+
+
+def redact(text: str) -> tuple[str, int]:
+    hits = 0
+    for pat in SECRET_PATTERNS:
+        text, n = pat.subn("[REDACTED]", text)
+        hits += n
+    text, n = ASSIGNMENT.subn(r"\1[REDACTED]", text)
+    return text, hits + n
+
+
+def _order_key(message_id: str) -> tuple[str, int]:
+    session, _, n = message_id.rpartition(":")
+    return session, int(n)
+
+
+def build_pairs(
+    messages: list[dict],
+    labels: dict[str, dict],
+    turns: dict[str, dict],
+    tasks: set[str] | None = None,
+    ctypes: set[str] | None = None,
+    high_only: bool = False,
+    max_tools: int | None = None,
+) -> tuple[list[dict], Counter]:
+    skipped: Counter = Counter()
+    by_id = {m["id"]: m for m in messages}
+    ids = sorted(by_id, key=_order_key)
+    pos = {i: k for k, i in enumerate(ids)}
+    pairs = []
+
+    for mid in ids:
+        lab = labels.get(mid)
+        if not lab or not lab["correction"]:
+            continue
+        if tasks and lab["task"] not in tasks:
+            skipped["task filtered"] += 1
+            continue
+        if ctypes and lab["ctype"] not in ctypes:
+            skipped["correction type filtered"] += 1
+            continue
+        if high_only and lab.get("conf") != "high":
+            skipped["low confidence"] += 1
+            continue
+        session, n = _order_key(mid)
+        prev_id, next_id = f"{session}:{n - 1}", f"{session}:{n + 1}"
+        if prev_id not in pos:
+            skipped["no earlier message"] += 1
+            continue
+        # Your next message (n+1) is both what follows the agent's new reply and your verdict on it.
+        if next_id not in pos:
+            skipped["session ended after correction"] += 1
+            continue
+        verdict = labels.get(next_id)
+        if verdict is None:
+            skipped["reaction not labelled"] += 1
+            continue
+        if verdict["correction"]:
+            skipped["new reply was corrected too"] += 1
+            continue
+
+        rejected = turns.get(mid, {}).get("prev_turn", "")
+        chosen = turns.get(next_id, {}).get("prev_turn", "")
+        if not rejected.strip() or not chosen.strip():
+            skipped["agent turn had no text"] += 1
+            continue
+        r_tools = turns[mid].get("prev_turn_tools", 0)
+        c_tools = turns[next_id].get("prev_turn_tools", 0)
+        if max_tools is not None and max(r_tools, c_tools) > max_tools:
+            skipped["too many tool calls"] += 1
+            continue
+
+        pairs.append({
+            "prompt": turns.get(prev_id, {}).get("prompt", ""),
+            "rejected": rejected,
+            "chosen": chosen,
+            "feedback": turns[mid].get("prompt", ""),
+            "id": mid,
+            "task": lab["task"],
+            "ctype": lab["ctype"],
+            "rejected_tool_calls": r_tools,
+            "chosen_tool_calls": c_tools,
+        })
+    return pairs, skipped
+
+
+def scrub(pairs: list[dict]) -> int:
+    total = 0
+    for p in pairs:
+        for k in ("prompt", "rejected", "chosen", "feedback"):
+            p[k], n = redact(p[k])
+            total += n
+    return total

pushback/extract.py

@@ -0,0 +1,147 @@
+"""Pull human messages (with the assistant turn they respond to) out of Claude Code transcripts.
+
+Claude Code writes one JSONL file per session under ~/.claude/projects/<project>/.
+We keep only what a person typed: tool results, system reminders, slash-command
+output and subagent (sidechain) traffic are dropped. Silent corrections -- a
+rejected tool call or an interrupt -- carry no text, so they are counted
+separately and attributed to the kind of action that was cut off.
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+from collections import Counter
+from dataclasses import dataclass, field
+
+DEFAULT_ROOT = os.path.join(os.path.expanduser("~"), ".claude", "projects")
+
+REJECTED = "doesn't want to proceed"
+INTERRUPTED = "[Request interrupted by user"
+
+CODE_EXT = {
+    ".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".sql", ".prisma", ".json", ".yml", ".yaml",
+    ".sh", ".css", ".html", ".go", ".rs", ".toml", ".java", ".kt", ".rb", ".php", ".cs",
+}
+
+
+@dataclass
+class Extracted:
+    messages: list[dict] = field(default_factory=list)
+    rejections: Counter = field(default_factory=Counter)
+    interrupts: Counter = field(default_factory=Counter)
+    # Full text for `export`: the whole agent turn before each message, and the untruncated message.
+    turns: dict[str, dict] = field(default_factory=dict)
+    sessions: int = 0
+
+
+def action_kind(name: str, inp: dict | None) -> str:
+    """Bucket a tool call so silent corrections can be attributed without storing its content."""
+    inp = inp or {}
+    if name in ("Edit", "Write", "MultiEdit", "NotebookEdit"):
+        ext = os.path.splitext(inp.get("file_path") or "")[1].lower()
+        return "edit-code" if ext in CODE_EXT else "edit-other"
+    if name in ("Bash", "PowerShell"):
+        cmd = inp.get("command") or ""
+        if "git " in cmd or "gh " in cmd:
+            return "shell-git"
+        if any(k in cmd for k in ("npm", "pnpm", "yarn", "pytest", "python", "node", "tsc", "npx", "cargo", "go ")):
+            return "shell-build"
+        return "shell-other"
+    if name.startswith("mcp__"):
+        return "mcp"
+    return "other-tool"
+
+
+def _is_human_text(text: str) -> bool:
+    if not text:
+        return False
+    # Harness-injected blocks (<system-reminder>, <command-name>, ...) start with a tag.
+    # Pasted content is still something the person chose to send.
+    return not (text.startswith("<") and not text.startswith("<pasted"))
+
+
+def extract(
+    root: str = DEFAULT_ROOT,
+    exclude_sessions: tuple[str, ...] = (),
+    max_user_chars: int = 500,
+    max_context_chars: int = 400,
+) -> Extracted:
+    out = Extracted()
+    files = sorted(glob.glob(os.path.join(root, "*", "*.jsonl")))
+    for path in files:
+        session = os.path.splitext(os.path.basename(path))[0]
+        if any(session.startswith(x) for x in exclude_sessions):
+            continue
+        out.sessions += 1
+        _extract_file(path, session, out, max_user_chars, max_context_chars)
+    return out
+
+
+def _extract_file(path, session, out, max_user_chars, max_context_chars):
+    last_text = ""
+    n = 0
+    tools: dict[str, tuple[str, dict]] = {}
+    last_tool: tuple[str, dict] | None = None
+    # One agent turn = every text block and tool call between two human messages.
+    turn_text: list[str] = []
+    turn_tools = 0
+    with open(path, encoding="utf-8", errors="ignore") as fh:
+        for line in fh:
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if entry.get("isSidechain"):
+                continue
+            message = entry.get("message")
+            content = message.get("content") if isinstance(message, dict) else None
+
+            if entry.get("type") == "assistant" and isinstance(content, list):
+                text = " ".join(b.get("text", "") for b in content if b.get("type") == "text").strip()
+                if text:
+                    last_text = text
+                    turn_text.append(text)
+                for b in content:
+                    if b.get("type") == "tool_use":
+                        turn_tools += 1
+                        tools[b.get("id")] = (b.get("name", ""), b.get("input"))
+                        last_tool = tools[b.get("id")]
+                continue
+
+            if entry.get("type") != "user" or entry.get("isMeta"):
+                continue
+
+            if isinstance(content, list):
+                texts = []
+                for b in content:
+                    if b.get("type") == "tool_result" and REJECTED in json.dumps(b.get("content")):
+                        name, inp = tools.get(b.get("tool_use_id"), ("", None))
+                        out.rejections[action_kind(name, inp)] += 1
+                    elif b.get("type") == "text":
+                        texts.append(b.get("text", ""))
+            else:
+                texts = [content or ""]
+
+            for text in texts:
+                text = text.strip()
+                if INTERRUPTED in text:
+                    out.interrupts[action_kind(*last_tool) if last_tool else "none"] += 1
+                    continue
+                if not _is_human_text(text):
+                    continue
+                out.messages.append({
+                    # Stable across re-extracts: sessions only ever append, so session:index never shifts.
+                    "id": f"{session}:{n}",
+                    "session": session[:8],
+                    "prev_assistant": last_text[-max_context_chars:].replace("\n", " "),
+                    "user": text[:max_user_chars].replace("\n", " "),
+                })
+                out.turns[f"{session}:{n}"] = {
+                    "prompt": text,
+                    "prev_turn": "\n\n".join(turn_text),
+                    "prev_turn_tools": turn_tools,
+                }
+                turn_text, turn_tools = [], 0
+                n += 1

pushback/label.py

@@ -0,0 +1,120 @@
+"""Send batches of messages to Claude and store one label per message.
+
+Resumable: labels are appended to labels.jsonl batch by batch, and a re-run
+skips every id already present. A batch that fails is logged and left
+unlabelled, so a network blip never becomes a silent "not a correction".
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import anthropic
+
+from .prompt import CTYPES, SCHEMA, SYSTEM, TASKS
+
+DEFAULT_MODEL = "claude-opus-5"
+
+
+def load_labels(path: str) -> dict[str, dict]:
+    labels: dict[str, dict] = {}
+    if os.path.exists(path):
+        with open(path, encoding="utf-8") as fh:
+            for line in fh:
+                if line.strip():
+                    d = json.loads(line)
+                    labels[d["id"]] = d
+    return labels
+
+
+def validate(raw: list[dict], keys: list[str]) -> list[dict]:
+    """Map the model's per-batch numbers back to message ids, keeping only well-formed labels.
+
+    Anything missing or malformed stays unlabelled and is retried on the next run.
+    """
+    good = []
+    seen = set()
+    for d in raw:
+        try:
+            i = int(d["id"])
+        except (KeyError, TypeError, ValueError):
+            continue
+        if not 0 <= i < len(keys) or i in seen:
+            continue
+        if d.get("task") not in TASKS or d.get("ctype") not in CTYPES or not isinstance(d.get("correction"), bool):
+            continue
+        if not d["correction"]:
+            d["ctype"] = "none"
+        seen.add(i)
+        good.append({"id": keys[i], **{k: d[k] for k in ("task", "correction", "ctype", "conf") if k in d}})
+    return good
+
+
+def _request_kwargs(model: str, batch: list[dict], effort: str) -> dict:
+    items = [{"id": n, "prev_assistant": m["prev_assistant"], "user": m["user"]} for n, m in enumerate(batch)]
+    return {
+        "model": model,
+        "max_tokens": 16000,
+        "system": SYSTEM,
+        "messages": [{"role": "user", "content": json.dumps(items, ensure_ascii=False)}],
+        "output_config": {"effort": effort, "format": {"type": "json_schema", "schema": SCHEMA}},
+    }
+
+
+def label_batch(client, model: str, batch: list[dict], effort: str = "low") -> list[dict]:
+    response = client.messages.create(**_request_kwargs(model, batch, effort))
+    if response.stop_reason == "refusal":
+        raise RuntimeError("model declined this batch (stop_reason=refusal)")
+    if response.stop_reason == "max_tokens":
+        raise RuntimeError("output hit max_tokens; use a smaller --batch-size")
+    text = next((b.text for b in response.content if b.type == "text"), "")
+    return validate(json.loads(text).get("labels", []), [m["id"] for m in batch])
+
+
+def run(
+    messages: list[dict],
+    labels_path: str,
+    model: str = DEFAULT_MODEL,
+    batch_size: int = 40,
+    workers: int = 4,
+    effort: str = "low",
+    client=None,
+    log=print,
+) -> tuple[int, int]:
+    done = load_labels(labels_path)
+    todo = [m for m in messages if m["id"] not in done]
+    batches = [todo[i:i + batch_size] for i in range(0, len(todo), batch_size)]
+    if not batches:
+        log(f"all {len(messages)} messages already labelled")
+        return 0, 0
+
+    client = client or anthropic.Anthropic()
+    lock = threading.Lock()
+    written = failed = 0
+
+    def work(batch):
+        return batch, label_batch(client, model, batch, effort)
+
+    with ThreadPoolExecutor(max_workers=workers) as pool, open(labels_path, "a", encoding="utf-8") as out:
+        futures = [pool.submit(work, b) for b in batches]
+        for n, fut in enumerate(as_completed(futures), 1):
+            try:
+                batch, labels = fut.result()
+            except anthropic.AuthenticationError:
+                raise
+            except (anthropic.APIError, RuntimeError, json.JSONDecodeError) as e:
+                # APIError covers status errors AND connection errors; both leave the batch for a re-run.
+                failed += 1
+                log(f"[{n}/{len(batches)}] batch failed, will retry on next run: {type(e).__name__}: {e}")
+                continue
+            with lock:
+                for d in labels:
+                    out.write(json.dumps(d) + "\n")
+                out.flush()
+                written += len(labels)
+            missing = len(batch) - len(labels)
+            log(f"[{n}/{len(batches)}] labelled {len(labels)}" + (f", {missing} left for re-run" if missing else ""))
+    return written, failed

pushback/prompt.py

@@ -0,0 +1,71 @@
+"""The labelling instructions and the JSON schema the model must answer in.
+
+These are the instructions that scored precision 0.97 / recall 0.80 against 91
+hand labels on the author's own logs. Change them and re-run `pushback audit`
+before trusting new numbers.
+"""
+
+TASKS = ["code", "writing", "media", "research", "ops", "meta"]
+CTYPES = [
+    "none", "writing_content", "tone_style", "wrong_assumption", "scope",
+    "broken_output", "code", "misread_request", "process",
+]
+
+SYSTEM = """You label messages a user sent to an AI coding agent. The agent is also used for writing, research and ops.
+
+Each item has: id, prev_assistant (the end of the agent's previous reply), user (the user's next message).
+Give every item three labels.
+
+task: the work in progress, judged from BOTH prev_assistant and user. Pick one.
+- code: writing, fixing or reviewing code, tests, builds, PRs, schemas
+- writing: prose for humans, such as social posts, business messages, emails, scripts, docs, READMEs, proposals
+- media: images, thumbnails, video, animation, design visuals
+- research: benchmarks, experiments, data analysis, paper reading, market research
+- ops: deploys, git/GitHub admin, accounts, config, credentials, infra, tool setup, browser automation
+- meta: planning and strategy talk, advice, saving memory or context, chit-chat, session management
+
+correction: true ONLY if the user rejects, fixes or redirects something the agent just did, said, wrote or proposed because it was wrong, unwanted, incomplete or not what they asked.
+It is false for all of these:
+- new tasks, next steps, "now do X", moving on, "save to memory"
+- answers to the agent's questions (a plain "no" to a yes/no question is an answer)
+- choosing between options the agent offered, or reprioritising ("first do X")
+- approvals, thanks, reassurance, jokes
+- pasted logs or content with no reaction to the agent's work
+- complaints about third parties or tools
+- the user's own new idea or realisation
+
+ctype: "none" unless correction is true, otherwise one of:
+- writing_content: the text missed or misstated content ("you didn't mention X", "start it with Y")
+- tone_style: voice, length, format, "sounds AI", "looks AI", design style
+- wrong_assumption: the agent assumed something false about the user, their state or their facts
+- scope: did too much, added unrequested things, offered unwanted things
+- broken_output: a non-code output (file, image, video, deploy) is missing, stale or broken
+- code: a code bug, failing build or test, or wrong technical implementation
+- misread_request: misunderstood what was asked
+- process: how the agent works (tool or model choice, don't ask, verify first)
+
+conf: "high" or "low".
+Return exactly one label per input id, and never repeat message text."""
+
+SCHEMA = {
+    "type": "object",
+    "properties": {
+        "labels": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "id": {"type": "integer"},
+                    "task": {"type": "string", "enum": TASKS},
+                    "correction": {"type": "boolean"},
+                    "ctype": {"type": "string", "enum": CTYPES},
+                    "conf": {"type": "string", "enum": ["high", "low"]},
+                },
+                "required": ["id", "task", "correction", "ctype", "conf"],
+                "additionalProperties": False,
+            },
+        }
+    },
+    "required": ["labels"],
+    "additionalProperties": False,
+}

pushback/report.py

@@ -0,0 +1,81 @@
+"""Turn labels (and an optional audit) into the correction-rate table. Counts only, never message text."""
+
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+
+from .prompt import TASKS
+from .stats import corrected_count, wilson
+
+
+def build(labels: dict[int, dict], audit: dict[int, dict] | None = None, silent: dict | None = None) -> dict:
+    total = Counter(d["task"] for d in labels.values())
+    corr = Counter(d["task"] for d in labels.values() if d["correction"])
+    ctypes = defaultdict(Counter)
+    for d in labels.values():
+        if d["correction"]:
+            ctypes[d["task"]][d["ctype"]] += 1
+
+    rows = []
+    for task in TASKS:
+        n, k = total[task], corr[task]
+        if not n:
+            continue
+        lo, hi = wilson(k, n)
+        rows.append({"task": task, "messages": n, "corrections": k, "rate": k / n, "low": lo, "high": hi,
+                     "top": ctypes[task].most_common(3)})
+    rows.sort(key=lambda r: r["rate"], reverse=True)
+
+    flagged = sum(corr.values())
+    out = {"messages": len(labels), "flagged": flagged, "rows": rows, "audit": None,
+           "silent": silent or {}}
+
+    if audit:
+        pos = [a for a in audit.values() if a["stratum"] == "flagged"]
+        neg = [a for a in audit.values() if a["stratum"] == "unflagged"]
+        out["audit"] = {
+            "n": len(audit),
+            **corrected_count(
+                flagged, len(labels) - flagged,
+                sum(a["human"] for a in pos), len(pos),
+                sum(a["human"] for a in neg), len(neg),
+            ),
+        }
+    return out
+
+
+def render(r: dict) -> str:
+    lines = []
+    n, f = r["messages"], r["flagged"]
+    lines.append(f"{n} messages labelled, {f} flagged as corrections ({f / n:.1%})." if n else "no labels yet")
+    lines.append("")
+    lines.append("| task | messages | corrections | rate | 95% CI | most common |")
+    lines.append("|---|---:|---:|---:|---|---|")
+    for row in r["rows"]:
+        top = ", ".join(f"{c} {k}" for c, k in row["top"])
+        lines.append(f"| {row['task']} | {row['messages']} | {row['corrections']} | {row['rate']:.1%} "
+                     f"| {row['low']:.1%}-{row['high']:.1%} | {top} |")
+
+    a = r["audit"]
+    lines.append("")
+    if not a:
+        lines.append("No audit yet: these are raw model labels. Run `pushback audit` before quoting any number.")
+    elif a["estimate"] is None:
+        lines.append(f"Audit has {a['n']} answers but needs both flagged and unflagged samples.")
+    else:
+        lines.append(f"Audit ({a['n']} hand checks): precision {a['precision']:.0%}, "
+                     f"miss rate {a['miss_rate']:.1%} on unflagged messages.")
+        lines.append(f"Estimated true corrections: {a['estimate']:.0f} "
+                     f"(range {a['low']:.0f}-{a['high']:.0f}) = {a['estimate'] / n:.1%} of messages.")
+        if a["n"] < 40:
+            lines.append("Fewer than 40 hand checks: the range is wide. Audit more before posting.")
+
+    s = r["silent"]
+    if s and (s.get("rejections") or s.get("interrupts")):
+        rej, intr = s.get("rejections", {}), s.get("interrupts", {})
+        lines.append("")
+        lines.append(f"Silent corrections (not in the table): {sum(rej.values())} rejected tool calls, "
+                     f"{sum(intr.values())} interrupts.")
+        both = Counter(rej) + Counter(intr)
+        lines.append("By action: " + ", ".join(f"{k} {v}" for k, v in both.most_common()))
+    return "\n".join(lines)

pushback/rules.py

@@ -0,0 +1,154 @@
+"""Turn recurring corrections into rules you can paste into CLAUDE.md.
+
+The model groups corrections that share a cause and drafts one instruction per
+group. Two checks keep that honest:
+
+- Support is counted here from the correction ids the model cites, never taken
+  from the model's own numbers, and a rule needs `min_support` real corrections.
+- Given your existing rule files, the model marks rules you already have. Those
+  are the interesting ones: a rule that exists and still gets broken isn't working.
+"""
+
+from __future__ import annotations
+
+import json
+
+from .prompt import TASKS
+
+AGENT_CHARS = 500
+CORRECTION_CHARS = 400
+
+SYSTEM = """You are given corrections a user made to their AI agent. Each item has an id, the task type, the end of the agent's turn that was corrected, and the user's correction.
+
+Find recurring patterns: the same kind of mistake corrected at least 3 times. For each pattern, write one rule the agent can follow to avoid it.
+
+Rules:
+- Write each rule as an instruction for a CLAUDE.md file, in plain words. Make it specific and checkable. "Be careful" or "write better" is not a rule.
+- Generalise. Never include client names, company names, people's names, credentials or quoted message text in a rule.
+- "why" is one sentence on what kept going wrong.
+- "ids" lists every correction that supports the pattern. Only cite ids that really show the same mistake.
+- If existing rules are provided and one of them already asks for this, put a short quote of it (15 words max) in "covered_by". Otherwise leave "covered_by" as an empty string.
+- Prefer fewer, sharper rules. Skip one-off corrections."""
+
+SCHEMA = {
+    "type": "object",
+    "properties": {
+        "rules": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "rule": {"type": "string"},
+                    "why": {"type": "string"},
+                    "task": {"type": "string", "enum": TASKS},
+                    "ids": {"type": "array", "items": {"type": "integer"}},
+                    "covered_by": {"type": "string"},
+                },
+                "required": ["rule", "why", "task", "ids", "covered_by"],
+                "additionalProperties": False,
+            },
+        }
+    },
+    "required": ["rules"],
+    "additionalProperties": False,
+}
+
+
+def collect(labels: dict[str, dict], turns: dict[str, dict], tasks=None, ctypes=None, limit=400) -> list[dict]:
+    """Every labelled correction with the context the model needs. Newest last, capped at `limit`."""
+    items = []
+    for mid, lab in labels.items():
+        if not lab["correction"] or (tasks and lab["task"] not in tasks) or (ctypes and lab["ctype"] not in ctypes):
+            continue
+        turn = turns.get(mid, {})
+        items.append({
+            "id": mid,
+            "task": lab["task"],
+            "agent": turn.get("prev_turn", "")[-AGENT_CHARS:],
+            "correction": turn.get("prompt", "")[:CORRECTION_CHARS],
+        })
+    items.sort(key=lambda x: (x["id"].rpartition(":")[0], int(x["id"].rpartition(":")[2])))
+    return items[-limit:]
+
+
+def build_request(items: list[dict], existing: str, model: str, effort: str = "high") -> dict:
+    payload = [{"n": n, "task": x["task"], "agent": x["agent"], "correction": x["correction"]}
+               for n, x in enumerate(items)]
+    content = "CORRECTIONS (the \"n\" field is the id to cite):\n" + json.dumps(payload, ensure_ascii=False)
+    if existing.strip():
+        content = "EXISTING RULES:\n" + existing.strip() + "\n\n" + content
+    return {
+        "model": model,
+        "max_tokens": 16000,
+        "system": SYSTEM,
+        "messages": [{"role": "user", "content": content}],
+        "output_config": {"effort": effort, "format": {"type": "json_schema", "schema": SCHEMA}},
+    }
+
+
+def parse(raw: dict, items: list[dict], min_support: int = 3) -> list[dict]:
+    """Map cited numbers back to correction ids, recount support, drop weak or malformed rules."""
+    rules = []
+    for r in raw.get("rules", []):
+        if not isinstance(r, dict) or not str(r.get("rule", "")).strip():
+            continue
+        ids = []
+        for i in r.get("ids", []):
+            if isinstance(i, int) and 0 <= i < len(items) and items[i]["id"] not in ids:
+                ids.append(items[i]["id"])
+        if len(ids) < min_support:
+            continue
+        rules.append({
+            "rule": r["rule"].strip(),
+            "why": str(r.get("why", "")).strip(),
+            "task": r.get("task") if r.get("task") in TASKS else "meta",
+            "support": len(ids),
+            "ids": ids,
+            "covered_by": str(r.get("covered_by", "")).strip(),
+        })
+    rules.sort(key=lambda r: r["support"], reverse=True)
+    return rules
+
+
+def ask(client, request: dict) -> dict:
+    response = client.messages.create(**request)
+    if response.stop_reason == "refusal":
+        raise RuntimeError("model declined (stop_reason=refusal)")
+    if response.stop_reason == "max_tokens":
+        raise RuntimeError("output hit max_tokens; narrow the input with --task or --limit")
+    text = next((b.text for b in response.content if b.type == "text"), "")
+    return json.loads(text)
+
+
+def _short(message_id: str) -> str:
+    """session-uuid:12 -> first 8 chars of the session, which is enough to find it."""
+    session, _, n = message_id.rpartition(":")
+    return f"{session[:8]}:{n}"
+
+
+def render(rules: list[dict], n_corrections: int) -> str:
+    broken = [r for r in rules if r["covered_by"]]
+    new = [r for r in rules if not r["covered_by"]]
+    out = [f"# Rules from {n_corrections} corrections", ""]
+    out.append("Each rule is backed by at least the number of corrections shown. Read the evidence ids "
+               "in your own data before adopting a rule; the model drafted these, you decide.")
+
+    def block(title, note, group):
+        if not group:
+            return
+        out.extend(["", f"## {title}", "", note, ""])
+        for r in group:
+            out.append(f"- **{r['rule']}**  ")
+            out.append(f"  {r['why']} ({r['task']}, {r['support']} corrections)")
+            if r["covered_by"]:
+                out.append(f"  You already have: \"{r['covered_by']}\"")
+            shown = [_short(i) for i in r["ids"][:8]]
+            out.append(f"  Evidence: {', '.join(shown)}" + (" …" if len(r["ids"]) > 8 else ""))
+
+    block("Rules you already have that keep getting broken",
+          "A rule that exists and still gets corrected isn't working. Make it more specific, "
+          "move it somewhere the agent reads at the right moment, or turn it into a check.", broken)
+    block("New rules to consider", "Recurring corrections with no rule behind them yet.", new)
+    if not rules:
+        out.extend(["", "No pattern reached the minimum support. Try more data or a lower --min-support."])
+    return "\n".join(out) + "\n"

pushback/stats.py

@@ -0,0 +1,45 @@
+"""Small, dependency-free statistics: Wilson intervals and the audit-corrected estimate."""
+
+from __future__ import annotations
+
+import math
+
+
+def wilson(k: int, n: int, z: float = 1.96) -> tuple[float, float]:
+    if n == 0:
+        return (0.0, 0.0)
+    p = k / n
+    denom = 1 + z * z / n
+    centre = p + z * z / (2 * n)
+    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n))
+    return ((centre - half) / denom, (centre + half) / denom)
+
+
+def corrected_count(
+    flagged: int, unflagged: int,
+    audit_pos_yes: int, audit_pos_n: int,
+    audit_neg_yes: int, audit_neg_n: int,
+) -> dict:
+    """Estimate true corrections from a stratified audit.
+
+    The audit samples separately from messages the model flagged and messages it
+    did not. Precision comes from the flagged sample, the miss rate from the
+    unflagged one, and each is scaled back to its stratum's size:
+
+        true ~= flagged * precision + unflagged * miss_rate
+    """
+    precision = audit_pos_yes / audit_pos_n if audit_pos_n else None
+    miss = audit_neg_yes / audit_neg_n if audit_neg_n else None
+    if precision is None or miss is None:
+        return {"precision": precision, "miss_rate": miss, "estimate": None, "low": None, "high": None}
+    p_lo, p_hi = wilson(audit_pos_yes, audit_pos_n)
+    m_lo, m_hi = wilson(audit_neg_yes, audit_neg_n)
+    est = flagged * precision + unflagged * miss
+    # Conservative bounds: combine the interval ends of both strata.
+    return {
+        "precision": precision,
+        "miss_rate": miss,
+        "estimate": est,
+        "low": flagged * p_lo + unflagged * m_lo,
+        "high": flagged * p_hi + unflagged * m_hi,
+    }

pushback-0.1.0.dist-info/METADATA

@@ -0,0 +1,168 @@
+Metadata-Version: 2.4
+Name: pushback
+Version: 0.1.0
+Summary: How often do you correct your coding agent, and at what kind of work? Measured from your own Claude Code transcripts.
+Author: Intikhab Azam
+License: MIT
+Project-URL: Homepage, https://github.com/intikhab49/pushback
+Project-URL: Repository, https://github.com/intikhab49/pushback
+Project-URL: Issues, https://github.com/intikhab49/pushback/issues
+Keywords: claude-code,coding-agents,ai-agents,llm,claude,claude-md,dpo,preference-data,llm-evaluation
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: anthropic>=1.0
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == "test"
+Dynamic: license-file
+
+# pushback
+
+How often do you correct your coding agent, and on what kind of work?
+
+`pushback` reads your Claude Code transcripts, labels every message you sent
+with Claude, and tells you what share of your messages were corrections,
+broken down by the kind of work in progress (code, writing, media, research,
+ops, meta). A hand-check step tells you how far to trust the labels.
+
+On the author's own 1,633 messages:
+
+| task | messages | corrections | rate | 95% CI |
+|---|---:|---:|---:|---|
+| media | 114 | 49 | 43.0% | 34.3%–52.2% |
+| writing | 429 | 106 | 24.7% | 20.9%–29.0% |
+| research | 188 | 13 | 6.9% | 4.1%–11.5% |
+| code | 340 | 20 | 5.9% | 3.8%–8.9% |
+| meta | 297 | 14 | 4.7% | 2.8%–7.8% |
+| ops | 265 | 12 | 4.5% | 2.6%–7.7% |
+
+Against 91 hand-checked messages the labels had precision 0.97 and recall 0.80.
+That's one person's logs. Run it on yours.
+
+These rates describe a workflow, not a model. The author's code work runs
+through skills, reference files, memory and a plan before the agent writes
+anything, and CI catches mistakes before a human has to. Writing got none of
+that. A low rate means the process around the agent is doing its job.
+
+## Run it
+
+```
+pip install pushback          # or, from a clone: pip install -e .
+pushback extract            # reads ~/.claude/projects, writes ./pushback-data/
+pushback label              # sends your messages to Claude, resumable
+pushback audit              # hand-check 40 messages
+pushback report --markdown  # the table, with the audit folded in
+pushback rules              # draft CLAUDE.md rules from your recurring corrections
+pushback export             # your corrections as prompt/chosen/rejected pairs
+```
+
+`label` uses the Anthropic SDK, so it picks up `ANTHROPIC_API_KEY` or an
+`ant auth login` profile. It defaults to `claude-opus-5` at low effort. Change
+it with `--model`. To send through a gateway, set `ANTHROPIC_BASE_URL`.
+
+## Turn recurring corrections into rules
+
+```
+pushback rules                                  # drafts rules into pushback-data/rules.md
+pushback rules --existing CLAUDE.md notes/*.md  # also check against the rules you already have
+pushback rules --prompt-file                    # no API key? writes the request to a file instead
+pushback rules --from-response reply.json       # ...and reads Claude's answer back
+```
+
+The model groups corrections that share a cause and drafts one CLAUDE.md
+instruction per group. Support is counted from the correction ids it cites,
+not from its own numbers, and a rule needs at least 3 real corrections.
+
+When you pass your existing rule files, the output splits in two:
+
+- **Rules you already have that keep getting broken.** These are the useful
+  ones. A rule that exists and still gets corrected isn't working: it's too
+  vague, or the agent doesn't read it at the right moment.
+- **New rules to consider.** Recurring corrections with no rule behind them.
+
+No API key: `--prompt-file` writes the whole request to `rules-prompt.md`.
+Ask Claude Code to answer it, save the JSON reply, then run `--from-response`.
+
+## Export your corrections as preference pairs
+
+```
+pushback export                                        # all pairs -> pushback-data/dpo.jsonl
+pushback export --ctype writing_content tone_style     # the cleanest pairs
+pushback export --minimal                              # only prompt/chosen/rejected (TRL DPO columns)
+```
+
+Each correction you made becomes one row:
+
+- `prompt`: your message the agent was answering
+- `rejected`: the agent turn you corrected
+- `chosen`: the agent's reply to your correction, kept only if your next message wasn't another correction
+- `feedback`: the correction itself, for critique-and-revise formats
+
+On the author's logs, 214 corrections gave 124 pairs. The biggest loss:
+70 corrections (a third) had their fix corrected too, so there was no
+accepted answer to pair with.
+
+What to know before you train on it:
+
+- "You didn't correct it again" is weak evidence that the fix was good.
+- The fix was written after seeing your feedback. For corrections of a wrong
+  assumption ("I already sent that"), `chosen` answers the correction rather
+  than the prompt. Those make poor DPO pairs, so filter with `--ctype`.
+- Only agent text is exported. File edits and commands are tool calls, so a
+  code correction's pair may hold the explanation without the diff. Use
+  `--max-tools` to drop tool-heavy turns.
+- Secrets are scrubbed on a best-effort basis (API key shapes, tokens,
+  `NAME_KEY=value`). Read the file before you use it.
+- Your transcripts probably contain other people's information. Keep the
+  export local unless every conversation in it is yours to share.
+- If the agent is Claude, Anthropic's terms restrict using its outputs to
+  build competing models. Treat this as a personal dataset or eval set.
+
+## What counts as a correction
+
+A message where you reject, fix or redirect something the agent just did,
+said, wrote or proposed. New tasks, answers to its questions, picking between
+its options, approvals and pasted logs don't count. The full rubric is in
+`src/pushback/prompt.py`. If you change it, run `audit` again.
+
+Rejected tool calls and interrupts carry no text, so they're counted
+separately and bucketed by the action you stopped (a code edit, a shell
+command, and so on).
+
+## Privacy
+
+- `extract` and `report` never leave your machine. `report` prints counts only.
+- `label` sends each message, plus the end of the agent reply before it, to the
+  model provider. If your logs contain client work, check that provider's data
+  policy first. `label` asks before it sends anything.
+- `pushback-data/` holds your raw messages. It's in `.gitignore`. Keep it there.
+
+## Your history is shorter than you think
+
+Claude Code deletes transcripts older than 30 days by default. To keep more,
+set `cleanupPeriodDays` in `~/.claude/settings.json`, for example
+`"cleanupPeriodDays": 365`. It only affects transcripts that still exist.
+
+## How the numbers are computed
+
+- Rates per task come with Wilson 95% intervals.
+- `audit` samples half from messages the model flagged and half from the rest.
+  The report estimates the true count as
+  `flagged × precision + unflagged × miss rate`, with a range built from both
+  strata's intervals.
+- A batch that fails stays unlabelled and is retried on the next run. It's
+  never counted as "not a correction".
+
+## Limits
+
+- Claude Code transcripts only, for now.
+- Task type is judged from the last agent reply and your message, not the whole session.
+- The rubric was tuned on one person's logs.
+
+MIT licensed.

pushback-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+pushback/__init__.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
+pushback/audit.py,sha256=-1K939SLre8IletzqlMts0iGSxNjxJQFOMXnzaszcwI,2249
+pushback/cli.py,sha256=f__k1j3yz9dVmPvrlkNMoGp3F8kFvrEnmtUPKLsQ66U,11163
+pushback/export.py,sha256=PKuzwe6kj2tlyAvplPgSjhx3_WwykU3nI0Ug0hcjrf8,4955
+pushback/extract.py,sha256=j-SS3lyW9H4SlbTDKAoSCTBPyMfvQeTAkizwscUpEGs,5899
+pushback/label.py,sha256=0a9zRAC57725BMPxQPgsNXeRae57-q_uZPkccJ-x9pg,4483
+pushback/prompt.py,sha256=y11dy6oGhv07t8S3WJL2hOOQa6Y2jFowu6v_y0vqvHI,3507
+pushback/report.py,sha256=LM0lBLLEpvBMuKJKjpBL0ahmKlZL0wDTRzauzSQMtC4,3488
+pushback/rules.py,sha256=NvhqFtjLyeleub43D89v6KnjBYrpIKaO6RNGH6EMh6A,6908
+pushback/stats.py,sha256=LgonOTb0d4qbdV7_hiBiy22TzCRvttiLuyjTGuzhs_0,1660
+pushback-0.1.0.dist-info/licenses/LICENSE,sha256=06Uabh9RYOxqqsXpZVspeFKTQ1dOjWytufNmNiG9Qvg,1070
+pushback-0.1.0.dist-info/METADATA,sha256=Y2xjagII8L0hiqjWMRDPKfFpA2oY6iFqMq8QqWLTL3U,7750
+pushback-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+pushback-0.1.0.dist-info/entry_points.txt,sha256=fH1HzcrKGg0OvRNi7Z99ixoQehjS5DoA5ZFfO5MsJCA,47
+pushback-0.1.0.dist-info/top_level.txt,sha256=BYG9bCjnor0P-1h5upRN9oACJaNVSV4mP99DR0aeg00,9
+pushback-0.1.0.dist-info/RECORD,,

pushback-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pushback-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pushback = pushback.cli:main

pushback-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Intikhab Azam
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pushback-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+pushback