doubleblind-audit 0.1.0__py3-none-any.whl

doubleblind/__init__.py

@@ -0,0 +1,3 @@
+"""doubleblind - two layers that cannot see each other's mistakes."""
+
+__version__ = "0.1.0"

doubleblind/__main__.py

@@ -0,0 +1,3 @@
+import sys
+from .cli import main
+sys.exit(main())

doubleblind/cli.py

@@ -0,0 +1,323 @@
+"""Command line: trace numbers, lint a review brief, build a packet."""
+
+from __future__ import annotations
+
+import argparse
+import fnmatch
+import os
+import sys
+
+from . import packet as _packet
+from . import trace as _trace
+
+ALLOW_DEFAULT = "doubleblind-allow.txt"
+
+# Above this many evidence values, coincidental matches stop being negligible.
+POOL_WARN = 2000
+
+
+# --------------------------------------------------------------- allow files
+
+def read_allow(path: str | None, document: str) -> dict[str, str]:
+    """Numbers exempt from tracing in ONE document, each with a stated reason.
+
+    Two rules, both of which exist because the obvious design fails:
+
+    * **A reason is mandatory.** An allow list without reasons becomes the place
+      inconvenient numbers go to stop being checked, and nobody reads it again.
+    * **Every entry names the document it applies to.** A repository-wide list
+      silently disarms the check everywhere: the entry that exempts an
+      illustrative figure in the README also exempted the fabricated figure in a
+      test fixture, and the test that was supposed to see the fixture fail
+      started passing. An exemption is about one claim in one document.
+
+    Format: ``<document glob>  <number>  <reason>``
+    """
+    path = path or ALLOW_DEFAULT
+    if not os.path.exists(path):
+        return {}
+    out: dict[str, str] = {}
+    doc = os.path.normpath(document)
+    base = os.path.basename(doc)
+    with open(path, encoding="utf-8") as fh:
+        lines = fh.readlines()
+    for i, line in enumerate(lines, 1):
+        line = line.split("#", 1)[0].rstrip()
+        if not line.strip():
+            continue
+        parts = line.split(None, 2)
+        if len(parts) < 3 or not parts[2].strip():
+            sys.exit(f"{path}:{i}: each line needs <document> <number> <reason> "
+                     f"and the reason may not be empty -> {line.strip()!r}")
+        pattern, number, reason = parts[0], parts[1], parts[2].strip()
+        if fnmatch.fnmatch(doc, pattern) or fnmatch.fnmatch(base, pattern):
+            out[number] = reason
+    return out
+
+
+# ----------------------------------------------------------------- reporting
+
+def _c(s: str, code: str, on: bool) -> str:
+    return f"\033[{code}m{s}\033[0m" if on else s
+
+
+def cmd_trace(args: argparse.Namespace) -> int:
+    colour = sys.stdout.isatty() and not args.no_color
+    with open(args.document, encoding="utf-8", errors="replace") as fh:
+        prose = fh.read()
+    evidence = _trace.load_evidence(args.data)
+    evidence += _trace.run_derivations(args.derive or [])
+    if not evidence:
+        print(f"no numbers found in {args.data} - nothing to trace against", file=sys.stderr)
+        return 2
+    allow = read_allow(args.allow, args.document)
+    findings = _trace.trace(prose, evidence, allow)
+
+    bad = [f for f in findings if f.verdict == "unsupported"]
+    ok = [f for f in findings if f.verdict == "traced"]
+    allowed = [f for f in findings if f.verdict == "allowed"]
+
+    for f in bad:
+        n = f.number
+        print(_c(f"  UNSUPPORTED  {n.raw}", "31;1", colour) +
+              f"   {args.document}:{n.line_no}")
+        print(f"      {n.line[:100]}")
+        if f.nearest:
+            near = ", ".join(f"{v.value:g} ({os.path.basename(v.source)} {v.where})"
+                             for v in f.nearest)
+            print(f"      nearest committed values: {near}")
+    if args.verbose:
+        for f in ok:
+            print(f"  ok           {f.number.raw:<12} <- {os.path.basename(f.matched.source)} "
+                  f"{f.matched.where}{'' if f.scale == 'exact' else f.scale}")
+        for f in allowed:
+            print(f"  allowed      {f.number.raw:<12} {f.reason}")
+
+    vague = _trace.vague_quantities(prose)
+    if vague and not args.no_vague:
+        print()
+        for line_no, phrase, line in vague:
+            print(_c(f"  UNCHECKABLE  \"{phrase}\"", "33;1", colour) +
+                  f"   {args.document}:{line_no}")
+            print(f"      {line[:100]}")
+            print("      A quantity written as words cannot be traced to a file. "
+                  "Write the number.")
+
+    rescaled = sum(1 for f in ok if f.scale == " x100")
+    print()
+    print(f"{len(findings)} numbers in {args.document}; {len(ok)} traced"
+          + (f" ({rescaled} of them only after reading a stored fraction as a percentage)"
+             if rescaled else "")
+          + f", {len(allowed)} allowed, {len(bad)} unsupported.")
+    print(f"Evidence: {len(evidence)} values from "
+          f"{len(set(v.source for v in evidence))} sources.")
+    if len(evidence) > POOL_WARN:
+        print()
+        print(_c(f"  NOTE  {len(evidence)} values is a large pool.", "33;1", colour))
+        print("      Raw per-item dumps contain every id, index and token count, so a")
+        print("      round number in prose will match one of them by coincidence and be")
+        print("      reported as traced. Point --data at summary files and use --derive")
+        print("      for quantities a script recomputes, or this check gets weaker the")
+        print("      more data you give it.")
+
+    failed = bool(bad) or (bool(vague) and args.strict)
+    if not failed:
+        print()
+        print("Every number here exists in a committed file. That is all this says.")
+        print("It does not say the sentences around them are true: a correct number")
+        print("inside a claim that does not follow from it passes this check every")
+        print("time. Run the reviewer layer - `doubleblind review` - for that.")
+    return 1 if failed else 0
+
+
+def cmd_lint(args: argparse.Namespace) -> int:
+    colour = sys.stdout.isatty() and not args.no_color
+    with open(args.brief, encoding="utf-8", errors="replace") as fh:
+        text = fh.read()
+    leaks = _packet.intent_leaks(text)
+    by_mech: dict[str, list] = {}
+    for l in leaks:
+        by_mech.setdefault(l.mechanism, []).append(l)
+    for mech in sorted(by_mech):
+        print(_c(f"  {mech}", "33;1", colour))
+        for l in by_mech[mech]:
+            print(f"      {args.brief}:{l.line_no}  \"{l.phrase}\"")
+            print(f"      {l.line[:100]}")
+        print(f"      -> {by_mech[mech][0].fix}")
+        print()
+    if leaks:
+        print(f"{len(leaks)} phrase(s) in this brief tell the reviewer what to conclude.")
+        print("A reviewer that knows the wanted answer is not a second opinion.")
+        return 1
+    print("No intent leakage found. The brief asks a question that cannot be")
+    print("answered by agreeing.")
+    return 0
+
+
+def cmd_packet(args: argparse.Namespace) -> int:
+    text = _packet.build_packet(args.document, args.data, )
+    if args.output == "-":
+        sys.stdout.write(text)
+    else:
+        with open(args.output, "w", encoding="utf-8") as fh:
+            fh.write(text)
+        # The full digest, not a prefix: this line is what a blindness record
+        # quotes, and a truncated hash is a guard against a slip rather than a
+        # commitment to what the reviewer was given.
+        print(f"packet: {args.output}  ({len(text)} bytes)")
+        print(f"sha256  {_packet.packet_fingerprint(text)}")
+    return 0
+
+
+ADAPTERS = {
+    "claude": (
+        "Claude Code",
+        "Agent(\n"
+        "    subagent_type='general-purpose',\n"
+        "    model='<a model that is NOT the one that wrote the artifact>',\n"
+        "    run_in_background=False,\n"
+        "    prompt=open('{packet}').read(),\n"
+        ")\n"
+        "# A sub-agent starts with no conversation history by construction, so\n"
+        "# zero context needs no extra work. Pick a different model explicitly:\n"
+        "# same model with a fresh context still carries the priors that wrote it.",
+    ),
+    "codex": (
+        "Codex CLI",
+        "codex exec --skip-git-repo-check < {packet}\n"
+        "# `exec` starts a fresh session. Do not use `codex resume`.",
+    ),
+    "deepseek": (
+        "DeepSeek",
+        "curl -s https://api.deepseek.com/chat/completions \\\n"
+        "  -H \"Authorization: Bearer $DEEPSEEK_API_KEY\" -H 'Content-Type: application/json' \\\n"
+        "  -d \"$(jq -Rs '{{model:\"deepseek-reasoner\",messages:[{{role:\"user\",content:.}}]}}' {packet})\"\n"
+        "# One request, one message. Sending history is what you are avoiding.",
+    ),
+    "kimi": (
+        "Kimi / Moonshot",
+        "curl -s https://api.moonshot.cn/v1/chat/completions \\\n"
+        "  -H \"Authorization: Bearer $MOONSHOT_API_KEY\" -H 'Content-Type: application/json' \\\n"
+        "  -d \"$(jq -Rs '{{model:\"kimi-k2-turbo-preview\",messages:[{{role:\"user\",content:.}}]}}' {packet})\"",
+    ),
+    "generic": (
+        "any OpenAI-compatible endpoint",
+        "curl -s \"$OPENAI_BASE_URL/chat/completions\" \\\n"
+        "  -H \"Authorization: Bearer $OPENAI_API_KEY\" -H 'Content-Type: application/json' \\\n"
+        "  -d \"$(jq -Rs '{{model:\"'$MODEL'\",messages:[{{role:\"user\",content:.}}]}}' {packet})\"",
+    ),
+}
+
+
+def cmd_review(args: argparse.Namespace) -> int:
+    text = _packet.build_packet(args.document, args.data)
+    out = args.output
+    with open(out, "w", encoding="utf-8") as fh:
+        fh.write(text)
+    fp = _packet.packet_fingerprint(text)
+    name, cmd = ADAPTERS[args.agent]
+    print(f"packet: {out}  ({len(text)} bytes)")
+    print(f"sha256  {fp}")
+    print()
+    print(f"Send it with {name}:")
+    print()
+    for line in cmd.format(packet=out).splitlines():
+        print(f"    {line}")
+    print()
+    print("Then, before you believe the verdict, record two things next to it:")
+    print("  1. which model answered - it must not be the one that wrote the artifact;")
+    print(f"  2. this packet's sha256 - {fp}")
+    print("Without both, 'a different model checked it' is a claim about a")
+    print("conversation nobody can inspect.")
+    return 0
+
+
+def cmd_render(args: argparse.Namespace) -> int:
+    """Run a figure script and audit what it leaves in `fig`."""
+    import runpy
+
+    from . import render as _render
+
+    ns = runpy.run_path(args.script)
+    fig = ns.get("fig")
+    if fig is None:
+        for v in ns.values():
+            if type(v).__name__ == "Figure":
+                fig = v
+                break
+    if fig is None:
+        print(f"{args.script} leaves no Figure at module level.\n"
+              "  Assign the figure to `fig` so it can be audited after the script runs.",
+              file=sys.stderr)
+        return 2
+    problems = _render.audit(fig, scale=args.scale, min_ratio=args.min_ratio)
+    if not problems:
+        print()
+        print("Nothing in this figure is geometrically wrong at that scale. That is")
+        print("all this says: whether the shape a reader takes from it is the shape")
+        print("the data supports is not a thing any of these rules can see.")
+    return 1 if problems else 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(
+        prog="doubleblind",
+        description="Two layers that cannot see each other's mistakes.")
+    ap.add_argument("--no-color", action="store_true")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    def add(name, help):
+        # --no-color is accepted on either side of the subcommand; people type
+        # it where it reads naturally, not where argparse prefers it.
+        sp = sub.add_parser(name, help=help)
+        sp.add_argument("--no-color", action="store_true")
+        return sp
+
+    t = add("trace", "every number in a document must exist in a committed file")
+    t.add_argument("document")
+    t.add_argument("--data", nargs="+", required=True, help="result files or directories")
+    t.add_argument("--allow", help=f"exemptions with reasons (default: {ALLOW_DEFAULT})")
+    t.add_argument("--derive", action="append", metavar="CMD",
+                   help="a committed command whose printed numbers also count as "
+                        "evidence; repeatable")
+    t.add_argument("--strict", action="store_true", help="fail on word-quantities too")
+    t.add_argument("--no-vague", action="store_true", help="do not report word-quantities")
+    t.add_argument("-v", "--verbose", action="store_true")
+    t.set_defaults(func=cmd_trace)
+
+    l = add("lint", "find phrases that tell the reviewer what to conclude")
+    l.add_argument("brief")
+    l.set_defaults(func=cmd_lint)
+
+    p = add("packet", "assemble artifact + evidence into one review packet")
+    p.add_argument("document", nargs="+")
+    p.add_argument("--data", nargs="*", default=[])
+    p.add_argument("-o", "--output", default="packet.md")
+    p.set_defaults(func=cmd_packet)
+
+    g = add("render", "audit a figure for what a reader sees and a number-checker cannot")
+    g.add_argument("script", help="a python script that leaves its figure in `fig`")
+    g.add_argument("--scale", type=float, default=0.30,
+                   help="fraction of drawn size the figure will be seen at "
+                        "(0.30 = a link unfurl, 1.0 = read at full size)")
+    g.add_argument("--min-ratio", type=float, default=4.5,
+                   help="WCAG contrast required of body text")
+    g.set_defaults(func=cmd_render)
+
+    r = add("review", "build the packet and print how to send it")
+    r.add_argument("document", nargs="+")
+    r.add_argument("--data", nargs="*", default=[])
+    r.add_argument("--agent", choices=sorted(ADAPTERS), default="claude")
+    r.add_argument("-o", "--output", default="packet.md")
+    r.set_defaults(func=cmd_review)
+
+    args = ap.parse_args(argv)
+    for attr in ("no_color", "strict", "no_vague", "verbose", "allow", "derive",
+                 "scale", "min_ratio"):
+        if not hasattr(args, attr):
+            setattr(args, attr, False if attr not in ("allow", "derive") else None)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

doubleblind/packet.py

@@ -0,0 +1,209 @@
+"""Assemble a review packet, and refuse to send one that gives the answer away.
+
+A reviewer with no context is only useful while it stays without context. The
+way that property is lost is almost never dramatic - nobody pastes the
+conversation. It is lost in the request itself:
+
+    "Confirm that all four estimators fall below the card number."
+
+There is exactly one way to answer that and still sound useful, and the model
+takes it. The same request without its answer -
+
+    "For each numeric claim, state what these files actually support."
+
+- cannot be answered by agreeing, because it does not contain anything to agree
+  with.
+
+So this module does two things. It builds a packet out of the artifact and the
+evidence it rests on, and it reads the instructions you were about to send and
+tells you which phrases have the conclusion in them. The second is the part
+that matters; the first is bookkeeping.
+
+The patterns below are not a style guide. Each is a way a request stops being a
+question, grouped by the mechanism rather than the wording, because the wording
+changes and the mechanism does not.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+from dataclasses import dataclass
+
+__all__ = ["Leak", "intent_leaks", "build_packet", "packet_fingerprint", "REVIEWER_BRIEF"]
+
+
+@dataclass
+class Leak:
+    line_no: int
+    phrase: str
+    mechanism: str
+    line: str
+    fix: str
+
+
+# (regex, mechanism, suggested rewrite)
+_PATTERNS: list[tuple[str, str, str]] = [
+    # --- the request contains its own answer -------------------------------
+    (r"\b(?:please\s+)?(?:confirm|verify|validate|make sure|ensure)\s+(?:that|the|these|this|all|it)\b",
+     "answer-in-the-request",
+     "Ask what the evidence supports, not whether a stated thing is true: "
+     "'state what these files support' instead of 'confirm that X'."),
+    (r"(?:^|(?<=[.!?])\s+|\bplease\s+|\bcan you\s+|\bcould you\s+)check\s+that\b",
+     "answer-in-the-request",
+     "'Check that X' presumes X. 'Check whether X' does not - or better, ask "
+     "what the evidence gives."),
+    (r"\bdouble[- ]check\b",
+     "answer-in-the-request",
+     "'Double-check' presumes a first check that passed. Ask for a first check."),
+    (r"\b(?:is|are|does|do|was|were)\s+(?:this|that|it|the\s+\w+)\s+(?:correct|right|accurate|fine|ok|okay)\b\s*\?",
+     "answer-in-the-request",
+     "A yes/no question about correctness invites yes. Ask for the derivation instead."),
+
+    # --- the conclusion is stated before the review ------------------------
+    (r"\bwe\s+(?:show|find|demonstrate|prove|establish|conclude)\b",
+     "conclusion-stated",
+     "Remove the claim from the brief. The artifact can state it; the request must not."),
+    (r"\b(?:as|which is)\s+expected\b",
+     "conclusion-stated",
+     "Delete. It tells the reviewer which outcome is the acceptable one."),
+    (r"\bthis\s+(?:demonstrates|proves|confirms|establishes|shows)\b",
+     "conclusion-stated",
+     "Delete; let the reviewer decide what it demonstrates."),
+    (r"\bthe\s+(?:result|answer|number|conclusion)\s+should\s+be\b",
+     "conclusion-stated",
+     "Stating the expected value makes disagreement a contradiction of you."),
+
+    # --- polarity steering --------------------------------------------------
+    (r"\b(?:should|ought to|is supposed to|must)\s+(?:be|show|give|match|agree)\b",
+     "polarity-steering",
+     "Replace the expectation with the question: 'what do the files give?'"),
+    (r"\b(?:obviously|clearly|of course|naturally|needless to say)\b",
+     "polarity-steering",
+     "These words mark a claim as not-up-for-review. Remove them."),
+    (r"\b(?:minor|small|cosmetic|trivial)\s+(?:issue|problem|change|fix)\b",
+     "polarity-steering",
+     "Pre-sizing a problem tells the reviewer how seriously to take it."),
+
+    # --- social pressure ----------------------------------------------------
+    (r"\bI\s+(?:already\s+)?(?:checked|verified|confirmed|tested|reviewed)\b",
+     "social-pressure",
+     "Saying you checked makes a finding an accusation. Leave it out."),
+    (r"\bI'?m\s+(?:fairly\s+|pretty\s+|quite\s+)?(?:sure|confident|certain)\b",
+     "social-pressure",
+     "Stating your confidence sets the bar a finding has to clear."),
+    (r"\b(?:quick|brief|fast|light)\s+(?:sanity\s+)?(?:check|look|pass|review)\b",
+     "social-pressure",
+     "Asking for a quick look licenses a shallow one."),
+    (r"\b(?:just|only)\s+(?:need|want)\s+(?:you\s+)?to\b",
+     "social-pressure",
+     "Minimising the task lowers the effort it gets."),
+
+    # --- scope narrowing ----------------------------------------------------
+    (r"\b(?:ignore|skip|don'?t worry about|no need to (?:check|look at))\b",
+     "scope-narrowing",
+     "Exclusions hide exactly where the problem is. If something is out of "
+     "scope, remove it from the packet instead of naming it."),
+    (r"\b(?:only|just)\s+(?:look at|check|review|consider)\b",
+     "scope-narrowing",
+     "Narrowing the scope in the brief pre-decides where the error is not."),
+
+    # --- authorship leak ----------------------------------------------------
+    (r"\b(?:my|our)\s+(?:readme|paper|repo|repository|result|analysis|code|figure|draft)\b",
+     "authorship-leak",
+     "Ownership words make the reviewer a guest in your work. Refer to it as "
+     "'the attached document'."),
+    (r"\bI\s+(?:wrote|made|built|produced|ran)\b",
+     "authorship-leak",
+     "Same: the reviewer should not know who produced the artifact."),
+]
+
+_COMPILED = [(re.compile(p, re.I), mech, fix) for p, mech, fix in _PATTERNS]
+
+
+def intent_leaks(text: str) -> list[Leak]:
+    """Phrases in a review request that tell the reviewer what to conclude."""
+    out: list[Leak] = []
+    lines = text.splitlines()
+    for i, line in enumerate(lines, 1):
+        if line.lstrip().startswith(">"):
+            continue  # quoted material is the artifact, not the request
+        for rx, mech, fix in _COMPILED:
+            for m in rx.finditer(line):
+                out.append(Leak(i, m.group(0).strip(), mech, line.strip(), fix))
+    return out
+
+
+REVIEWER_BRIEF = """\
+You are reviewing an attached document against the data files attached with it.
+
+You have not seen this work before and you are not being told what it concluded.
+Do not try to infer what answer is wanted; there is no wanted answer.
+
+For every numeric claim and every causal or comparative claim in the document:
+
+1. Name the file and field that would settle it.
+2. State what that file actually contains.
+3. Say whether the sentence in the document follows from it - not whether the
+   number matches, but whether the sentence follows. A correct number inside a
+   sentence that does not follow from it is the most common defect here and the
+   one you are most needed for.
+
+Report each finding as:
+
+    <file>:<line>  CLAIM: "<quoted sentence>"
+                   EVIDENCE: <what the data says>
+                   COMMAND: <one shell command that settles it>
+
+A finding with no command is an opinion. Drop it or turn it into one.
+
+If a claim is supported, say so and move on. Do not pad the report, and do not
+soften a finding because the rest of the document is careful.
+"""
+
+
+def packet_fingerprint(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def build_packet(artifacts: list[str], evidence: list[str], brief: str | None = None,
+                 max_bytes: int = 400_000) -> str:
+    """Return a single self-contained packet: brief, artifacts, evidence.
+
+    Nothing about how the artifact came to exist goes in - no commit messages,
+    no changelog, no summary of what was tried. Those are the parts that carry
+    intent, and they are the reason this function takes explicit paths rather
+    than a repository.
+    """
+    parts = [brief if brief is not None else REVIEWER_BRIEF, ""]
+
+    def _emit(path: str, kind: str) -> None:
+        try:
+            with open(path, encoding="utf-8", errors="replace") as fh:
+                raw = fh.read()
+        except OSError as exc:
+            parts.append(f"<<{kind} {path}: unreadable: {exc}>>")
+            return
+        if len(raw) > max_bytes:
+            raw = raw[:max_bytes] + f"\n<<truncated at {max_bytes} bytes>>\n"
+        parts.append(f"===== {kind}: {path} =====")
+        parts.append(raw.rstrip())
+        parts.append("")
+
+    files: list[str] = []
+    for p in evidence:
+        if os.path.isdir(p):
+            for root, _d, names in os.walk(p):
+                if any(part.startswith(".") and part not in (".", "..")
+                       for part in root.split(os.sep)):
+                    continue
+                files += [os.path.join(root, n) for n in sorted(names) if not n.startswith(".")]
+        else:
+            files.append(p)
+
+    for a in artifacts:
+        _emit(a, "DOCUMENT UNDER REVIEW")
+    for f in sorted(set(files)):
+        _emit(f, "DATA FILE")
+    return "\n".join(parts)