ms408 0.1.0__py3-none-any.whl

ms408/__init__.py

@@ -0,0 +1,18 @@
+"""MS408 research program: corpus pipeline, validation harness, statistics.
+
+All reported numbers must come from scripts in this package writing to results/ (L3 firewall).
+
+Public evaluator entry point:
+
+    from ms408 import evaluate
+    verdict = evaluate(open("my_tokens.txt").read().split())
+
+`evaluate(tokens)` scores a word-token stream against the Voynich manuscript's
+discriminator bands and returns a per-axis verdict with each axis's honest caveat
+attached. Matching is necessary, not sufficient (L7). See `ms408.signature` and
+docs/LIMITS.md.
+"""
+
+from .signature import axis_values, evaluate, format_verdict, vms_bands
+
+__all__ = ["evaluate", "axis_values", "vms_bands", "format_verdict"]

ms408/__main__.py

@@ -0,0 +1,45 @@
+"""CLI: evaluate a token stream against the VMS discriminator bands.
+
+    python -m ms408 my_tokens.txt          # whitespace-separated word tokens
+    python -m ms408 --json my_tokens.txt   # machine-readable verdict
+    cat my_tokens.txt | python -m ms408 -  # read tokens from stdin
+
+The verdict carries each axis's value, the VMS reference band, whether you land in it, and
+the standing caveat for that axis. Matching is NECESSARY, not sufficient (L7): an in-band
+result means your hypothesis is not excluded, not that it is the manuscript's mechanism.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from .signature import evaluate, format_verdict
+
+
+def main(argv: list | None = None) -> int:
+    ap = argparse.ArgumentParser(
+        prog="ms408",
+        description="Evaluate a token stream against the Voynich discriminator bands.",
+    )
+    ap.add_argument("tokens", help="path to a whitespace-separated token file, or '-' for stdin")
+    ap.add_argument("--json", action="store_true", help="emit the raw verdict as JSON")
+    ap.add_argument("--seed", type=int, default=408, help="determinism seed (default 408)")
+    args = ap.parse_args(argv)
+
+    text = sys.stdin.read() if args.tokens == "-" else open(args.tokens, encoding="utf-8").read()
+    tokens = text.split()
+    if len(tokens) < 2:
+        ap.error("need at least 2 whitespace-separated tokens")
+
+    verdict = evaluate(tokens, seed=args.seed)
+    if args.json:
+        print(json.dumps(verdict, indent=2))
+    else:
+        print(format_verdict(verdict))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ms408/acquire.py

@@ -0,0 +1,64 @@
+"""Fetch and verify pinned external sources (T0.2).
+
+Usage:
+    python -m ms408.acquire            # fetch anything missing, verify everything
+    python -m ms408.acquire zl gc      # fetch/verify named sources only
+    python -m ms408.acquire --verify   # verify only, no network
+"""
+
+from __future__ import annotations
+
+import hashlib
+import sys
+import time
+import urllib.request
+
+from .sources import RAW_ROOT, SOURCES, Source, path_for
+
+FETCH_DELAY_S = 1.0  # politeness between requests to the same host
+
+
+def sha256_of(path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def fetch(source: Source, force: bool = False) -> str:
+    """Download one source if missing (or force), then verify. Returns status."""
+    dest = RAW_ROOT / source.dest
+    if dest.exists() and not force:
+        return verify(source)
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    req = urllib.request.Request(source.url, headers={"User-Agent": "ms408-research/0.1"})
+    with urllib.request.urlopen(req) as resp, open(dest, "wb") as out:
+        out.write(resp.read())
+    time.sleep(FETCH_DELAY_S)
+    return verify(source)
+
+
+def verify(source: Source) -> str:
+    dest = RAW_ROOT / source.dest
+    if not dest.exists():
+        return "missing"
+    return "ok" if sha256_of(dest) == source.sha256 else "CHECKSUM MISMATCH"
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = list(sys.argv[1:] if argv is None else argv)
+    verify_only = "--verify" in args
+    names = [a for a in args if not a.startswith("--")] or list(SOURCES)
+    failed = False
+    for name in names:
+        source = SOURCES[name]
+        status = verify(source) if verify_only else fetch(source)
+        print(f"{name:14s} {status:18s} {path_for(name)}")
+        if status != "ok":
+            failed = True
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ms408/annotate/__init__.py

@@ -0,0 +1,6 @@
+"""T1.3 bulk illustration annotation (schema v0.1-coarse, L31).
+
+Firewall note (L3/L1): the model produces descriptive feature values; it does
+not compute any program statistic. Annotations feed the anchor hunt (T2.3),
+which computes co-occurrence statistics in deterministic code.
+"""

ms408/annotate/pipeline.py

@@ -0,0 +1,223 @@
+"""T1.3 bulk annotation runner (schema v0.1-coarse; budget cap L28 = $100).
+
+Sonnet 4.6 does vision annotation via strict tool use (guaranteed schema-valid
+output — replaces the Haiku format-validation step, which strict mode makes
+redundant). Fable 5 QA is a separate module (qa.py). Cost is tracked per call
+and the run aborts before exceeding the cap.
+
+Resumable: pages already in the output JSONL are skipped, so a re-run continues.
+
+Usage:
+    python -m ms408.annotate.pipeline --limit 5        # pilot
+    python -m ms408.annotate.pipeline --section H      # one section
+    python -m ms408.annotate.pipeline                  # all illustrated pages
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+
+from ..dataset import git_commit
+from ..env import require
+from ..ivtff import IVTFFDocument
+from ..scans import SCANS_ROOT
+from ..sources import path_for
+from .schema import SCHEMA_VERSION, SECTION_BLOCKS, tool_schema
+
+ROOT = Path(__file__).resolve().parents[3]
+RESULTS_DIR = ROOT / "results" / "annotations"
+OUTPUT = RESULTS_DIR / "t13_annotations.jsonl"
+MANIFEST = RESULTS_DIR / "t13_manifest.json"
+
+ANNOTATOR_MODEL = "claude-sonnet-4-6"
+BUDGET_CAP_USD = 100.0
+# Sonnet 4.6 pricing, $/token
+PRICE_IN = 3.0 / 1_000_000
+PRICE_OUT = 15.0 / 1_000_000
+
+SYSTEM = (
+    "You are a careful manuscript-illustration annotator for a scholarly project on "
+    "Beinecke MS 408 (the Voynich Manuscript). You describe what is drawn using a fixed "
+    "controlled vocabulary. Strict rules:\n"
+    "- Describe MORPHOLOGY only. Never identify a plant species, a zodiac sign, or a "
+    "real-world referent. 'A branched root', never 'a mandrake'. 'An animal', never 'a bull'.\n"
+    "- Judge only from what is visibly drawn. When the scan does not support a confident "
+    "call, choose 'unclear' rather than guessing.\n"
+    "- The manuscript's pigments are faded; read colors conservatively.\n"
+    "- Fill every field of the annotate_page tool. Put any ambiguity in 'notes' (brief, "
+    "descriptive, never identificational)."
+)
+
+
+def illustrated_pages(section: str | None = None) -> list:
+    zl = IVTFFDocument.load(path_for("zl"))
+    pages = []
+    for page in zl.pages:
+        code = page.illustration_type
+        if code is None or code not in SECTION_BLOCKS:
+            continue
+        if section and code != section:
+            continue
+        pages.append((page.name, code))
+    return pages
+
+
+MAX_EDGE_PX = 7000  # API rejects any image dimension > 8000px
+# API's 10 MB cap is on the base64-ENCODED size; base64 inflates ~4/3, so keep
+# the raw JPEG under ~7.3 MB (encoded ~9.8 MB, safely under 10,485,760)
+MAX_RAW_BYTES = 7_300_000
+
+
+def _image_block(path: Path) -> dict:
+    raw = path.read_bytes()
+    if max(_dimensions(path)) <= MAX_EDGE_PX and len(raw) <= MAX_RAW_BYTES:
+        data = base64.standard_b64encode(raw).decode()
+        return {"type": "image",
+                "source": {"type": "base64", "media_type": "image/jpeg", "data": data}}
+
+    import io
+
+    from PIL import Image
+
+    with Image.open(path) as img:
+        img = img.convert("RGB")
+        # cap the long edge, then shrink further if still over the byte budget
+        scale = min(1.0, MAX_EDGE_PX / max(img.size))
+        for _ in range(8):
+            w, h = round(img.width * scale), round(img.height * scale)
+            buffer = io.BytesIO()
+            img.resize((w, h), Image.LANCZOS).save(buffer, format="JPEG", quality=88)
+            raw = buffer.getvalue()
+            if len(raw) <= MAX_RAW_BYTES:
+                break
+            scale *= 0.85
+    data = base64.standard_b64encode(raw).decode()
+    return {"type": "image",
+            "source": {"type": "base64", "media_type": "image/jpeg", "data": data}}
+
+
+def _dimensions(path: Path) -> tuple:
+    from PIL import Image
+
+    with Image.open(path) as img:
+        return img.size
+
+
+def _load_scan_map() -> dict:
+    return json.loads((ROOT / "data" / "processed" / "scan_map.json").read_text())["pages"]
+
+
+def _done_pages() -> set:
+    if not OUTPUT.exists():
+        return set()
+    return {json.loads(line)["page"] for line in OUTPUT.read_text().splitlines() if line.strip()}
+
+
+def annotate_page(client, page_name: str, code: str, scan_map: dict) -> dict:
+    entry = scan_map.get(page_name, {})
+    files = entry.get("files", [])
+    tool = {
+        "name": "annotate_page",
+        "description": "Record the coarse descriptive annotation for this manuscript page.",
+        "strict": True,
+        "input_schema": tool_schema(code),
+    }
+    content = [_image_block(SCANS_ROOT / f) for f in files]
+    section_name = SECTION_BLOCKS[code][0] or "text/stars (common block only)"
+    foldout_note = (
+        "Multiple scan tiles are provided for this foldout panel; annotate the panel "
+        "as a whole. " if len(files) > 1 else ""
+    )
+    content.append({"type": "text", "text":
+                    f"This is folio {page_name}, illustration section '{section_name}'. "
+                    f"{foldout_note}Call annotate_page with your descriptive annotation."})
+    response = client.messages.create(
+        model=ANNOTATOR_MODEL,
+        max_tokens=2000,
+        system=SYSTEM,
+        tools=[tool],
+        tool_choice={"type": "tool", "name": "annotate_page"},
+        messages=[{"role": "user", "content": content}],
+    )
+    features = next(b.input for b in response.content if b.type == "tool_use")
+    cost = (response.usage.input_tokens * PRICE_IN
+            + response.usage.output_tokens * PRICE_OUT)
+    record = {
+        "schema_version": SCHEMA_VERSION,
+        "page": page_name,
+        "section": code,
+        "scan_tiles": files,
+        "foldout_ambiguous": entry.get("ambiguous", False),
+        "annotator_model": ANNOTATOR_MODEL,
+        "common": {k: features[k] for k in features if k in _COMMON_KEYS},
+        "section_features": {k: v for k, v in features.items()
+                             if k not in _COMMON_KEYS and k != "notes"},
+        "notes": features.get("notes", ""),
+        "qa": {"reviewed": False, "reviewer_model": None, "disagreements": []},
+        "_cost_usd": round(cost, 5),
+        "_input_tokens": response.usage.input_tokens,
+        "_output_tokens": response.usage.output_tokens,
+    }
+    return record
+
+
+_COMMON_KEYS = {"illustration_coverage_pct", "text_image_relationship", "color_palette"}
+
+
+def run(section: str | None = None, limit: int | None = None) -> dict:
+    import anthropic
+
+    require("ANTHROPIC_API_KEY")
+    client = anthropic.Anthropic()
+    RESULTS_DIR.mkdir(parents=True, exist_ok=True)
+    scan_map = _load_scan_map()
+    done = _done_pages()
+    pages = [(n, c) for n, c in illustrated_pages(section) if n not in done]
+    if limit:
+        pages = pages[:limit]
+
+    spent = sum(json.loads(line).get("_cost_usd", 0)
+                for line in OUTPUT.read_text().splitlines()) if OUTPUT.exists() else 0.0
+    annotated = 0
+    with open(OUTPUT, "a") as out:
+        for page_name, code in pages:
+            if spent > BUDGET_CAP_USD:
+                print(f"BUDGET CAP ${BUDGET_CAP_USD} reached (${spent:.2f}); stopping.")
+                break
+            record = annotate_page(client, page_name, code, scan_map)
+            out.write(json.dumps(record) + "\n")
+            out.flush()
+            spent += record["_cost_usd"]
+            annotated += 1
+            print(f"{page_name:8s} {code}  ${record['_cost_usd']:.4f}  "
+                  f"(total ${spent:.3f})")
+
+    manifest = {
+        "schema_version": SCHEMA_VERSION,
+        "built_at": datetime.now(UTC).isoformat(timespec="seconds"),
+        "git_commit": git_commit(),
+        "annotator_model": ANNOTATOR_MODEL,
+        "budget_cap_usd": BUDGET_CAP_USD,
+        "pages_annotated_total": len(_done_pages()),
+        "spent_usd_total": round(spent, 4),
+    }
+    MANIFEST.write_text(json.dumps(manifest, indent=2) + "\n")
+    return manifest
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description="T1.3 bulk annotation")
+    parser.add_argument("--section", choices=sorted(SECTION_BLOCKS))
+    parser.add_argument("--limit", type=int)
+    args = parser.parse_args(argv)
+    manifest = run(section=args.section, limit=args.limit)
+    print(json.dumps(manifest, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ms408/annotate/qa.py

@@ -0,0 +1,204 @@
+"""T1.3 QA — Fable 5 independent re-annotation of a sample, drift scoring (L31).
+
+Per WORKFLOW §5: Fable 5 re-annotates a random 12% sample (min 5/batch, fixed
+seed) blind; agreement is scored per field with the schema-derived metric
+(exact for enum/bool/count-band, Jaccard≥0.67 for multi, ±1 for counts). Batch
+fails if overall disagreement > 0.20, critical-field disagreement > 0.15, or any
+single field disagrees on > 40% of sampled pages (L31 provisional thresholds).
+
+Fable 5 never sees the Sonnet annotation — it produces an independent one from
+the same scans, so agreement measures reproducibility, not self-consistency.
+
+Usage:
+    python -m ms408.annotate.qa
+"""
+
+from __future__ import annotations
+
+import json
+import random
+from datetime import UTC, datetime
+
+from ..dataset import git_commit
+from ..env import require
+from ..scans import SCANS_ROOT
+from .pipeline import (
+    OUTPUT,
+    RESULTS_DIR,
+    SYSTEM,
+    _COMMON_KEYS,
+    _image_block,
+)
+from .schema import SECTION_BLOCKS, critical_fields, fields_for, tool_schema
+
+QA_MODEL = "claude-fable-5"
+QA_FALLBACK = "claude-opus-4-8"  # Fable false-positive-refuses manuscript annotation
+QA_SAMPLE_FRACTION = 0.12
+QA_SAMPLE_MIN = 5
+SEED = 408
+QA_OUTPUT = RESULTS_DIR / "t13_qa.json"
+
+# ratified at G2 (2026-07-06, L32): the provisional 0.20/0.15 bands assumed
+# same-model QA; these reflect real cross-model (Sonnet vs Opus/Fable) variance
+# on coarse morphological calls. root_type/leaf_arrangement are known-noisy
+# (~0.35, irreducible perceptual ambiguity) and pass single-field at 0.40.
+THRESH_OVERALL = 0.25
+THRESH_CRITICAL = 0.25
+THRESH_SINGLE_FIELD = 0.40
+
+# billed at whichever model served the call; fallback credit reprices refusals
+PRICE = {"claude-fable-5": (10.0e-6, 50.0e-6),
+         "claude-opus-4-8": (5.0e-6, 25.0e-6)}
+
+
+def _agree(kind: str, a, b) -> bool:
+    if kind == "multi":
+        sa, sb = set(a or []), set(b or [])
+        if not sa and not sb:
+            return True
+        union = sa | sb
+        # threshold is exactly 2/3 (the schema's "0.67" rounding), so a 2-of-3
+        # overlap counts as agreement rather than failing by a rounding hair
+        return len(sa & sb) / len(union) >= 2 / 3 if union else True
+    if a is None or b is None:
+        return a == b  # a genuinely absent field counts as a disagreement
+    if kind == "count":
+        return abs(int(a) - int(b)) <= 1
+    return a == b  # enum, bool, count-band (enum strings)
+
+
+def score_page(section: str, sonnet: dict, fable: dict) -> dict:
+    fields = fields_for(section)
+    critical = set(critical_fields(section))
+    disagreements = []
+    critical_disagreements = []
+    merged_a = {**sonnet["common"], **sonnet["section_features"]}
+    merged_b = {**fable["common"], **fable["section_features"]}
+    for field in fields:
+        if not _agree(field.kind, merged_a.get(field.name), merged_b.get(field.name)):
+            disagreements.append(field.name)
+            if field.name in critical:
+                critical_disagreements.append(field.name)
+    return {
+        "scored_fields": len(fields),
+        "disagreements": disagreements,
+        "critical_disagreements": critical_disagreements,
+        "page_disagreement_rate": round(len(disagreements) / len(fields), 4),
+    }
+
+
+def reannotate(client, record: dict) -> tuple:
+    code = record["section"]
+    tool = {"name": "annotate_page", "description": "Record the coarse descriptive "
+            "annotation for this manuscript page.", "strict": True,
+            "input_schema": tool_schema(code)}
+    content = [_image_block(SCANS_ROOT / f) for f in record["scan_tiles"]]
+    section_name = SECTION_BLOCKS[code][0] or "text/stars (common block only)"
+    content.append({"type": "text", "text":
+                    f"This is folio {record['page']}, illustration section "
+                    f"'{section_name}'. Call annotate_page with your descriptive annotation."})
+    # Fable 5 false-positive-refuses manuscript annotation; server-side fallback
+    # transparently re-serves the refusal on Opus 4.8 (an independent reviewer)
+    response = client.beta.messages.create(
+        model=QA_MODEL, max_tokens=3000, system=SYSTEM, tools=[tool],
+        tool_choice={"type": "tool", "name": "annotate_page"},
+        betas=["server-side-fallback-2026-06-01"],
+        fallbacks=[{"model": QA_FALLBACK}],
+        messages=[{"role": "user", "content": content}],
+    )
+    served_by = response.model
+    if response.stop_reason == "refusal":
+        return None, 0.0, served_by  # whole chain refused; page skipped in QA
+    features = next((b.input for b in response.content if b.type == "tool_use"), {})
+    fable = {
+        "common": {k: features[k] for k in features if k in _COMMON_KEYS},
+        "section_features": {k: v for k, v in features.items()
+                             if k not in _COMMON_KEYS and k != "notes"},
+    }
+    price_in, price_out = PRICE.get(served_by, PRICE[QA_FALLBACK])
+    cost = response.usage.input_tokens * price_in + response.usage.output_tokens * price_out
+    return fable, cost, served_by
+
+
+def run() -> dict:
+    import anthropic
+    from collections import Counter
+
+    require("ANTHROPIC_API_KEY")
+    client = anthropic.Anthropic()
+    records = [json.loads(line) for line in OUTPUT.read_text().splitlines() if line.strip()]
+
+    # stratified sample: at least QA_SAMPLE_MIN, ~12%, seeded, spread across sections
+    rng = random.Random(SEED)
+    by_section: dict = {}
+    for r in records:
+        by_section.setdefault(r["section"], []).append(r)
+    sample = []
+    for section, group in sorted(by_section.items()):
+        k = max(1, round(len(group) * QA_SAMPLE_FRACTION))
+        sample.extend(rng.sample(group, min(k, len(group))))
+    if len(sample) < QA_SAMPLE_MIN:
+        remaining = [r for r in records if r not in sample]
+        sample.extend(rng.sample(remaining, min(QA_SAMPLE_MIN - len(sample), len(remaining))))
+
+    scored, field_disagreements, spent = [], Counter(), 0.0
+    served_by = Counter()
+    refused = []
+    for record in sample:
+        fable, cost, model = reannotate(client, record)
+        spent += cost
+        served_by[model] += 1
+        if fable is None:
+            refused.append(record["page"])
+            continue
+        result = score_page(record["section"], record, fable)
+        result.update({"page": record["page"], "section": record["section"],
+                       "qa_served_by": model})
+        field_disagreements.update(result["disagreements"])
+        scored.append(result)
+        print(f"{record['page']:8s} {record['section']}  {model:16s} "
+              f"disagree={result['page_disagreement_rate']:.2f}  "
+              f"critical={len(result['critical_disagreements'])}")
+
+    total_fields = sum(s["scored_fields"] for s in scored)
+    total_disagree = sum(len(s["disagreements"]) for s in scored)
+    total_critical_fields = sum(len(critical_fields(s["section"])) for s in scored)
+    total_critical_disagree = sum(len(s["critical_disagreements"]) for s in scored)
+    worst_field, worst_count = (field_disagreements.most_common(1) or [(None, 0)])[0]
+
+    overall = total_disagree / total_fields if total_fields else 0.0
+    critical_rate = (total_critical_disagree / total_critical_fields
+                     if total_critical_fields else 0.0)
+    worst_single = worst_count / len(sample) if sample else 0.0
+    passed = (overall <= THRESH_OVERALL and critical_rate <= THRESH_CRITICAL
+              and worst_single <= THRESH_SINGLE_FIELD)
+
+    report = {
+        "built_at": datetime.now(UTC).isoformat(timespec="seconds"),
+        "git_commit": git_commit(),
+        "qa_model": QA_MODEL,
+        "qa_served_by": dict(served_by),
+        "qa_refused_pages": refused,
+        "sampled_pages": len(sample),
+        "scored_pages": len(scored),
+        "sample_seed": SEED,
+        "overall_disagreement_rate": round(overall, 4),
+        "critical_disagreement_rate": round(critical_rate, 4),
+        "worst_field": worst_field,
+        "worst_field_rate": round(worst_single, 4),
+        "thresholds": {"overall": THRESH_OVERALL, "critical": THRESH_CRITICAL,
+                       "single_field": THRESH_SINGLE_FIELD},
+        "batch_passed": passed,
+        "field_disagreement_counts": dict(field_disagreements.most_common()),
+        "per_page": scored,
+        "qa_cost_usd": round(spent, 4),
+    }
+    RESULTS_DIR.mkdir(parents=True, exist_ok=True)
+    QA_OUTPUT.write_text(json.dumps(report, indent=2) + "\n")
+    return report
+
+
+if __name__ == "__main__":
+    report = run()
+    print(json.dumps({k: v for k, v in report.items()
+                      if k not in ("per_page", "field_disagreement_counts")}, indent=2))