sdm-learn 0.1.0__py3-none-any.whl

sdm/__init__.py

@@ -0,0 +1,77 @@
+"""Self-documenting models: the document *is* the model.
+
+There are no weights and no classifier head. A document holds the learned
+hypothesis, an LLM reads it to classify inputs, and then rewrites it from the
+revealed labels. What you train is readable, and what you read is what runs.
+
+Four components plug into one interface, one file each, and the imports only
+point downward:
+
+    gateway.py   the wire to an LLM, and SetupNeeded
+    data.py      the dataset, the input representation, row rendering
+    formats.py   how the hypothesis is written down and revised
+    signals.py   the audio codec, imported only on demand
+    learn.py     run artifacts, the gate, the baseline, train / evaluate
+
+``import sdm`` is the only entry point: everything public is re-exported here,
+so callers never see the split.
+
+The gate algorithm runs this loop per epoch::
+
+    predict the feedback batches from the current document  (labels hidden)
+    reveal the labels and ask for one bounded revision
+    score the current and revised documents head to head on protected rows
+    keep the revision only if it wins
+
+``state.json`` is the truth; the skill file is rendered from it.
+
+    import sdm
+
+    data = sdm.dataset(
+        train=[{"text": "wire me $500 today", "label": "spam"},
+               {"text": "lunch at one?", "label": "ham"}],
+        classes=("spam", "ham"),
+        description="Each input is one email body.",
+    )
+    skill = sdm.train(data, algo="sdm")
+    print(skill)
+"""
+
+from __future__ import annotations
+
+from . import datasets, science
+from .data import (REPRESENTATIONS, Dataset, Report, content, dataset,
+                   describe, from_jsonl, labeled)
+from .formats import (DEFAULT_FORMAT_MD, PACKAGED_FORMAT_FILE, apply_edits, clean_condition, codecs,
+                      credit, format_file, formats, get_format, load_formats,
+                      new_state, parse_edits, parse_predictions, render,
+                      validate_html, validate_svg, validate_text,
+                      write_format_file)
+from .gateway import SetupNeeded
+from .learn import (ALGOS, GATE_SCORING, Skill, epoch_points, decide,
+                    evaluate, load, parse_paired, proposal_history_text,
+                    split_holdout, stratified_holdout, train)
+from .experiment import run_experiment
+from .prauc import average_precision, evaluate_prauc
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # the API
+    "Dataset", "Report", "Skill", "SetupNeeded",
+    "dataset", "datasets", "science", "from_jsonl", "load", "train", "evaluate",
+    "run_experiment", "evaluate_prauc", "average_precision",
+    # the four components
+    "REPRESENTATIONS", "describe",
+    "formats", "get_format", "load_formats", "DEFAULT_FORMAT_MD",
+    "format_file", "write_format_file", "PACKAGED_FORMAT_FILE",
+    "ALGOS", "GATE_SCORING", "codecs",
+    # the pieces the README and the checks reach for
+    "render", "new_state", "parse_edits", "apply_edits", "clean_condition",
+    "parse_predictions", "parse_paired", "credit",
+    "validate_text", "validate_html", "validate_svg",
+    "stratified_holdout", "split_holdout", "epoch_points", "decide",
+    "proposal_history_text",
+    "content", "labeled",
+    "__version__",
+]

sdm/__main__.py

@@ -0,0 +1,58 @@
+"""``python -m sdm``, and the ``sdm`` console script.
+
+Small on purpose. The library is the product; this exists so a pip-installed
+user can get an editable ``format.md``, confirm the install works without an
+API key, and run the shipped benchmarks.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from . import __version__, format_file, formats, write_format_file
+from .gateway import SetupNeeded
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="sdm", description="Self-documenting models.")
+    parser.add_argument("--version", action="version",
+                        version=f"sdm {__version__}")
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    initialize = commands.add_parser(
+        "init", help="write an editable format.md into this directory")
+    initialize.add_argument("--force", action="store_true",
+                            help="overwrite an existing format.md")
+
+    commands.add_parser(
+        "formats", help="list the skill formats format.md defines")
+
+    from . import benchmarks
+    benchmarks.add_arguments(commands.add_parser(
+        "run", help="run a shipped benchmark: --dataset, --algo, --format"))
+
+    options = parser.parse_args(argv)
+    try:
+        if options.command == "run":
+            return benchmarks.run(options)
+
+        if options.command == "init":
+            path = write_format_file(force=options.force)
+            print(f"wrote {path}")
+            print("Edit the 'format:' line to change the skill format.")
+            return 0
+
+        active = format_file()
+        print(f"reading {active}")
+        for name, skill_format in sorted(formats().items()):
+            print(f"  {name:10s} mechanism: {skill_format.mechanism}")
+        return 0
+    except SetupNeeded as error:
+        print(f"sdm: {error}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sdm/benchmarks.py

@@ -0,0 +1,221 @@
+"""The shipped benchmarks, and the one command that runs them.
+
+    sdm run --dataset pods --algo sdm --format html
+    sdm run --dataset data/mine          # train.jsonl and test.jsonl inside
+
+Each benchmark is a loader plus the gate settings its protocol calls for. The
+command line only overrides those defaults; the lifecycle (zero-training
+control, gated training, evaluation, resume, result files) is the shared
+:func:`sdm.run_experiment`. PODS and DF2 add the original paper's one-vs-all
+PR-AUC after top-1 accuracy. A directory of JSONL rows is a benchmark too,
+with the generic defaults.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from . import datasets, science
+from .data import from_jsonl
+from .experiment import run_experiment
+from .learn import ALGOS, GATE_SCORING, load
+from .prauc import evaluate_prauc
+
+
+@dataclass(frozen=True)
+class Benchmark:
+    """One dataset and the training settings its protocol uses."""
+
+    load: object
+    protocol: str
+    settings: dict = field(default_factory=dict)
+    train: int | None = None
+    test: int | None = None
+    image_size: int | None = None
+    prauc: bool = False
+
+
+PERSONAL = {"holdout_per_class": 1, "epochs": 7, "gate_scoring": "separate",
+            "proposal_history": True, "max_semantic_edits": 60}
+SCIENCE = {"epochs": 4}
+
+BENCHMARKS = {
+    "cifar10": Benchmark(
+        load=lambda o: datasets.cifar10(o.train, o.test, root=o.data_dir or "data/cifar"),
+        protocol="direct RGB images, 10 classes",
+        settings={"holdout_per_class": 1, "epochs": 5,
+                  "max_semantic_edits": 100, "max_support_edits": 100},
+        train=100, test=100),
+    "pods": Benchmark(
+        load=lambda o: datasets.pods("test", o.test, image_size=o.image_size,
+                                     root=o.data_dir or "data/pods"),
+        protocol="PODS paper class split: 70 test objects, 3 training photos "
+                 "each, test photos per object from the full test split",
+        settings=PERSONAL, test=10, image_size=512, prauc=True),
+    "df2": Benchmark(
+        load=lambda o: datasets.df2("train", o.test, image_size=o.image_size,
+                                    val_per_class=0,
+                                    root=o.data_dir or "data/personal/df2/df2"),
+        protocol="PODS paper protocol on DF2: 139 test shirts, 3 training "
+                 "photos each, test photos per shirt",
+        settings=PERSONAL, test=10, image_size=512, prauc=True),
+    "camelyon17": Benchmark(
+        load=lambda o: datasets.camelyon17(
+            o.train, o.test, image_size=o.image_size, seed=o.seed,
+            root=o.data_dir or "data/wilds/camelyon17"),
+        protocol="WILDS in-distribution training patches, official "
+                 "out-of-distribution test patches",
+        settings={"holdout_per_class": 10, "epochs": 4,
+                  "gate_scoring": "separate"},
+        train=100, test=500, image_size=288),
+    # The four science benchmarks of reproduce/ai4science. Their reported
+    # numbers were produced with the SLMD-dev learner (see that README); here
+    # they run under the generic gate settings.
+    "gravityspy": Benchmark(
+        load=lambda o: science.gravityspy(
+            o.train, o.test, image_size=o.image_size, seed=o.seed,
+            root=o.data_dir or "data/gravityspy"),
+        protocol="Gravity Spy v1.0 official train / test samples, the ten "
+                 "largest glitch classes, four-window spectrograms with a Hz "
+                 "ruler and time axis",
+        settings=SCIENCE, train=300, test=400, image_size=412),
+    "galaxy10": Benchmark(
+        load=lambda o: science.galaxy10(
+            o.train, o.test, image_size=o.image_size, seed=o.seed,
+            root=o.data_dir or "data/galaxy10"),
+        protocol="Galaxy10 DECaLS colour cutouts, ten Galaxy Zoo morphology "
+                 "classes, class-balanced seeded draw (no official split)",
+        settings=SCIENCE, train=100, test=100, image_size=224),
+    "bloodmnist": Benchmark(
+        load=lambda o: science.bloodmnist(
+            o.train, o.test, image_size=o.image_size, seed=o.seed,
+            root=o.data_dir or "data/bloodmnist"),
+        protocol="BloodMNIST 224 px official train / test split, eight "
+                 "peripheral blood cell types",
+        settings=SCIENCE, train=300, test=160, image_size=224),
+    "eclipsing": Benchmark(
+        load=lambda o: science.eclipsing(
+            o.train, o.test, seed=o.seed, root=o.data_dir or "data/ogle3"),
+        protocol="OGLE-III LMC eclipsing binaries as order-4 Fourier "
+                 "coefficients of the folded I-band light curve; contact / "
+                 "semidetached / detached",
+        settings=SCIENCE, train=60, test=150),
+}
+
+
+def custom(path: Path) -> Benchmark:
+    """A directory holding ``train.jsonl`` and ``test.jsonl`` as a benchmark.
+
+    Rows need a ``label`` and ``text`` or ``image`` (paths relative to the
+    file); classes are the labels seen. ``--description`` tells the model
+    what one input is.
+    """
+    train, test = path / "train.jsonl", path / "test.jsonl"
+    for file in (train, test):
+        if not file.is_file():
+            raise FileNotFoundError(
+                f"{file} is missing; a custom --dataset is a directory with "
+                "train.jsonl and test.jsonl")
+    with open(train) as handle:
+        first = json.loads(next(handle))
+    representation = "image" if "image" in first else "text"
+
+    def load(options):
+        return from_jsonl(train, test, limit=options.train,
+                          test_limit=options.test, name=path.name,
+                          representation=representation,
+                          description=options.description or "",
+                          **({"image_size": options.image_size}
+                             if options.image_size else {}))
+
+    return Benchmark(load=load, protocol=f"JSONL rows under {path}",
+                     settings={"epochs": 4})
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--dataset", required=True,
+                        help=f"one of {', '.join(sorted(BENCHMARKS))}, or a "
+                             "directory with train.jsonl and test.jsonl")
+    parser.add_argument("--description",
+                        help="custom dataset: what one input is")
+    parser.add_argument("--algo", choices=tuple(ALGOS), default="sdm",
+                        help="sdm: learn a document under the gate; "
+                             "base: the zero-training control")
+    parser.add_argument("--format", default="markdown",
+                        help="a section of format.md: markdown, text, html, "
+                             "image, audio, or your own")
+    parser.add_argument("--style",
+                        help="what the document should contain, in words, "
+                             "e.g. \"decision tree, tables, illustrations\"")
+    parser.add_argument("--train", type=int, help="training rows")
+    parser.add_argument("--test", type=int,
+                        help="test rows (PODS, DF2: per object)")
+    parser.add_argument("--image-size", type=int, help="pixels per side")
+    parser.add_argument("--batch-size", type=int, default=10)
+    parser.add_argument("--epochs", type=int, help="gated revisions")
+    parser.add_argument("--holdout-per-class", type=int,
+                        help="rows per class the gate protects")
+    parser.add_argument("--gate-scoring", choices=GATE_SCORING)
+    parser.add_argument("--max-semantic-edits", type=int,
+                        help="rule edits one candidate may make")
+    parser.add_argument("--no-proposal-history", action="store_true",
+                        help="hide earlier candidates' verdicts from the writer")
+    parser.add_argument("--no-baseline", action="store_true",
+                        help="skip the zero-training control")
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument("--data-dir", help="where the dataset lives")
+    parser.add_argument("--run-dir",
+                        help="default: runs/<dataset>-<algo>-<format>")
+
+
+def settings_for(benchmark: Benchmark, options) -> dict:
+    """The benchmark's defaults, with explicit command-line values winning."""
+    settings = dict(benchmark.settings)
+    for key in ("epochs", "holdout_per_class", "gate_scoring",
+                "max_semantic_edits"):
+        value = getattr(options, key, None)
+        if value is not None:
+            settings[key] = value
+    if options.no_proposal_history:
+        settings.pop("proposal_history", None)
+    if options.style:
+        settings["style"] = options.style
+    return settings
+
+
+def run(options) -> int:
+    if options.dataset in BENCHMARKS:
+        benchmark = BENCHMARKS[options.dataset]
+    elif Path(options.dataset).is_dir():
+        benchmark = custom(Path(options.dataset))
+    else:
+        raise SystemExit(f"sdm run: unknown dataset {options.dataset!r}; choose "
+                         f"{', '.join(sorted(BENCHMARKS))} or a directory with "
+                         "train.jsonl and test.jsonl")
+    name = Path(options.dataset).name
+    for key in ("train", "test", "image_size"):
+        if getattr(options, key) is None:
+            setattr(options, key, getattr(benchmark, key))
+    data = benchmark.load(options)
+    run_dir = options.run_dir or f"runs/{name}-{options.algo}-{options.format}"
+    settings = settings_for(benchmark, options)
+    print(f"{name}: {len(data.classes)} classes, {len(data.train)} "
+          f"training rows, {len(data.test)} test rows; {benchmark.protocol}")
+    print(f"algo={options.algo}; format={options.format}", flush=True)
+    run_experiment(
+        data, run_dir=run_dir, algo=options.algo, format=options.format,
+        batch_size=options.batch_size,
+        run_baseline=options.algo == "sdm" and not options.no_baseline,
+        metadata={"protocol": benchmark.protocol, "seed": options.seed},
+        seed=options.seed, **settings)
+    if benchmark.prauc:
+        skill = load(run_dir)
+        result = evaluate_prauc(skill, data, run_dir=run_dir, tag="prauc")
+        print(f"PR-AUC {100 * result['mean_prauc']:.1f}, top-1 from the same "
+              f"scores {100 * result['top1_from_scores']:.1f}% "
+              f"({result['unparsed_inputs']} unparsed inputs); saved to "
+              f"{run_dir}/prauc.json")
+    return 0

sdm/data.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .gateway import _png_block, image_block
+
+
+DEFAULT_TEXT_DESCRIPTION = (
+    "Each input is one example rendered as text. Infer the meaning of its "
+    "fields from the labeled examples you are shown."
+)
+DEFAULT_IMAGE_DESCRIPTION = (
+    "Each input is one image. Infer what distinguishes the classes from the "
+    "labeled examples you are shown."
+)
+
+
+class TextRepresentation:
+    """Rows are serialized into the prompt text.
+
+    ``inline`` picks which half of this interface the prompts consult:
+    ``serialize`` for a representation that becomes text, ``blocks`` for one
+    that becomes attachments. Never both.
+    """
+
+    name = "text"
+    inline = True
+    default_description = DEFAULT_TEXT_DESCRIPTION
+
+    @staticmethod
+    def serialize(row: dict) -> str:
+        if "text" in row:
+            return str(row["text"])
+        return "\n".join(
+            f"{key}={value}" for key, value in sorted(row.items())
+            if key not in {"id", "label", "index", "image"})
+
+    @staticmethod
+    def blocks(rows: list[dict], tag: str, image_size: int) -> list:
+        return []
+
+
+class ImageRepresentation(TextRepresentation):
+    """Rows are attached as images, one content block each."""
+
+    name = "image"
+    inline = False
+    default_description = DEFAULT_IMAGE_DESCRIPTION
+
+    @staticmethod
+    def blocks(rows: list[dict], tag: str, image_size: int) -> list:
+        attached = []
+        for number, row in enumerate(rows, 1):
+            attached.append({"type": "text", "text": f"{tag} P{number:02d}"})
+            attached.append(image_block(row["image"], image_size))
+        return attached
+
+
+REPRESENTATIONS = {"text": TextRepresentation, "image": ImageRepresentation}
+
+
+# --------------------------------------------------------------------------
+
+@dataclass
+class Dataset:
+    """Labeled rows plus the little context the model needs to read them.
+
+    Every row needs a ``label``. Its input is ``text``, or ``image`` (a PIL
+    image, an RGB array, or a path), or any other scalar fields.
+    """
+
+    train: list[dict] = field(default_factory=list)
+    test: list[dict] = field(default_factory=list)
+    classes: tuple[str, ...] = ()
+    name: str = "custom"
+    title: str = ""
+    description: str = ""
+    representation: str = "text"
+    image_size: int = 224
+    modality: str | None = None
+
+    def __post_init__(self):
+        self.train = [dict(row) for row in self.train]
+        self.test = [dict(row) for row in self.test]
+        if self.modality is not None:
+            self.representation = self.modality
+        self.modality = self.representation
+        if not self.classes:
+            self.classes = tuple(sorted(
+                {str(row["label"]) for row in self.train + self.test
+                 if "label" in row}
+            ))
+        self.classes = tuple(str(label) for label in self.classes)
+        if self.representation not in REPRESENTATIONS:
+            raise ValueError(
+                f"unknown representation {self.representation!r}; choose one "
+                f"of {', '.join(sorted(REPRESENTATIONS))}")
+        if self.image_size <= 0:
+            raise ValueError("image_size must be positive")
+        if not self.title:
+            self.title = f"{self.name} skill"
+        for rows in (self.train, self.test):
+            for position, row in enumerate(rows):
+                row.setdefault("index", position)
+
+    def __len__(self):
+        return len(self.train)
+
+    def __repr__(self):
+        return (f"Dataset(name={self.name!r}, train={len(self.train)}, "
+                f"test={len(self.test)}, classes={len(self.classes)})")
+
+
+
+@dataclass
+class Report:
+    """The outcome of scoring a frozen skill."""
+
+    correct: int
+    total: int
+    rows: list[dict] = field(default_factory=list)
+    requests: int = 0
+
+    @property
+    def accuracy(self) -> float:
+        return self.correct / self.total if self.total else 0.0
+
+    def __repr__(self):
+        return (f"Report(accuracy={self.accuracy:.1%}, "
+                f"{self.correct}/{self.total}, requests={self.requests})")
+
+
+
+def dataset(train, test=(), **options) -> Dataset:
+    """Wrap in-memory labeled rows into a :class:`Dataset`."""
+    return Dataset(train=list(train), test=list(test), **options)
+
+
+def from_jsonl(train, test=None, *, limit=None, test_limit=None,
+               **options) -> Dataset:
+    """Read JSONL rows; relative ``image`` paths resolve from each file."""
+    wants_images = (options.get("representation") == "image"
+                    or options.get("modality") == "image")
+
+    def read(path, cap):
+        path = Path(path)
+        if not path.exists():
+            raise FileNotFoundError(f"missing {path}")
+        with open(path) as handle:
+            rows = [json.loads(line) for line in handle]
+        rows = rows if cap is None else rows[:cap]
+        if wants_images:
+            for row in rows:
+                image = row.get("image")
+                if isinstance(image, str) and not Path(image).is_absolute():
+                    row["image"] = str((path.parent / image).resolve())
+        return rows
+
+    return Dataset(train=read(train, limit),
+                   test=read(test, test_limit) if test else [], **options)
+
+def describe(data: Dataset) -> str:
+    if data.description:
+        return data.description
+    return REPRESENTATIONS[data.representation].default_description
+
+
+def labeled(label: str, pngs) -> list[tuple[str, bytes]]:
+    """Tag each rendering of one document so the model can tell them apart."""
+    return [(label, png) for png in pngs]
+
+
+def content(prompt: str, rows: list[dict], representation: str, tag: str,
+            image_size: int, views=()):
+    """Prompt text, the skill's renderings, then the inputs.
+
+    ``views`` holds ``(label, png)`` pairs; each label is numbered separately
+    so a two-document gate prompt reads A 01, A 02, B 01 rather than 01..04.
+    """
+    attached = REPRESENTATIONS[representation].blocks(rows, tag, image_size)
+    if not attached and not views:
+        return prompt
+    blocks = [{"type": "text", "text": prompt}]
+    counts: dict[str, int] = {}
+    for label, png in views:
+        counts[label] = counts.get(label, 0) + 1
+        blocks.append({"type": "text",
+                       "text": f"{label} {counts[label]:02d}"})
+        blocks.append(_png_block(png))
+    return blocks + attached
+
+
+def _inputs(rows: list[dict], representation: str, tag: str = "input") -> str:
+    if not REPRESENTATIONS[representation].inline:
+        return ""
+    serialize = REPRESENTATIONS[representation].serialize
+    return "\n\n".join(
+        f'<{tag} id="P{number:02d}">\n{serialize(row)}\n</{tag}>'
+        for number, row in enumerate(rows, 1))
+
+
+def _revealed(rows: list[dict], predictions: list[dict], representation: str,
+              cites_rules: bool = True) -> str:
+    """The labeled feedback block: truth, prediction, citations, evidence."""
+    def cited(prediction, attribute: bool):
+        if not cites_rules:
+            return ""
+        joined = ",".join(prediction["rules_used"]) or "none"
+        return f' rules="{joined}"' if attribute else f"rules={joined}; "
+
+    if not REPRESENTATIONS[representation].inline:
+        return "\n".join(
+            f'P{number:02d}: true={row["label"]}; '
+            f'predicted={prediction["pred"]}; '
+            f'{cited(prediction, False)}'
+            f'evidence={prediction["evidence"]}'
+            for number, (row, prediction)
+            in enumerate(zip(rows, predictions), 1))
+    serialize = REPRESENTATIONS[representation].serialize
+    return "\n\n".join(
+        f'<example id="P{number:02d}" true="{row["label"]}" '
+        f'predicted="{prediction["pred"]}"{cited(prediction, True)}>\n'
+        f'{serialize(row)}\n'
+        f'<evidence>{prediction["evidence"]}</evidence>\n</example>'
+        for number, (row, prediction) in enumerate(zip(rows, predictions), 1))