sdm-learn 0.1.0.dev0__py3-none-any.whl

sdm/__init__.py

@@ -0,0 +1,71 @@
+"""Self-documenting models: the document *is* the model.
+
+There are no weights and no classifier head. A document holds the learned
+hypothesis, an LLM reads it to classify inputs, and then rewrites it from the
+revealed labels. What you train is readable, and what you read is what runs.
+
+Four components plug into one interface, one file each, and the imports only
+point downward:
+
+    gateway.py   the wire to an LLM, and SetupNeeded
+    data.py      the dataset, the input representation, row rendering
+    formats.py   how the hypothesis is written down and revised
+    signals.py   the audio and video codecs, imported only on demand
+    learn.py     run artifacts, the gate, the baseline, train / evaluate
+
+``import sdm`` is the only entry point: everything public is re-exported here,
+so callers never see the split.
+
+The gate algorithm runs this loop per cycle::
+
+    predict the feedback batches from the current document  (labels hidden)
+    reveal the labels and ask for one bounded revision
+    score the current and revised documents head to head on protected rows
+    keep the revision only if it wins
+
+``state.json`` is the truth; the skill file is rendered from it.
+
+    import sdm
+
+    data = sdm.dataset(
+        train=[{"text": "wire me $500 today", "label": "spam"},
+               {"text": "lunch at one?", "label": "ham"}],
+        classes=("spam", "ham"),
+        description="Each input is one email body.",
+    )
+    skill = sdm.train(data, algo="gate")
+    print(skill)
+"""
+
+from __future__ import annotations
+
+from .data import (REPRESENTATIONS, Dataset, Report, content, dataset,
+                   describe, from_jsonl, labeled)
+from .formats import (DEFAULT_FORMAT_MD, PACKAGED_FORMAT_FILE, apply_edits, clean_condition, codecs,
+                      credit, format_file, formats, get_format, load_formats,
+                      new_state, parse_edits, parse_predictions, render,
+                      validate_html, validate_svg, validate_text,
+                      write_format_file)
+from .gateway import SetupNeeded
+from .learn import (ALGOS, Skill, cycle_points, decide, evaluate, load,
+                    parse_paired, stratified_holdout, train)
+
+__version__ = "0.1.0.dev0"
+
+__all__ = [
+    # the API
+    "Dataset", "Report", "Skill", "SetupNeeded",
+    "dataset", "from_jsonl", "load", "train", "evaluate",
+    # the four components
+    "REPRESENTATIONS", "describe",
+    "formats", "get_format", "load_formats", "DEFAULT_FORMAT_MD",
+    "format_file", "write_format_file", "PACKAGED_FORMAT_FILE",
+    "ALGOS", "codecs",
+    # the pieces the README and the checks reach for
+    "render", "new_state", "parse_edits", "apply_edits", "clean_condition",
+    "parse_predictions", "parse_paired", "credit",
+    "validate_text", "validate_html", "validate_svg",
+    "stratified_holdout", "cycle_points", "decide",
+    "content", "labeled",
+    "__version__",
+]

sdm/__main__.py

@@ -0,0 +1,51 @@
+"""``python -m sdm``, and the ``sdm`` console script.
+
+Small on purpose. The library is the product; this exists so a pip-installed
+user can get an editable ``format.md`` and confirm the install works without
+an API key.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from . import __version__, format_file, formats, write_format_file
+from .gateway import SetupNeeded
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="sdm", description="Self-documenting models.")
+    parser.add_argument("--version", action="version",
+                        version=f"sdm {__version__}")
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    initialize = commands.add_parser(
+        "init", help="write an editable format.md into this directory")
+    initialize.add_argument("--force", action="store_true",
+                            help="overwrite an existing format.md")
+
+    commands.add_parser(
+        "formats", help="list the skill formats format.md defines")
+
+    options = parser.parse_args(argv)
+    try:
+        if options.command == "init":
+            path = write_format_file(force=options.force)
+            print(f"wrote {path}")
+            print("Edit the 'format:' line to change the skill format.")
+            return 0
+
+        active = format_file()
+        print(f"reading {active}")
+        for name, skill_format in sorted(formats().items()):
+            print(f"  {name:10s} mechanism: {skill_format.mechanism}")
+        return 0
+    except SetupNeeded as error:
+        print(f"sdm: {error}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sdm/data.py

@@ -0,0 +1,238 @@
+"""The data half: what a row is, how it reaches the model, what comes back.
+
+Two of the four components live here. The **input representation** decides how
+one labeled row becomes prompt material: serialized into the text, or attached
+as an image. The **dataset** is just rows with labels, plus the prose that
+tells the model what a row means.
+
+The row-rendering helpers at the bottom belong to the representation, not to
+any skill format: every prompt in the project builds its input blocks through
+them, so a new representation changes every prompt at once.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .gateway import _png_block, image_block
+
+
+DEFAULT_TEXT_DESCRIPTION = (
+    "Each input is one example rendered as text. Infer the meaning of its "
+    "fields from the labeled examples you are shown."
+)
+DEFAULT_IMAGE_DESCRIPTION = (
+    "Each input is one image. Infer what distinguishes the classes from the "
+    "labeled examples you are shown."
+)
+
+
+class TextRepresentation:
+    """Rows are serialized into the prompt text.
+
+    ``inline`` picks which half of this interface the prompts consult:
+    ``serialize`` for a representation that becomes text, ``blocks`` for one
+    that becomes attachments. Never both.
+    """
+
+    name = "text"
+    inline = True
+    default_description = DEFAULT_TEXT_DESCRIPTION
+
+    @staticmethod
+    def serialize(row: dict) -> str:
+        if "text" in row:
+            return str(row["text"])
+        return "\n".join(
+            f"{key}={value}" for key, value in sorted(row.items())
+            if key not in {"id", "label", "index", "image"})
+
+    @staticmethod
+    def blocks(rows: list[dict], tag: str, image_size: int) -> list:
+        return []
+
+
+class ImageRepresentation(TextRepresentation):
+    """Rows are attached as images, one content block each."""
+
+    name = "image"
+    inline = False
+    default_description = DEFAULT_IMAGE_DESCRIPTION
+
+    @staticmethod
+    def blocks(rows: list[dict], tag: str, image_size: int) -> list:
+        attached = []
+        for number, row in enumerate(rows, 1):
+            attached.append({"type": "text", "text": f"{tag} P{number:02d}"})
+            attached.append(image_block(row["image"], image_size))
+        return attached
+
+
+REPRESENTATIONS = {"text": TextRepresentation, "image": ImageRepresentation}
+
+
+# --------------------------------------------------------------------------
+
+@dataclass
+class Dataset:
+    """Labeled rows plus the little context the model needs to read them.
+
+    Every row needs a ``label``. Its input is ``text``, or ``image`` (a PIL
+    image, an RGB array, or a path), or any other scalar fields.
+    """
+
+    train: list[dict] = field(default_factory=list)
+    test: list[dict] = field(default_factory=list)
+    classes: tuple[str, ...] = ()
+    name: str = "custom"
+    title: str = ""
+    description: str = ""
+    representation: str = "text"
+    image_size: int = 224
+    modality: str | None = None
+
+    def __post_init__(self):
+        self.train = [dict(row) for row in self.train]
+        self.test = [dict(row) for row in self.test]
+        if self.modality is not None:
+            self.representation = self.modality
+        self.modality = self.representation
+        if not self.classes:
+            self.classes = tuple(sorted(
+                {str(row["label"]) for row in self.train + self.test
+                 if "label" in row}
+            ))
+        self.classes = tuple(str(label) for label in self.classes)
+        if self.representation not in REPRESENTATIONS:
+            raise ValueError(
+                f"unknown representation {self.representation!r}; choose one "
+                f"of {', '.join(sorted(REPRESENTATIONS))}")
+        if self.image_size <= 0:
+            raise ValueError("image_size must be positive")
+        if not self.title:
+            self.title = f"{self.name} skill"
+        for rows in (self.train, self.test):
+            for position, row in enumerate(rows):
+                row.setdefault("index", position)
+
+    def __len__(self):
+        return len(self.train)
+
+    def __repr__(self):
+        return (f"Dataset(name={self.name!r}, train={len(self.train)}, "
+                f"test={len(self.test)}, classes={len(self.classes)})")
+
+
+@dataclass
+class Report:
+    """The outcome of scoring a frozen skill."""
+
+    correct: int
+    total: int
+    rows: list[dict] = field(default_factory=list)
+    requests: int = 0
+
+    @property
+    def accuracy(self) -> float:
+        return self.correct / self.total if self.total else 0.0
+
+    def __repr__(self):
+        return (f"Report(accuracy={self.accuracy:.1%}, "
+                f"{self.correct}/{self.total}, requests={self.requests})")
+
+
+
+def dataset(train, test=(), **options) -> Dataset:
+    """Wrap in-memory labeled rows into a :class:`Dataset`."""
+    return Dataset(train=list(train), test=list(test), **options)
+
+
+def from_jsonl(train, test=None, *, limit=None, test_limit=None,
+               **options) -> Dataset:
+    """Read JSONL rows; relative ``image`` paths resolve from each file."""
+    wants_images = (options.get("representation") == "image"
+                    or options.get("modality") == "image")
+
+    def read(path, cap):
+        path = Path(path)
+        if not path.exists():
+            raise FileNotFoundError(f"missing {path}")
+        with open(path) as handle:
+            rows = [json.loads(line) for line in handle]
+        rows = rows if cap is None else rows[:cap]
+        if wants_images:
+            for row in rows:
+                image = row.get("image")
+                if isinstance(image, str) and not Path(image).is_absolute():
+                    row["image"] = str((path.parent / image).resolve())
+        return rows
+
+    return Dataset(train=read(train, limit),
+                   test=read(test, test_limit) if test else [], **options)
+
+def describe(data: Dataset) -> str:
+    if data.description:
+        return data.description
+    return REPRESENTATIONS[data.representation].default_description
+
+
+def labeled(label: str, pngs) -> list[tuple[str, bytes]]:
+    """Tag each rendering of one document so the model can tell them apart."""
+    return [(label, png) for png in pngs]
+
+
+def content(prompt: str, rows: list[dict], representation: str, tag: str,
+            image_size: int, views=()):
+    """Prompt text, then the skill's own renderings, then the input rows.
+
+    ``views`` holds ``(label, png)`` pairs; each label is numbered separately
+    so a two-document gate prompt reads A 01, A 02, B 01 rather than 01..04.
+    """
+    attached = REPRESENTATIONS[representation].blocks(rows, tag, image_size)
+    if not attached and not views:
+        return prompt
+    blocks = [{"type": "text", "text": prompt}]
+    counts: dict[str, int] = {}
+    for label, png in views:
+        counts[label] = counts.get(label, 0) + 1
+        blocks.append({"type": "text",
+                       "text": f"{label} {counts[label]:02d}"})
+        blocks.append(_png_block(png))
+    return blocks + attached
+
+
+def _inputs(rows: list[dict], representation: str, tag: str = "input") -> str:
+    if not REPRESENTATIONS[representation].inline:
+        return ""
+    serialize = REPRESENTATIONS[representation].serialize
+    return "\n\n".join(
+        f'<{tag} id="P{number:02d}">\n{serialize(row)}\n</{tag}>'
+        for number, row in enumerate(rows, 1))
+
+
+def _revealed(rows: list[dict], predictions: list[dict], representation: str,
+              cites_rules: bool = True) -> str:
+    """The labeled feedback block: truth, prediction, citations, evidence."""
+    def cited(prediction, attribute: bool):
+        if not cites_rules:
+            return ""
+        joined = ",".join(prediction["rules_used"]) or "none"
+        return f' rules="{joined}"' if attribute else f"rules={joined}; "
+
+    if not REPRESENTATIONS[representation].inline:
+        return "\n".join(
+            f'P{number:02d}: true={row["label"]}; '
+            f'predicted={prediction["pred"]}; '
+            f'{cited(prediction, False)}'
+            f'evidence={prediction["evidence"]}'
+            for number, (row, prediction)
+            in enumerate(zip(rows, predictions), 1))
+    serialize = REPRESENTATIONS[representation].serialize
+    return "\n\n".join(
+        f'<example id="P{number:02d}" true="{row["label"]}" '
+        f'predicted="{prediction["pred"]}"{cited(prediction, True)}>\n'
+        f'{serialize(row)}\n'
+        f'<evidence>{prediction["evidence"]}</evidence>\n</example>'
+        for number, (row, prediction) in enumerate(zip(rows, predictions), 1))

sdm/format.md

@@ -0,0 +1,84 @@
+---
+format: markdown
+---
+
+# Skill formats
+
+Pick a format by editing the `format:` line above. Each section below defines
+one. The `mechanism:` line says which machinery the harness uses:
+
+- `edits` numbered rules revised by one bounded atomic diff
+- `document` one text document replaced outright, or kept with NOOP
+- `signal` a source the model writes but never re-reads, only its rendering
+
+Sections using `edits` or `document` are pure configuration, so a new
+text-based format needs nothing but another section here. Safety checks stay
+in Python: `validator:` selects them.
+
+## markdown
+
+mechanism: edits
+max_words: 8000
+max_rules: 100
+
+Use one operation per line and never put "|" inside a condition. Rules are a
+rulebook, not an example log: no input IDs, no labels copied from individual
+examples, no memorized values. Prefer revising an existing rule over adding a
+near-duplicate. Write only the predicate: do not start it with IF or end it
+with a prediction. IDs and the reliability counters are maintained by the
+harness; do not write them.
+
+## html
+
+mechanism: document
+validator: html
+block: skill_html
+max_words: 8000
+max_chars: 30000
+
+One standalone static page, ending with `</html>`. No scripts, no frames, no
+forms, no event handler attributes, and no external references of any kind.
+Inline SVG is encouraged where a sketch beats a sentence: small labelled
+prototype drawings per class, a workflow diagram for the reading procedure,
+and contrast pairs for classes that get confused. Track recurring mistakes in
+an error-log section so the same confusion is not repeated.
+
+## image
+
+mechanism: document
+validator: svg
+render: svg
+block: skill_svg
+max_words: 2000
+max_chars: 20000
+
+One standalone `<svg>` element, ending with `</svg>`. This is a diagram you
+will be shown as a picture on every future prediction, so draw the decision
+procedure rather than writing paragraphs: labelled class prototypes, a
+flowchart of the reading procedure, and contrast pairs for confusable classes.
+Text inside the SVG is fine, but anything that only makes sense as prose
+belongs in a caption. No external references.
+
+## audio
+
+mechanism: signal
+signal: audio
+
+Write a synthesis score, one event per line:
+TONE <frequency_hz 50-4000> <seconds 0.05-2>
+REST <seconds 0.05-2>
+Lines starting with # are stripped before rendering and will NOT survive.
+At most 400 events and 30s total.
+
+## video
+
+mechanism: signal
+signal: video
+
+Write a standalone HTML storyboard. Each frame is:
+<section class="frame" data-seconds="N">
+  <svg width="640" height="360" viewBox="0 0 640 360"> ... </svg>
+</section>
+with 1 <= N <= 10, at most 12 frames, 60s total. No scripts, no external
+references. Anything you draw or write inside the SVGs survives as pixels;
+nothing outside them does.