world-model-optimizer 0.2.0__py3-none-any.whl

wmo/harness/skills.py

@@ -0,0 +1,116 @@
+"""Skills: reusable techniques a harness ships, one `SKILL.md`-style unit each.
+
+A skill is one markdown file with tiny frontmatter (`name`, `description`) and a body. Harnesses
+surface skills by **progressive disclosure**: only the name+description index is preloaded into the
+system prompt (`render_index`); the agent pulls a full body on demand with the `read_skill` tool,
+so always-loaded context stays small while the library grows.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from pydantic import BaseModel, field_validator
+
+_NAME_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
+_FRONTMATTER_RE = re.compile(r"^---\n(.*?)\n---\n(.*)$", re.DOTALL)
+
+
+class Skill(BaseModel):
+    """One reusable skill: a name, a one-line trigger description, and a body to reuse.
+
+    Validation lives on the MODEL, not just the markdown parser: skill names become file paths
+    (`skills/<name>.md`), so a programmatically-constructed skill with a hostile name (`../../x`)
+    must be impossible, not merely improbable. The description is coerced to one line (it lives in
+    frontmatter, where a newline silently truncates on round-trip); the body is stripped so
+    round-trips compare equal.
+    """
+
+    name: str
+    description: str
+    body: str
+
+    @field_validator("name")
+    @classmethod
+    def _kebab_name(cls, v: str) -> str:
+        if not _NAME_RE.match(v):
+            raise ValueError(f"skill name {v!r} must be kebab-case ([a-z0-9-])")
+        return v
+
+    @field_validator("description")
+    @classmethod
+    def _one_line(cls, v: str) -> str:
+        return " ".join(v.split())
+
+    @field_validator("body")
+    @classmethod
+    def _stripped(cls, v: str) -> str:
+        return v.strip()
+
+    def to_markdown(self) -> str:
+        """Serialize to a SKILL.md-style file (frontmatter + body)."""
+        return f"---\nname: {self.name}\ndescription: {self.description}\n---\n{self.body}"
+
+    @classmethod
+    def from_markdown(cls, text: str) -> Skill:
+        match = _FRONTMATTER_RE.match(text)
+        if match is None:
+            raise ValueError("skill file has no frontmatter")
+        meta = parse_frontmatter(match.group(1))
+        name = meta.get("name", "")
+        if not _NAME_RE.match(name):
+            raise ValueError(f"skill name {name!r} must be kebab-case")
+        return cls(name=name, description=meta.get("description", ""), body=match.group(2))
+
+
+def parse_frontmatter(block: str) -> dict[str, str]:
+    """Parse the tiny `key: value` frontmatter (one field per line)."""
+    out: dict[str, str] = {}
+    for line in block.splitlines():
+        key, sep, value = line.partition(":")
+        if sep:
+            out[key.strip()] = value.strip()
+    return out
+
+
+class SkillLibrary:
+    """An in-memory set of skills, loadable from / writable to a `skills/` directory."""
+
+    def __init__(self, skills: list[Skill] | None = None) -> None:
+        self._skills: dict[str, Skill] = {s.name: s for s in (skills or [])}
+
+    @classmethod
+    def from_dir(cls, root: str | Path) -> SkillLibrary:
+        """Load every `*.md` under `root` (a malformed skill file is an error, not skipped —
+        a harness that ships a broken skill should fail loudly at load time, not at rollout)."""
+        library = cls()
+        root = Path(root)
+        if not root.exists():
+            return library
+        for path in sorted(root.glob("*.md")):
+            skill = Skill.from_markdown(path.read_text(encoding="utf-8"))
+            library._skills[skill.name] = skill
+        return library
+
+    def write_dir(self, root: str | Path) -> None:
+        """Write one `<name>.md` per skill under `root` (created if missing)."""
+        root = Path(root)
+        root.mkdir(parents=True, exist_ok=True)
+        for skill in self._skills.values():
+            (root / f"{skill.name}.md").write_text(skill.to_markdown(), encoding="utf-8")
+
+    def __len__(self) -> int:
+        return len(self._skills)
+
+    def names(self) -> list[str]:
+        return sorted(self._skills)
+
+    def get(self, name: str) -> Skill | None:
+        return self._skills.get(name)
+
+    def render_index(self) -> str:
+        """Progressive-disclosure index: name + description only (bodies via read_skill)."""
+        if not self._skills:
+            return ""
+        return "\n".join(f"- {s.name}: {s.description}" for s in self._skills.values())

wmo/harness/source_tree.py

@@ -0,0 +1,319 @@
+"""Portable source trees for editing and reconstructing complete harnesses."""
+
+from __future__ import annotations
+
+import hashlib
+import tomllib
+from pathlib import PurePosixPath
+
+import tomli_w
+from pydantic import BaseModel, ConfigDict, field_validator, model_validator
+
+from wmo.core.text import validate_durable_text
+
+# _SAFE_PATH_RE and _SLUG_RE are doc's path and surface-id grammars; sharing them keeps this
+# module's file checks and Surface validation from ever drifting apart.
+from wmo.harness.doc import (
+    _SAFE_PATH_RE,
+    _SLUG_RE,
+    _STORE_METADATA_FILES,
+    CODE_RUNTIME_ID,
+    MAX_OUTPUT_TOKENS_ID,
+    MAX_SURFACE_PATH_BYTES,
+    MAX_TURNS_ID,
+    RUNTIME_KIND_ID,
+    TEMPERATURE_ID,
+    TOOL_POLICY_ID,
+    HarnessDoc,
+    Surface,
+    SurfaceKind,
+    code_surface_id,
+)
+from wmo.harness.skills import Skill
+
+SYSTEM_FILE = "SYSTEM.md"
+CONFIG_FILE = "config.toml"
+RUNTIME_FILE = "runtime.py"
+SKILLS_DIR = "skills"
+
+_RESERVED_RENDER_FILES = frozenset({SYSTEM_FILE, CONFIG_FILE, RUNTIME_FILE})
+_TREE_HASH_BYTES = 16
+MAX_SOURCE_PATH_BYTES = MAX_SURFACE_PATH_BYTES
+
+
+class HarnessSourceFile(BaseModel):
+    """One UTF-8 text file at a canonical path inside a harness source tree."""
+
+    model_config = ConfigDict(frozen=True)
+
+    path: str
+    content: str
+
+    @model_validator(mode="after")
+    def _validate_file(self) -> HarnessSourceFile:
+        candidate = PurePosixPath(self.path)
+        if (
+            candidate.is_absolute()
+            or not candidate.parts
+            or candidate.as_posix() != self.path
+            or ".." in candidate.parts
+            or not _SAFE_PATH_RE.fullmatch(self.path)
+            or len(self.path.encode("utf-8")) > MAX_SOURCE_PATH_BYTES
+        ):
+            raise ValueError(f"source path {self.path!r} must be a canonical relative POSIX path")
+        validate_durable_text(self.content, field=f"source file {self.path!r}")
+        return self
+
+
+class HarnessSourceTree(BaseModel):
+    """A complete editable file representation that can be parsed into a HarnessDoc.
+
+    Round-tripping through `from_doc`/`to_doc` is lossless only on the canonical subset: a
+    multi-prompt document renders as one SYSTEM.md and reparses as a single `prompt:core`
+    surface, and validation-only `Surface.budget` values are dropped.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    files: tuple[HarnessSourceFile, ...]
+
+    @field_validator("files")
+    @classmethod
+    def _canonical_files(
+        cls, files: tuple[HarnessSourceFile, ...]
+    ) -> tuple[HarnessSourceFile, ...]:
+        paths = [item.path for item in files]
+        duplicates = sorted({path for path in paths if paths.count(path) > 1})
+        if duplicates:
+            raise ValueError(f"duplicate source path(s): {duplicates}")
+        metadata = sorted(path for path in paths if path in _STORE_METADATA_FILES)
+        if metadata:
+            raise ValueError(f"source tree cannot contain store metadata file(s): {metadata}")
+        # Renders land on real filesystems, so structural conflicts must fail here, at
+        # validation time, not later inside a store write.
+        by_fold: dict[str, str] = {}
+        for path in sorted(paths):
+            claimed = by_fold.setdefault(path.casefold(), path)
+            if claimed != path:
+                raise ValueError(
+                    f"source paths {claimed!r} and {path!r} differ only by letter case and "
+                    "would collide on a case-insensitive filesystem; rename one"
+                )
+        directory_prefixes: dict[str, str] = {}
+        for path in paths:
+            parts = path.split("/")
+            for index in range(1, len(parts)):
+                directory_prefixes.setdefault("/".join(parts[:index]), path)
+        for path in sorted(paths):
+            child = directory_prefixes.get(path)
+            if child is not None:
+                raise ValueError(
+                    f"source path {path!r} is a file but is also the directory holding "
+                    f"{child!r}; rename one so no file path is a directory prefix of another"
+                )
+        return tuple(sorted(files, key=lambda item: item.path))
+
+    @classmethod
+    def from_doc(cls, doc: HarnessDoc) -> HarnessSourceTree:
+        """Render a harness into the exact source files an external editor should see."""
+        files: dict[str, str] = {
+            SYSTEM_FILE: doc.system_prompt(),
+            CONFIG_FILE: tomli_w.dumps(
+                {
+                    "harness": {
+                        "tools": doc.tools(),
+                        "max_turns": doc.max_turns(),
+                        "max_output_tokens": doc.max_output_tokens(),
+                        "temperature": doc.temperature(),
+                        "runtime_kind": doc.runtime_kind(),
+                    }
+                }
+            ),
+        }
+        for skill in doc.skills():
+            files[f"{SKILLS_DIR}/{skill.name}.md"] = skill.to_markdown()
+        runtime = doc.surface(CODE_RUNTIME_ID)
+        if runtime is not None:
+            files[RUNTIME_FILE] = runtime.content
+        for surface in doc.code_files():
+            assert surface.path is not None
+            if surface.path in _RESERVED_RENDER_FILES or surface.path.startswith(f"{SKILLS_DIR}/"):
+                raise ValueError(
+                    f"code surface path {surface.path!r} collides with a reserved file"
+                )
+            files[surface.path] = surface.content
+        return cls(
+            files=tuple(
+                HarnessSourceFile(path=path, content=content) for path, content in files.items()
+            )
+        )
+
+    def to_doc(self, name: str) -> HarnessDoc:
+        """Parse the complete source tree into one validated harness document."""
+        files = self.file_map()
+        if SYSTEM_FILE not in files:
+            raise ValueError(f"harness source tree is missing required {SYSTEM_FILE}")
+        surfaces = [
+            Surface(
+                id="prompt:core",
+                kind=SurfaceKind.PROMPT,
+                content=files[SYSTEM_FILE],
+            )
+        ]
+        config_text = files.get(CONFIG_FILE)
+        if config_text is not None:
+            parsed_config = tomllib.loads(config_text)
+            unknown_tables = sorted(set(parsed_config) - {"harness"})
+            if unknown_tables:
+                raise ValueError(
+                    f"{CONFIG_FILE} contains unknown top-level table(s): {unknown_tables}"
+                )
+            config = parsed_config.get("harness", {})
+            if not isinstance(config, dict):
+                raise ValueError(f"{CONFIG_FILE} [harness] must be a table")
+            allowed_fields = {
+                "tools",
+                "max_turns",
+                "max_output_tokens",
+                "temperature",
+                "runtime_kind",
+            }
+            unknown_fields = sorted(set(config) - allowed_fields)
+            if unknown_fields:
+                raise ValueError(
+                    f"{CONFIG_FILE} [harness] contains unknown field(s): {unknown_fields}"
+                )
+            tools = config.get("tools")
+            if tools is not None:
+                if not isinstance(tools, list) or not all(isinstance(item, str) for item in tools):
+                    raise ValueError(f"{CONFIG_FILE} harness.tools must be an array of strings")
+                surfaces.append(
+                    Surface(
+                        id=TOOL_POLICY_ID,
+                        kind=SurfaceKind.TOOL_POLICY,
+                        content="\n".join(tools),
+                    )
+                )
+            scalar_fields = (
+                ("max_turns", MAX_TURNS_ID),
+                ("max_output_tokens", MAX_OUTPUT_TOKENS_ID),
+                ("temperature", TEMPERATURE_ID),
+            )
+            for field, surface_id in scalar_fields:
+                if field in config:
+                    surfaces.append(
+                        Surface(
+                            id=surface_id,
+                            kind=SurfaceKind.PARAM,
+                            content=str(config[field]),
+                        )
+                    )
+            runtime_kind = config.get("runtime_kind")
+            if runtime_kind is not None and not isinstance(runtime_kind, str):
+                raise ValueError(f"{CONFIG_FILE} harness.runtime_kind must be a string")
+            if runtime_kind and runtime_kind != "kit-python":
+                surfaces.append(
+                    Surface(
+                        id=RUNTIME_KIND_ID,
+                        kind=SurfaceKind.PARAM,
+                        content=str(runtime_kind),
+                    )
+                )
+        runtime = files.get(RUNTIME_FILE)
+        if runtime is not None:
+            surfaces.append(Surface(id=CODE_RUNTIME_ID, kind=SurfaceKind.CODE, content=runtime))
+        for item in self.files:
+            path = PurePosixPath(item.path)
+            if path.parts[0] != SKILLS_DIR:
+                continue
+            if len(path.parts) != 2 or path.suffix != ".md":
+                raise ValueError(f"skill source path {item.path!r} must be skills/<skill-name>.md")
+            skill = Skill.from_markdown(item.content)
+            if path.stem != skill.name:
+                raise ValueError(
+                    f"skill source path {item.path!r} does not match declared name {skill.name!r}"
+                )
+            surfaces.append(
+                Surface(
+                    id=f"skill:{skill.name}",
+                    kind=SurfaceKind.SKILL,
+                    content=skill.to_markdown(),
+                )
+            )
+        code_paths_by_id: dict[str, str] = {}
+        for item in self.files:
+            if item.path in _RESERVED_RENDER_FILES or item.path.startswith(f"{SKILLS_DIR}/"):
+                continue
+            surface_id = _code_surface_id(item.path)
+            if surface_id == CODE_RUNTIME_ID:
+                raise ValueError(
+                    f"code file path {item.path!r} would alias the reserved in-process runtime "
+                    f"surface {CODE_RUNTIME_ID!r} (which renders as {RUNTIME_FILE}); rename "
+                    "the file"
+                )
+            claimed = code_paths_by_id.get(surface_id)
+            if claimed is not None:
+                raise ValueError(
+                    f"code file paths {claimed!r} and {item.path!r} both map to surface id "
+                    f"{surface_id!r} ('/' and '.' both become '-'); rename one so every "
+                    "path keeps a distinct id"
+                )
+            code_paths_by_id[surface_id] = item.path
+            surfaces.append(
+                Surface(
+                    id=surface_id,
+                    kind=SurfaceKind.CODE,
+                    path=item.path,
+                    content=item.content,
+                )
+            )
+        return HarnessDoc(name=name, surfaces=surfaces)
+
+    def file_map(self) -> dict[str, str]:
+        """Return a new path-to-content mapping in canonical path order."""
+        return {item.path: item.content for item in self.files}
+
+    @property
+    def total_bytes(self) -> int:
+        """Return the UTF-8 payload size across all source files."""
+        return sum(len(item.content.encode("utf-8")) for item in self.files)
+
+    @property
+    def tree_hash(self) -> str:
+        """Return a content address covering every source path and byte."""
+        digest = hashlib.blake2b(digest_size=_TREE_HASH_BYTES)
+        for item in self.files:
+            path = item.path.encode("utf-8")
+            content = item.content.encode("utf-8")
+            digest.update(len(path).to_bytes(4, "big"))
+            digest.update(path)
+            digest.update(len(content).to_bytes(8, "big"))
+            digest.update(content)
+        return digest.hexdigest()
+
+    def validate_bounds(self, *, max_files: int, max_bytes: int) -> None:
+        """Reject a tree that exceeds explicit file-count or UTF-8 byte bounds."""
+        if isinstance(max_files, bool) or not isinstance(max_files, int) or max_files < 1:
+            raise ValueError("max_files must be a positive integer")
+        if isinstance(max_bytes, bool) or not isinstance(max_bytes, int) or max_bytes < 1:
+            raise ValueError("max_bytes must be a positive integer")
+        if len(self.files) > max_files:
+            raise ValueError(
+                f"source tree has {len(self.files)} files, more than {max_files} files allowed"
+            )
+        if self.total_bytes > max_bytes:
+            raise ValueError(
+                f"source tree has {self.total_bytes} bytes, more than {max_bytes} bytes allowed"
+            )
+
+
+def _code_surface_id(path: str) -> str:
+    """Derive a code file's surface id, failing with the path and the allowed grammar."""
+    surface_id = code_surface_id(path)
+    if not _SLUG_RE.fullmatch(surface_id.partition(":")[2]):
+        raise ValueError(
+            f"code file path {path!r} maps to surface id {surface_id!r}, which is not a valid "
+            "'code:<kebab-slug>' id; use lowercase [a-z0-9] runs separated by single '/', '.', "
+            "or '-' characters (for example src/agent-loop.ts)"
+        )
+    return surface_id

wmo/harness/store.py

@@ -0,0 +1,176 @@
+"""Harnesses on disk: immutable numbered versions plus movable aliases, per name.
+
+Like world models under `.wmo/models/<name>/`, a harness is a named artifact — but harnesses
+accumulate *versions*, because they are the thing `wmo optimize harness` iterates on. A
+version, once
+written, never changes; deployment state lives in movable aliases; and every eval result keys to an
+immutable version rather than to "whatever the harness currently is".
+
+    .wmo/harnesses/<name>/
+      aliases.toml        # [aliases]  champion = 3   (movable pointers; rollback = re-point)
+      v1/
+        doc.json          # the authoritative HarnessDoc serialization
+        SYSTEM.md         # rendered export of the same document, for running the harness
+        config.toml       #   outside wmo — regenerated on every save, never read back
+        skills/<slug>.md  #   when doc.json is present
+      v3/ ...
+
+`doc.json` is authoritative; the rendered files are an export. A directory with rendered files but
+no `doc.json` (a hand-authored harness) still loads: the files parse into a single-prompt document.
+Rendered-only loads are strict: unknown config.toml tables or fields, non-.md files under skills/,
+and a skill filename that does not match its frontmatter name are errors with actionable messages,
+not silently ignored. Dotfiles (.DS_Store and friends) are skipped.
+"""
+
+from __future__ import annotations
+
+import tomllib
+from pathlib import Path
+
+import tomli_w
+
+from wmo.config.store import validate_name
+from wmo.harness.doc import HarnessDoc
+from wmo.harness.source_tree import SYSTEM_FILE, HarnessSourceFile, HarnessSourceTree
+
+HARNESSES_DIR = "harnesses"
+CHAMPION_ALIAS = "champion"
+
+_DOC_FILE = "doc.json"
+_ALIASES_FILE = "aliases.toml"
+
+
+class HarnessStore:
+    """Named, versioned harnesses under `<root>/harnesses/<name>/`."""
+
+    def __init__(self, root: str | Path = ".wmo") -> None:
+        self.root = Path(root)
+
+    @property
+    def harnesses_dir(self) -> Path:
+        return self.root / HARNESSES_DIR
+
+    def dir_for(self, name: str) -> Path:
+        return self.harnesses_dir / validate_name(name)
+
+    # -- enumeration -------------------------------------------------------------------------
+
+    def list_names(self) -> list[str]:
+        if not self.harnesses_dir.exists():
+            return []
+        return sorted(
+            d.name for d in self.harnesses_dir.iterdir() if d.is_dir() and self.versions(d.name)
+        )
+
+    def versions(self, name: str) -> list[int]:
+        directory = self.dir_for(name)
+        if not directory.exists():
+            return []
+        found: list[int] = []
+        for child in directory.iterdir():
+            if child.is_dir() and child.name.startswith("v") and child.name[1:].isdigit():
+                found.append(int(child.name[1:]))
+        return sorted(found)
+
+    def exists(self, name: str) -> bool:
+        return bool(self.versions(name))
+
+    # -- aliases -----------------------------------------------------------------------------
+
+    def aliases(self, name: str) -> dict[str, int]:
+        path = self.dir_for(name) / _ALIASES_FILE
+        if not path.exists():
+            return {}
+        data = tomllib.loads(path.read_text(encoding="utf-8")).get("aliases", {})
+        return {k: v for k, v in data.items() if isinstance(v, int)}
+
+    def set_alias(self, name: str, alias: str, version: int) -> None:
+        """Point `alias` at `version` (moving it if it exists). Rollback is re-pointing."""
+        if version not in self.versions(name):
+            raise ValueError(f"harness {name!r} has no version v{version}")
+        current = self.aliases(name)
+        current[alias] = version
+        path = self.dir_for(name) / _ALIASES_FILE
+        path.write_text(tomli_w.dumps({"aliases": current}), encoding="utf-8")
+
+    # -- load / save ---------------------------------------------------------------------------
+
+    def resolve_version(self, name: str, ref: str | None = None) -> int:
+        """Resolve a version ref: `None` -> champion alias, else latest; `"vN"`/`"N"`; an alias."""
+        available = self.versions(name)
+        if not available:
+            raise FileNotFoundError(
+                f"no harness named {name!r} under {self.harnesses_dir} "
+                f"(have: {', '.join(self.list_names()) or 'none'})"
+            )
+        aliases = self.aliases(name)
+        if ref is None:
+            return aliases.get(CHAMPION_ALIAS, available[-1])
+        normalized = ref.removeprefix("v")
+        if normalized.isdigit():
+            version = int(normalized)
+            if version not in available:
+                raise ValueError(f"harness {name!r} has no version v{version}")
+            return version
+        if ref in aliases:
+            return aliases[ref]
+        raise ValueError(f"harness {name!r} has no version or alias {ref!r}")
+
+    def load(self, name: str, ref: str | None = None) -> HarnessDoc:
+        version = self.resolve_version(name, ref)
+        directory = self.dir_for(name) / f"v{version}"
+        doc_path = directory / _DOC_FILE
+        if doc_path.exists():
+            doc = HarnessDoc.model_validate_json(doc_path.read_text(encoding="utf-8"))
+        else:
+            doc = _parse_rendered(name, directory)
+        return doc.model_copy(update={"name": name, "version": version})
+
+    def save_version(self, doc: HarnessDoc, *, alias: str | None = None) -> HarnessDoc:
+        """Write `doc` as the next version of its name; optionally point `alias` at it.
+
+        Versions are append-only: this never touches an existing version directory.
+        """
+        validate_name(doc.name)
+        version = (self.versions(doc.name)[-1] + 1) if self.exists(doc.name) else 1
+        stamped = doc.model_copy(update={"version": version})
+        directory = self.dir_for(doc.name) / f"v{version}"
+        directory.mkdir(parents=True, exist_ok=False)  # append-only: collision is a bug
+        (directory / _DOC_FILE).write_text(stamped.model_dump_json(indent=2), encoding="utf-8")
+        for rel_path, content in _render(stamped).items():
+            target = directory / rel_path
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_text(content, encoding="utf-8")
+        if alias is not None:
+            self.set_alias(doc.name, alias, version)
+        return stamped
+
+
+def _render(doc: HarnessDoc) -> dict[str, str]:
+    """Render the document to its file export (relative path -> content)."""
+    return HarnessSourceTree.from_doc(doc).file_map()
+
+
+def _parse_rendered(name: str, directory: Path) -> HarnessDoc:
+    """Parse a rendered/hand-authored directory (no doc.json) into a document.
+
+    The whole `SYSTEM.md` becomes one `prompt:core` surface — section boundaries are not
+    recoverable from a rendered prompt, which is exactly why `doc.json` is the authoritative form.
+    """
+    files: list[HarnessSourceFile] = []
+    for path in sorted(directory.rglob("*")):
+        if not path.is_file():
+            continue
+        rel_path = path.relative_to(directory)
+        if any(part.startswith(".") for part in rel_path.parts):
+            # Finder and editors drop metadata like .DS_Store (often with NUL bytes) into
+            # hand-authored dirs. A dotfile can never be a harness surface (its code_surface_id
+            # is not a valid slug), so skip it instead of failing the load on its content.
+            continue
+        rel = rel_path.as_posix()
+        if rel in {_DOC_FILE, _ALIASES_FILE}:
+            continue
+        files.append(HarnessSourceFile(path=rel, content=path.read_text(encoding="utf-8")))
+    if not files:
+        raise ValueError(f"harness dir {directory} has neither {_DOC_FILE} nor {SYSTEM_FILE}")
+    return HarnessSourceTree(files=tuple(files)).to_doc(name)