rigorloop 0.1.0__py3-none-any.whl

rigorloop/__init__.py

@@ -0,0 +1,18 @@
+"""RigorLoop: statistically-sound agentic build loops over dev/val/test splits."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+
+def _package_version() -> str:
+    try:
+        return version("rigorloop")
+    except PackageNotFoundError:
+        try:
+            from rigorloop._version import __version__
+
+            return __version__
+        except ImportError:
+            return "0.0.0+unknown"
+
+
+__version__ = _package_version()

rigorloop/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

rigorloop/core/__init__.py

@@ -0,0 +1 @@
+"""The functional core: pure, effect-free decision logic."""

rigorloop/core/config_calcs.py

@@ -0,0 +1,302 @@
+"""Pure parsing of rigorloop.toml text into a typed RunConfig.
+
+"Parse, don't validate": the shell reads the raw file; past this boundary the
+core never re-checks configuration validity."""
+
+from __future__ import annotations
+
+import tomllib
+
+from rigorloop.core.types import (
+    NOTHING,
+    AgentConfig,
+    Check,
+    ConfigParseError,
+    CustomPython,
+    Err,
+    ExactMatch,
+    GuidanceSolution,
+    InvalidValue,
+    JsonEquality,
+    LlmJudge,
+    LoopConfig,
+    MissingKey,
+    NormalizedMatch,
+    Nothing,
+    NumericTolerance,
+    Ok,
+    Option,
+    RegexMatch,
+    Result,
+    RunConfig,
+    ScriptSolution,
+    SkillSolution,
+    SolutionKind,
+    Some,
+    SplitConfig,
+    SplitRatios,
+    TaskConfig,
+    TomlSyntax,
+    ValidationConfig,
+)
+
+DEFAULT_MODEL = "claude-sonnet-5"
+
+
+def _get_str(
+    table: dict[str, object], key: str, qualified: str, default: Option[str]
+) -> Result[str, ConfigParseError]:
+    match table.get(key), default:
+        case None, Nothing():
+            return Err(MissingKey(qualified))
+        case None, Some(value):
+            return Ok(value)
+        case str(value), _:
+            return Ok(value)
+        case other, _:
+            return Err(InvalidValue(qualified, f"expected a string, got {other!r}"))
+
+
+def _get_int(
+    table: dict[str, object], key: str, qualified: str, default: int
+) -> Result[int, ConfigParseError]:
+    value = table.get(key, default)
+    if isinstance(value, bool) or not isinstance(value, int):
+        return Err(InvalidValue(qualified, f"expected an integer, got {value!r}"))
+    if value <= 0:
+        return Err(InvalidValue(qualified, f"must be positive, got {value}"))
+    return Ok(value)
+
+
+def _get_float(
+    table: dict[str, object], key: str, qualified: str, default: float
+) -> Result[float, ConfigParseError]:
+    value = table.get(key, default)
+    if isinstance(value, bool) or not isinstance(value, int | float):
+        return Err(InvalidValue(qualified, f"expected a number, got {value!r}"))
+    return Ok(float(value))
+
+
+def _get_bool(
+    table: dict[str, object], key: str, qualified: str, default: bool
+) -> Result[bool, ConfigParseError]:
+    value = table.get(key, default)
+    if not isinstance(value, bool):
+        return Err(InvalidValue(qualified, f"expected a boolean, got {value!r}"))
+    return Ok(value)
+
+
+def _table(data: dict[str, object], key: str) -> dict[str, object]:
+    value = data.get(key)
+    return value if isinstance(value, dict) else {}
+
+
+def _parse_kind(name: str) -> Result[SolutionKind, ConfigParseError]:
+    match name:
+        case "script":
+            return Ok(ScriptSolution())
+        case "skill":
+            return Ok(SkillSolution())
+        case "guidance":
+            return Ok(GuidanceSolution())
+        case other:
+            return Err(
+                InvalidValue("task.solution_kind", f"expected script|skill|guidance, got {other!r}")
+            )
+
+
+def _parse_task(data: dict[str, object]) -> Result[TaskConfig, ConfigParseError]:
+    table = _table(data, "task")
+    description = _get_str(table, "description_file", "task.description_file", NOTHING)
+    if isinstance(description, Err):
+        return description
+    examples = _get_str(table, "examples_file", "task.examples_file", NOTHING)
+    if isinstance(examples, Err):
+        return examples
+    kind_name = _get_str(table, "solution_kind", "task.solution_kind", Some("script"))
+    if isinstance(kind_name, Err):
+        return kind_name
+    kind = _parse_kind(kind_name.value)
+    if isinstance(kind, Err):
+        return kind
+    return Ok(TaskConfig(description.value, kind.value, examples.value))
+
+
+def _parse_split(data: dict[str, object]) -> Result[SplitConfig, ConfigParseError]:
+    table = _table(data, "split")
+    seed = _get_int(table, "seed", "split.seed", 17)
+    if isinstance(seed, Err):
+        return seed
+    raw = table.get("ratios", [0.6, 0.2, 0.2])
+    if (
+        not isinstance(raw, list)
+        or len(raw) != 3
+        or not all(isinstance(x, int | float) and not isinstance(x, bool) for x in raw)
+    ):
+        return Err(InvalidValue("split.ratios", f"expected three numbers, got {raw!r}"))
+    ratios = SplitRatios(float(raw[0]), float(raw[1]), float(raw[2]))
+    return Ok(SplitConfig(ratios, seed.value))
+
+
+def _parse_loop(data: dict[str, object]) -> Result[LoopConfig, ConfigParseError]:
+    table = _table(data, "loop")
+    max_loops = _get_int(table, "max_loops", "loop.max_loops", 12)
+    if isinstance(max_loops, Err):
+        return max_loops
+    executors = _get_int(table, "executors_per_loop", "loop.executors_per_loop", 4)
+    if isinstance(executors, Err):
+        return executors
+    in_prompt = _get_int(table, "dev_examples_in_prompt", "loop.dev_examples_in_prompt", 30)
+    if isinstance(in_prompt, Err):
+        return in_prompt
+    max_fail = _get_int(
+        table, "max_consecutive_eval_failures", "loop.max_consecutive_eval_failures", 5
+    )
+    if isinstance(max_fail, Err):
+        return max_fail
+    detail = _get_int(table, "strategy_full_detail_loops", "loop.strategy_full_detail_loops", 4)
+    if isinstance(detail, Err):
+        return detail
+    return Ok(
+        LoopConfig(max_loops.value, executors.value, in_prompt.value, max_fail.value, detail.value)
+    )
+
+
+def _parse_validation(data: dict[str, object]) -> Result[ValidationConfig, ConfigParseError]:
+    table = _table(data, "validation")
+    val_every = _get_int(table, "val_every", "validation.val_every", 3)
+    if isinstance(val_every, Err):
+        return val_every
+    max_peeks = _get_int(table, "max_peeks", "validation.max_peeks", 10)
+    if isinstance(max_peeks, Err):
+        return max_peeks
+    min_gap = _get_int(table, "min_loops_between_peeks", "validation.min_loops_between_peeks", 2)
+    if isinstance(min_gap, Err):
+        return min_gap
+    patience = _get_int(table, "patience", "validation.patience", 2)
+    if isinstance(patience, Err):
+        return patience
+    target: Option[float] = NOTHING
+    if "target_pass_rate" in table:
+        parsed = _get_float(table, "target_pass_rate", "validation.target_pass_rate", 0.0)
+        if isinstance(parsed, Err):
+            return parsed
+        if not 0.0 < parsed.value <= 1.0:
+            return Err(InvalidValue("validation.target_pass_rate", "must be in (0, 1]"))
+        target = Some(parsed.value)
+    return Ok(
+        ValidationConfig(val_every.value, max_peeks.value, min_gap.value, patience.value, target)
+    )
+
+
+def _parse_agents(data: dict[str, object]) -> Result[AgentConfig, ConfigParseError]:
+    table = _table(data, "agents")
+    model = _get_str(table, "model", "agents.model", Some(DEFAULT_MODEL))
+    if isinstance(model, Err):
+        return model
+    timeout = _get_float(table, "timeout_s", "agents.timeout_s", 300.0)
+    if isinstance(timeout, Err):
+        return timeout
+    if timeout.value <= 0:
+        return Err(InvalidValue("agents.timeout_s", "must be positive"))
+    return Ok(AgentConfig(model.value, timeout.value))
+
+
+def _parse_check(index: int, table: dict[str, object]) -> Result[Check, ConfigParseError]:
+    where = f"checks[{index}]"
+    kind = _get_str(table, "type", f"{where}.type", NOTHING)
+    if isinstance(kind, Err):
+        return kind
+    match kind.value:
+        case "exact_match":
+            return Ok(ExactMatch())
+        case "normalized_match":
+            lowercase = _get_bool(table, "lowercase", f"{where}.lowercase", True)
+            strip = _get_bool(table, "strip", f"{where}.strip", True)
+            collapse = _get_bool(table, "collapse_whitespace", f"{where}.collapse_whitespace", True)
+            if isinstance(lowercase, Err):
+                return lowercase
+            if isinstance(strip, Err):
+                return strip
+            if isinstance(collapse, Err):
+                return collapse
+            return Ok(NormalizedMatch(lowercase.value, strip.value, collapse.value))
+        case "json_equality":
+            return Ok(JsonEquality())
+        case "regex_match":
+            pattern = _get_str(table, "pattern", f"{where}.pattern", NOTHING)
+            if isinstance(pattern, Err):
+                return pattern
+            return Ok(RegexMatch(pattern.value))
+        case "numeric_tolerance":
+            atol = _get_float(table, "atol", f"{where}.atol", 1e-6)
+            if isinstance(atol, Err):
+                return atol
+            rtol = _get_float(table, "rtol", f"{where}.rtol", 1e-6)
+            if isinstance(rtol, Err):
+                return rtol
+            return Ok(NumericTolerance(atol.value, rtol.value))
+        case "custom_python":
+            path = _get_str(table, "script_path", f"{where}.script_path", NOTHING)
+            if isinstance(path, Err):
+                return path
+            return Ok(CustomPython(path.value))
+        case "llm_judge":
+            rubric = _get_str(table, "rubric", f"{where}.rubric", NOTHING)
+            if isinstance(rubric, Err):
+                return rubric
+            n_samples = _get_int(table, "n_samples", f"{where}.n_samples", 3)
+            if isinstance(n_samples, Err):
+                return n_samples
+            threshold = _get_float(table, "pass_threshold", f"{where}.pass_threshold", 0.5)
+            if isinstance(threshold, Err):
+                return threshold
+            if not 0.0 < threshold.value <= 1.0:
+                return Err(InvalidValue(f"{where}.pass_threshold", "must be in (0, 1]"))
+            return Ok(LlmJudge(rubric.value, n_samples.value, threshold.value))
+        case other:
+            return Err(InvalidValue(f"{where}.type", f"unknown check type {other!r}"))
+
+
+def _parse_checks(data: dict[str, object]) -> Result[tuple[Check, ...], ConfigParseError]:
+    raw = data.get("checks")
+    if raw is None:
+        return Err(MissingKey("checks"))
+    if not isinstance(raw, list) or not all(isinstance(t, dict) for t in raw) or not raw:
+        return Err(InvalidValue("checks", "expected a non-empty array of [[checks]] tables"))
+    parsed = [_parse_check(i, t) for i, t in enumerate(raw)]
+    failures = [p for p in parsed if isinstance(p, Err)]
+    if failures:
+        return failures[0]
+    return Ok(tuple(p.value for p in parsed if isinstance(p, Ok)))
+
+
+def parse_config(toml_text: str) -> Result[RunConfig, ConfigParseError]:
+    # tomllib.loads is a pure parse; the exception is converted to a Result at
+    # this boundary and never propagates inward.
+    try:
+        data: dict[str, object] = tomllib.loads(toml_text)
+    except tomllib.TOMLDecodeError as exc:
+        return Err(TomlSyntax(str(exc)))
+
+    task = _parse_task(data)
+    if isinstance(task, Err):
+        return task
+    split = _parse_split(data)
+    if isinstance(split, Err):
+        return split
+    loop = _parse_loop(data)
+    if isinstance(loop, Err):
+        return loop
+    validation = _parse_validation(data)
+    if isinstance(validation, Err):
+        return validation
+    agents = _parse_agents(data)
+    if isinstance(agents, Err):
+        return agents
+    checks = _parse_checks(data)
+    if isinstance(checks, Err):
+        return checks
+    return Ok(
+        RunConfig(task.value, split.value, loop.value, validation.value, agents.value, checks.value)
+    )

rigorloop/core/dataset_calcs.py

@@ -0,0 +1,193 @@
+"""Pure dataset logic: parsing examples, deduplication, splitting, manifests,
+and statistical power warnings. No I/O; text in, typed values out."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import random
+from collections import Counter
+
+from rigorloop.core.types import (
+    BadRatios,
+    DevExample,
+    DuplicateWarning,
+    Err,
+    Example,
+    ExampleDigest,
+    ExampleParseError,
+    InvalidJsonLine,
+    MissingField,
+    NotAnObject,
+    Ok,
+    PowerWarning,
+    Result,
+    Split,
+    SplitError,
+    SplitManifest,
+    SplitRatios,
+    TestExample,
+    TooFewExamples,
+    ValExample,
+)
+
+_MIN_EXAMPLES = 5
+_PREVIEW_CHARS = 60
+
+
+def _content_hash(input_text: str, expected_output: str) -> str:
+    digest = hashlib.sha256(f"{input_text}\x1f{expected_output}".encode()).hexdigest()
+    return digest[:16]
+
+
+def _example_id(input_text: str) -> str:
+    return "ex-" + hashlib.sha256(input_text.encode()).hexdigest()[:12]
+
+
+def _field_as_text(value: object) -> str:
+    """Structured inputs are JSON-encoded; plain strings pass through."""
+    return value if isinstance(value, str) else json.dumps(value, sort_keys=True)
+
+
+def _parse_line(line_number: int, line: str) -> Result[Example, ExampleParseError]:
+    try:
+        obj = json.loads(line)
+    except json.JSONDecodeError as exc:
+        return Err(InvalidJsonLine(line_number, str(exc)))
+    if not isinstance(obj, dict):
+        return Err(NotAnObject(line_number))
+    if "input" not in obj:
+        return Err(MissingField(line_number, "input"))
+    if "expected_output" not in obj:
+        return Err(MissingField(line_number, "expected_output"))
+    input_text = _field_as_text(obj["input"])
+    expected = _field_as_text(obj["expected_output"])
+    return Ok(Example(_example_id(input_text), input_text, expected))
+
+
+def parse_examples(jsonl_text: str) -> Result[tuple[Example, ...], ExampleParseError]:
+    """Parse a JSONL document of {"input": ..., "expected_output": ...} records.
+
+    Blank lines are skipped. The first malformed line aborts the parse."""
+    numbered = [
+        (i, line) for i, line in enumerate(jsonl_text.splitlines(), start=1) if line.strip()
+    ]
+    parsed = [_parse_line(i, line) for i, line in numbered]
+    failures = [r for r in parsed if isinstance(r, Err)]
+    if failures:
+        return failures[0]
+    return Ok(tuple(r.value for r in parsed if isinstance(r, Ok)))
+
+
+def dedupe_examples(
+    examples: tuple[Example, ...],
+) -> tuple[tuple[Example, ...], tuple[DuplicateWarning, ...]]:
+    """Collapse examples with identical input text (first occurrence wins).
+
+    A duplicate straddling dev and test would silently corrupt the holdout."""
+    counts = Counter(ex.example_id for ex in examples)
+    first_by_id = {ex.example_id: ex for ex in reversed(examples)}
+    seen_order = tuple(dict.fromkeys(ex.example_id for ex in examples))
+    unique = tuple(first_by_id[ex_id] for ex_id in seen_order)
+    warnings = tuple(
+        DuplicateWarning(first_by_id[ex_id].input_text[:_PREVIEW_CHARS], counts[ex_id])
+        for ex_id in seen_order
+        if counts[ex_id] > 1
+    )
+    return unique, warnings
+
+
+def _split_sizes(n: int, ratios: SplitRatios) -> tuple[int, int, int]:
+    """Largest-remainder allocation: disjoint, exhaustive, each split non-empty
+    (callers guarantee n >= _MIN_EXAMPLES)."""
+    raw = (n * ratios.dev, n * ratios.val, n * ratios.test)
+    floors = tuple(max(1, math.floor(x)) for x in raw)
+    delta = n - sum(floors)
+    by_frac = sorted(range(3), key=lambda i: raw[i] - math.floor(raw[i]), reverse=True)
+    bumped = tuple(floors[i] + (1 if delta > 0 and i in by_frac[:delta] else 0) for i in range(3))
+    # If the minimum-1 bumps overshot (tiny n), take the excess from the largest split.
+    excess = sum(bumped) - n
+    largest = max(range(3), key=lambda i: bumped[i])
+    sizes = tuple(bumped[i] - (excess if i == largest else 0) for i in range(3))
+    return (sizes[0], sizes[1], sizes[2])
+
+
+def split_examples(
+    examples: tuple[Example, ...], ratios: SplitRatios, seed: int
+) -> Result[Split, SplitError]:
+    """Deterministically partition examples into dev/val/test.
+
+    Guarantees: disjoint, exhaustive, stable for a given seed, each split
+    non-empty. Input order does not matter (examples are sorted before the
+    seeded shuffle)."""
+    if abs(ratios.dev + ratios.val + ratios.test - 1.0) > 1e-9:
+        return Err(BadRatios(f"ratios must sum to 1, got {ratios}"))
+    if min(ratios.dev, ratios.val, ratios.test) <= 0:
+        return Err(BadRatios("every split ratio must be positive"))
+    if len(examples) < _MIN_EXAMPLES:
+        return Err(TooFewExamples(len(examples), _MIN_EXAMPLES))
+
+    ordered = sorted(examples, key=lambda e: e.example_id)
+    # random.Random(seed) is a pure function of the seed parameter: same seed,
+    # same shuffle, no ambient randomness touched.
+    shuffled = random.Random(seed).sample(ordered, k=len(ordered))
+    n_dev, n_val, _ = _split_sizes(len(shuffled), ratios)
+    return Ok(
+        Split(
+            dev=tuple(DevExample(e) for e in shuffled[:n_dev]),
+            val=tuple(ValExample(e) for e in shuffled[n_dev : n_dev + n_val]),
+            test=tuple(TestExample(e) for e in shuffled[n_dev + n_val :]),
+        )
+    )
+
+
+def build_manifest(
+    split: Split, ratios: SplitRatios, seed: int, eval_model: str, cli_version: str
+) -> SplitManifest:
+    """Content-hash manifest pinning the split and the evaluating model, so a
+    resume can never reshuffle examples across splits."""
+    return SplitManifest(
+        seed=seed,
+        ratios=ratios,
+        dev=tuple(_digest(d.example) for d in split.dev),
+        val=tuple(_digest(v.example) for v in split.val),
+        test=tuple(_digest(t.example) for t in split.test),
+        eval_model=eval_model,
+        cli_version=cli_version,
+    )
+
+
+def _digest(example: Example) -> ExampleDigest:
+    return ExampleDigest(
+        example.example_id, _content_hash(example.input_text, example.expected_output)
+    )
+
+
+def manifest_matches(manifest: SplitManifest, split: Split) -> bool:
+    """True iff the split's membership and contents match the manifest exactly."""
+    return (
+        manifest.dev == tuple(_digest(d.example) for d in split.dev)
+        and manifest.val == tuple(_digest(v.example) for v in split.val)
+        and manifest.test == tuple(_digest(t.example) for t in split.test)
+    )
+
+
+def power_warnings(split: Split, threshold: float = 0.10) -> tuple[PowerWarning, ...]:
+    """Warn when a split is too small to distinguish meaningful pass-rate
+    differences. Half-width is the worst-case (p=0.5) 95% margin ~1.96*sqrt(.25/n)."""
+    sizes = (("dev", len(split.dev)), ("validation", len(split.val)), ("test", len(split.test)))
+    widths = tuple((name, n, 1.96 * math.sqrt(0.25 / n)) for name, n in sizes)
+    return tuple(
+        PowerWarning(
+            split_name=name,
+            n=n,
+            half_width=hw,
+            message=(
+                f"{name} set of {n} examples can only distinguish pass-rate "
+                f"differences of ~±{round(hw * 100)} points"
+            ),
+        )
+        for name, n, hw in widths
+        if hw > threshold
+    )