synth-optimizers 0.2.12.dev20260816__tar.gz → 0.2.13__tar.gz

{synth_optimizers-0.2.12.dev20260816 → synth_optimizers-0.2.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synth-optimizers
-Version: 0.2.12.dev20260816
+Version: 0.2.13
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: synth-containers==0.4.1.dev20260814
 Requires-Dist: websocket-client>=1.8.0
@@ -51,6 +51,36 @@ local algorithm; GELO and SFT are hosted-only in the public package and run on
 Synth hosted optimizer infrastructure. Hosted GEPA, GELO, and SFT submission is
 covered in [`docs/hosted-optimizers.md`](docs/hosted-optimizers.md).
 
+### Hosted SFT control plane
+
+SFT is served by `synth-optimizers`; Optimizers-beta is an internal training executor,
+not a Workshop-facing API. For local QA, start beta with its executor token and then
+start the public façade:
+
+```bash
+# In the Optimizers-beta checkout:
+OPTIMIZERS_BETA_SERVICE_TOKEN=local-dev-token \
+  cargo run --bin optimizers-beta -- serve --bind 127.0.0.1:8879
+
+# In this checkout:
+export SYNTH_OPTIMIZERS_BETA_URL=http://127.0.0.1:8879
+export OPTIMIZERS_BETA_SERVICE_TOKEN=local-dev-token  # held only by the façade
+export SYNTH_OPTIMIZERS_SFT_SERVICE_TOKEN=local-qa-token  # Workshop / CLI callers
+synth-optimizers sft service --db .sft/service.sqlite --bind 127.0.0.1:8878
+```
+
+Submit, inspect, and cancel only through the façade:
+
+```bash
+synth-optimizers sft validate --config sft.toml
+synth-optimizers sft submit --config sft.toml --follow
+synth-optimizers sft watch RUN_ID --events
+synth-optimizers sft cancel RUN_ID
+```
+
+The façade keeps executor-only workspace paths and service credentials private. Its
+artifact proxy is available at `/v1/runs/RUN_ID/artifacts/{manifest,events}`.
+
 ## Future hosted-algorithm compatibility
 
 MAPO, OHCO, Online Reflexion, and MARL prompt-optimization identifiers are retained

{synth_optimizers-0.2.12.dev20260816 → synth_optimizers-0.2.13}/README.md

@@ -29,6 +29,36 @@ local algorithm; GELO and SFT are hosted-only in the public package and run on
 Synth hosted optimizer infrastructure. Hosted GEPA, GELO, and SFT submission is
 covered in [`docs/hosted-optimizers.md`](docs/hosted-optimizers.md).
 
+### Hosted SFT control plane
+
+SFT is served by `synth-optimizers`; Optimizers-beta is an internal training executor,
+not a Workshop-facing API. For local QA, start beta with its executor token and then
+start the public façade:
+
+```bash
+# In the Optimizers-beta checkout:
+OPTIMIZERS_BETA_SERVICE_TOKEN=local-dev-token \
+  cargo run --bin optimizers-beta -- serve --bind 127.0.0.1:8879
+
+# In this checkout:
+export SYNTH_OPTIMIZERS_BETA_URL=http://127.0.0.1:8879
+export OPTIMIZERS_BETA_SERVICE_TOKEN=local-dev-token  # held only by the façade
+export SYNTH_OPTIMIZERS_SFT_SERVICE_TOKEN=local-qa-token  # Workshop / CLI callers
+synth-optimizers sft service --db .sft/service.sqlite --bind 127.0.0.1:8878
+```
+
+Submit, inspect, and cancel only through the façade:
+
+```bash
+synth-optimizers sft validate --config sft.toml
+synth-optimizers sft submit --config sft.toml --follow
+synth-optimizers sft watch RUN_ID --events
+synth-optimizers sft cancel RUN_ID
+```
+
+The façade keeps executor-only workspace paths and service credentials private. Its
+artifact proxy is available at `/v1/runs/RUN_ID/artifacts/{manifest,events}`.
+
 ## Future hosted-algorithm compatibility
 
 MAPO, OHCO, Online Reflexion, and MARL prompt-optimization identifiers are retained

{synth_optimizers-0.2.12.dev20260816 → synth_optimizers-0.2.13}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "synth-optimizers"
-version = "0.2.12.dev20260816"
+version = "0.2.13"
 description = "Public Synth optimizer tooling with a Rust GEPA core."
 authors = [{name = "Synth Laboratories", email = "josh@usesynth.ai"}]
 readme = "README.md"
@@ -48,7 +48,7 @@ module-name = "synth_optimizers._synth_optimizers"
 python-source = "src"
 # Ship the bundled console docs (markdown + logo) as package data so a
 # pip-installed `gepa console` can serve them without a repo checkout.
-include = ["src/synth_optimizers/docs/**/*"]
+include = ["src/synth_optimizers/docs/**/*", "src/synth_optimizers/eval/catalog/*.toml"]
 
 [tool.uv]
 package = true

{synth_optimizers-0.2.12.dev20260816 → synth_optimizers-0.2.13}/rust/crates/synth_optimizer_platform/src/workspace.rs

@@ -4819,15 +4819,7 @@ impl WorkspaceStore {
         let mut stmt = self.conn.prepare(
             r#"
             SELECT run_id, frame_json
-            FROM sensor_frames sf
-            WHERE NOT EXISTS (
-                SELECT 1
-                FROM candidate_seed_rewards csr
-                WHERE csr.run_id = sf.run_id
-                  AND csr.candidate_id = sf.candidate_id
-                  AND csr.evaluation_stage = sf.evaluation_stage
-                  AND csr.seed_id = sf.task_id
-            )
+            FROM sensor_frames
             "#,
         )?;
         let mut rows = stmt.query([])?;
@@ -4860,6 +4852,10 @@ impl WorkspaceStore {
     }
 
     fn ensure_runtime_schema(&self) -> Result<()> {
+        // v0.2.0 workspaces called this identity `seed`. The v0.2.12
+        // backfill used `task_id` before migrating the column, which made the
+        // sidecar fail during startup even when the legacy table was empty.
+        self.ensure_column("sensor_frames", "task_id", "TEXT NOT NULL DEFAULT ''")?;
         self.ensure_column(
             "budget_commits",
             "wall_seconds",
@@ -9351,4 +9347,54 @@ mod nullable_cost_migration_tests {
         let _ = fs::remove_file(path.with_extension("sqlite-wal"));
         let _ = fs::remove_file(path.with_extension("sqlite-shm"));
     }
+
+    #[test]
+    fn opening_legacy_workspace_adds_sensor_frame_task_identity_before_backfill() {
+        let path = std::env::temp_dir().join(format!(
+            "synth-sensor-frame-migration-{}-{}.sqlite",
+            std::process::id(),
+            OffsetDateTime::now_utc().unix_timestamp_nanos()
+        ));
+        let conn = Connection::open(&path).unwrap();
+        conn.execute_batch(
+            r#"
+            CREATE TABLE sensor_frames (
+                run_id TEXT NOT NULL,
+                sensor_frame_id TEXT NOT NULL,
+                candidate_id TEXT NOT NULL,
+                rollout_id TEXT,
+                example_id TEXT NOT NULL,
+                seed INTEGER NOT NULL,
+                split TEXT NOT NULL,
+                evaluation_stage TEXT NOT NULL,
+                reward REAL NOT NULL,
+                status TEXT NOT NULL,
+                trace_digest_json TEXT,
+                usage_json TEXT NOT NULL,
+                failure_json TEXT,
+                frame_json TEXT NOT NULL,
+                updated_at TEXT NOT NULL,
+                PRIMARY KEY(run_id, sensor_frame_id)
+            );
+            "#,
+        )
+        .unwrap();
+        drop(conn);
+
+        let store = WorkspaceStore::open_existing(&path).unwrap();
+        let mut columns = store
+            .conn
+            .prepare("PRAGMA table_info(sensor_frames)")
+            .unwrap();
+        let task_id = columns
+            .query_map([], |row| row.get::<_, String>(1))
+            .unwrap()
+            .any(|name| name.unwrap() == "task_id");
+        assert!(task_id);
+        drop(columns);
+        drop(store);
+        let _ = fs::remove_file(&path);
+        let _ = fs::remove_file(path.with_extension("sqlite-wal"));
+        let _ = fs::remove_file(path.with_extension("sqlite-shm"));
+    }
 }

{synth_optimizers-0.2.12.dev20260816 → synth_optimizers-0.2.13}/src/synth_optimizers/__init__.py

@@ -84,6 +84,17 @@ from .hosted import (
     validate_online_reflexion_evidence_notes,
 )
 from .hosted_config import HostedOptimizerConfig
+from .sft import (
+    SFT_ALGORITHM_ID,
+    BetaSftExecutorClient,
+    SftArtifact,
+    SftConfig,
+    SftPublicServiceClient,
+    SftService,
+    SftServiceError,
+    create_sft_http_server,
+    serve_sft_service,
+)
 from .future_algorithms import (
     FUTURE_HOSTED_ALGORITHMS,
     FutureHostedAlgorithm,
@@ -226,6 +237,15 @@ __all__ = [
     "HostedOptimizerClient",
     "HostedOptimizerError",
     "HostedOptimizerConfig",
+    "SFT_ALGORITHM_ID",
+    "BetaSftExecutorClient",
+    "SftArtifact",
+    "SftConfig",
+    "SftPublicServiceClient",
+    "SftService",
+    "SftServiceError",
+    "create_sft_http_server",
+    "serve_sft_service",
     "FUTURE_HOSTED_ALGORITHMS",
     "FutureHostedAlgorithm",
     "FutureHostedAlgorithmSlug",

{synth_optimizers-0.2.12.dev20260816 → synth_optimizers-0.2.13}/src/synth_optimizers/cli.py

@@ -34,6 +34,7 @@ from .hosted import (
     HostedOptimizerError,
     validate_online_reflexion_evidence_notes,
 )
+from .sft import SftConfig, SftPublicServiceClient, SftServiceError, serve_sft_service
 from .tunnels import TunnelError, TunnelProvider
 from .victorialogs import project_gepa_run_artifacts, project_gepa_run_started
 
@@ -751,6 +752,97 @@ def _config_file_object(path: str) -> dict:
     return data
 
 
+def _sft_service_client(args: argparse.Namespace) -> SftPublicServiceClient:
+    token = os.environ.get(args.service_token_env) if args.service_token_env else None
+    return SftPublicServiceClient(args.service_url, token, timeout_seconds=args.timeout_seconds)
+
+
+def _sft_validate(args: argparse.Namespace) -> int:
+    try:
+        config = SftConfig.from_toml(
+            Path(args.config).read_text(encoding="utf-8"), run_id=args.run_id
+        )
+    except OSError as exc:
+        raise SystemExit(f"cannot read {args.config}: {exc}") from exc
+    except SftServiceError as exc:
+        raise SystemExit(str(exc)) from exc
+    payload = {
+        "algorithm": "sft",
+        "run_id": config.run_id,
+        "backend": config.backend,
+        "base_model": config.base_model,
+        "checkpoint_steps": list(config.checkpoint_steps),
+        "accelerator_slots": config.accelerator_slots,
+    }
+    print(
+        json.dumps(payload, indent=2, sort_keys=True)
+        if args.json
+        else f"valid SFT config run_id={config.run_id} backend={config.backend}"
+    )
+    return 0
+
+
+def _sft_submit(args: argparse.Namespace) -> int:
+    try:
+        config_toml = Path(args.config).read_text(encoding="utf-8")
+        client = _sft_service_client(args)
+        submitted = client.submit_toml(
+            config_toml,
+            run_id=args.run_id,
+            idempotency_key=args.idempotency_key,
+        )
+        if args.json and not args.follow:
+            print(json.dumps(submitted, indent=2, sort_keys=True))
+            return 0
+        run_id = str(submitted["run_id"])
+        print(f"submitted run_id={run_id} status={submitted.get('status', 'queued')}")
+        if not args.follow:
+            return 0
+        while True:
+            record = client.get(run_id)
+            status = str(record.get("status", "unknown"))
+            print(f"status={status}")
+            if status in {"succeeded", "failed", "cancelled"}:
+                if args.json:
+                    print(json.dumps(record, indent=2, sort_keys=True))
+                return 1 if status == "failed" else 0
+            time.sleep(args.poll_seconds)
+    except (OSError, SftServiceError) as exc:
+        raise SystemExit(str(exc)) from exc
+
+
+def _sft_watch(args: argparse.Namespace) -> int:
+    try:
+        client = _sft_service_client(args)
+        record = client.get(args.run_id)
+        if args.events:
+            page = client.optimizer_events(
+                args.run_id, after_sequence=args.after_seq, limit=args.limit
+            )
+            record["events"] = page.get("events", [])
+        print(
+            json.dumps(record, indent=2, sort_keys=True)
+            if args.json
+            else f"run_id={args.run_id} status={record.get('status')}"
+        )
+        return 1 if record.get("status") == "failed" else 0
+    except SftServiceError as exc:
+        raise SystemExit(str(exc)) from exc
+
+
+def _sft_cancel(args: argparse.Namespace) -> int:
+    try:
+        record = _sft_service_client(args).cancel(args.run_id)
+    except SftServiceError as exc:
+        raise SystemExit(str(exc)) from exc
+    print(
+        json.dumps(record, indent=2, sort_keys=True)
+        if args.json
+        else f"run_id={args.run_id} status={record.get('status')}"
+    )
+    return 0
+
+
 def _submit_hosted_gelo(args: argparse.Namespace) -> int:
     tunnel: Any | None = None
     try:
@@ -1577,6 +1669,45 @@ def build_parser() -> argparse.ArgumentParser:
         "--docs-set", default="gelo", help="Bundled docs set to serve (default: gelo)."
     )
 
+    sft = subcommands.add_parser("sft", help="Operate the public SFT control-plane service.")
+    sft_subcommands = sft.add_subparsers(dest="sft_command", required=True)
+    sft_validate = sft_subcommands.add_parser("validate")
+    sft_validate.add_argument("--config", required=True)
+    sft_validate.add_argument("--run-id")
+    sft_validate.add_argument("--json", action="store_true")
+
+    sft_service = sft_subcommands.add_parser("service")
+    sft_service.add_argument("--db", default=".sft/service.sqlite")
+    sft_service.add_argument("--bind", default="127.0.0.1:8878")
+    sft_service.add_argument(
+        "--service-token-env",
+        default="SYNTH_OPTIMIZERS_SFT_SERVICE_TOKEN",
+        help="Optional inbound bearer-token environment variable.",
+    )
+
+    for command_name in ("submit", "watch", "cancel"):
+        command = sft_subcommands.add_parser(command_name)
+        command.add_argument(
+            "--service-url",
+            default=os.environ.get("SYNTH_OPTIMIZERS_SFT_SERVICE_URL", "http://127.0.0.1:8878"),
+        )
+        command.add_argument("--service-token-env", default="SYNTH_OPTIMIZERS_SFT_SERVICE_TOKEN")
+        command.add_argument("--timeout-seconds", type=float, default=300.0)
+        command.add_argument("--json", action="store_true")
+    sft_submit = sft_subcommands.choices["submit"]
+    sft_submit.add_argument("--config", required=True)
+    sft_submit.add_argument("--run-id")
+    sft_submit.add_argument("--idempotency-key")
+    sft_submit.add_argument("--follow", action="store_true")
+    sft_submit.add_argument("--poll-seconds", type=float, default=1.0)
+    sft_watch = sft_subcommands.choices["watch"]
+    sft_watch.add_argument("run_id")
+    sft_watch.add_argument("--events", action="store_true")
+    sft_watch.add_argument("--after-seq", type=int, default=0)
+    sft_watch.add_argument("--limit", type=int, default=500)
+    sft_cancel = sft_subcommands.choices["cancel"]
+    sft_cancel.add_argument("run_id")
+
     mapo = subcommands.add_parser("mapo")
     mapo_subcommands = mapo.add_subparsers(dest="mapo_command", required=True)
     mapo_startup = mapo_subcommands.add_parser("startup")
@@ -2087,6 +2218,10 @@ def build_parser() -> argparse.ArgumentParser:
     gepa_runs_delete.add_argument("--yes", action="store_true", help="Apply the deletion.")
     gepa_runs_delete.add_argument("--json", action="store_true")
 
+    from .eval.commands import register as register_eval
+
+    register_eval(subcommands)
+
     events = subcommands.add_parser("events")
     events_subcommands = events.add_subparsers(dest="events_command", required=True)
     replay = events_subcommands.add_parser("replay")
@@ -2100,6 +2235,15 @@ def build_parser() -> argparse.ArgumentParser:
 
 def main(argv: Sequence[str] | None = None) -> int:
     args = build_parser().parse_args(argv)
+    if args.command == "eval":
+        from .eval.commands import dispatch as dispatch_eval
+        from .eval.models import EvalContractError
+
+        try:
+            return dispatch_eval(args)
+        except EvalContractError as exc:
+            print(f"error: {exc}", file=sys.stderr)
+            return 1
     if args.command == "gepa" and args.gepa_command == "run":
         from .gepa import GepaRun, UsageRegistrationConfig
 
@@ -2200,6 +2344,18 @@ def main(argv: Sequence[str] | None = None) -> int:
         docs = DocsSource([docs_root], title=args.title)
         serve_console(board, docs, host=args.host, port=args.port)
         return 0
+    if args.command == "sft" and args.sft_command == "validate":
+        return _sft_validate(args)
+    if args.command == "sft" and args.sft_command == "submit":
+        return _sft_submit(args)
+    if args.command == "sft" and args.sft_command == "watch":
+        return _sft_watch(args)
+    if args.command == "sft" and args.sft_command == "cancel":
+        return _sft_cancel(args)
+    if args.command == "sft" and args.sft_command == "service":
+        token = os.environ.get(args.service_token_env) if args.service_token_env else None
+        serve_sft_service(args.db, args.bind, service_token=token)
+        return 0
     if args.command == "mapo" and args.mapo_command == "startup":
         return _gelo_startup(args)
     if args.command == "mapo" and args.mapo_command == "submit":

synth_optimizers-0.2.13/src/synth_optimizers/eval/__init__.py

@@ -0,0 +1,118 @@
+"""`eval`: a local Optimizers algorithm that scores policy candidates.
+
+`eval` takes an immutable set of policy candidates and an allowlisted recipe,
+expands them into a fair `candidate × seed × scenario` matrix, runs each trial
+in the recipe's pinned container, and returns a promotable winner only when the
+declared gates pass.
+
+It is deliberately local. `eval` is not a hosted algorithm and must never be
+added to `synth_optimizers.hosted.OptimizerAlgorithmSlug` or to
+`future_algorithms.py`: that enum describes hosted API compatibility, which
+this is not.
+
+There are no Craftax-, GameBench-, or Harbor-specific execution paths here.
+Those are containers implementing one standard contract, `eval.target.v1`.
+"""
+
+from __future__ import annotations
+
+from .executor import (
+    ContainerRuntimeError,
+    OciTrialExecutor,
+    TrialExecution,
+    TrialExecutor,
+    TrialRunRequest,
+)
+from .home import EvalHome, RuntimeConfig
+from .models import (
+    BENCHMARK_STATUSES,
+    CANDIDATE_SET_SCHEMA,
+    CONTAINER_RESULT_SCHEMA,
+    CONTAINER_STATUSES,
+    EVAL_ALGORITHM_ID,
+    EVAL_ALGORITHM_VERSION,
+    POLICY_CANDIDATE_SCHEMA,
+    RUN_MANIFEST_SCHEMA,
+    SEED_LEDGER_SCHEMA,
+    SELECTION_STATUSES,
+    TARGET_MANIFEST_SCHEMA,
+    TRIAL_MANIFEST_SCHEMA,
+    TRIAL_RECORD_SCHEMA,
+    TRIAL_STATUSES,
+    WORKER_EVENT_SCHEMA,
+    WORKER_MANIFEST_SCHEMA,
+    CandidateScorecard,
+    CandidateSet,
+    ContainerResult,
+    EvalContractError,
+    MetricSpec,
+    PolicyCandidate,
+    SeedLedger,
+    SelectionDecision,
+    SelectionSpec,
+    TargetManifest,
+    TrialKey,
+    TrialLimits,
+    TrialRecord,
+    digest_of,
+    digest_of_tree,
+)
+from .recipes import EvalRecipe, catalog, get_recipe
+from .runner import EvalRunner, WorkerManifest, request_cancel, run_worker
+from .scoring import apply_elimination, decide, paired_lift, summarize_candidate
+from .semaphore import Lease, SemaphoreTimeout, TrialSemaphore
+
+__all__ = [
+    "BENCHMARK_STATUSES",
+    "CANDIDATE_SET_SCHEMA",
+    "CONTAINER_RESULT_SCHEMA",
+    "CONTAINER_STATUSES",
+    "EVAL_ALGORITHM_ID",
+    "EVAL_ALGORITHM_VERSION",
+    "POLICY_CANDIDATE_SCHEMA",
+    "RUN_MANIFEST_SCHEMA",
+    "SEED_LEDGER_SCHEMA",
+    "SELECTION_STATUSES",
+    "TARGET_MANIFEST_SCHEMA",
+    "TRIAL_MANIFEST_SCHEMA",
+    "TRIAL_RECORD_SCHEMA",
+    "TRIAL_STATUSES",
+    "WORKER_EVENT_SCHEMA",
+    "WORKER_MANIFEST_SCHEMA",
+    "CandidateScorecard",
+    "CandidateSet",
+    "ContainerResult",
+    "ContainerRuntimeError",
+    "EvalContractError",
+    "EvalHome",
+    "EvalRecipe",
+    "EvalRunner",
+    "Lease",
+    "MetricSpec",
+    "OciTrialExecutor",
+    "PolicyCandidate",
+    "RuntimeConfig",
+    "SeedLedger",
+    "SelectionDecision",
+    "SelectionSpec",
+    "SemaphoreTimeout",
+    "TargetManifest",
+    "TrialExecution",
+    "TrialExecutor",
+    "TrialKey",
+    "TrialLimits",
+    "TrialRecord",
+    "TrialRunRequest",
+    "TrialSemaphore",
+    "WorkerManifest",
+    "apply_elimination",
+    "catalog",
+    "decide",
+    "digest_of",
+    "digest_of_tree",
+    "get_recipe",
+    "paired_lift",
+    "request_cancel",
+    "run_worker",
+    "summarize_candidate",
+]

synth_optimizers-0.2.13/src/synth_optimizers/eval/__main__.py

@@ -0,0 +1,10 @@
+"""`python -m synth_optimizers.eval …` for app-owned worker launches."""
+
+from __future__ import annotations
+
+import sys
+
+from .commands import main
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))

synth_optimizers-0.2.13/src/synth_optimizers/eval/catalog/eval.craftax.code-policy.smoke.v1.toml

@@ -0,0 +1,55 @@
+# Craftax native code-policy smoke. Report-only: it never promotes, so a green
+# run means "the rig and the three policies produced valid evidence", not "a
+# winner was found".
+#
+# `image_digest` is intentionally absent until craftax-eval-target is published
+# and pinned; the recipe reports `unavailable` rather than pretending to a tag.
+
+[recipe]
+id = "eval.craftax.code-policy.smoke.v1"
+title = "Craftax code-policy smoke"
+description = "Score staged Craftax code policies on one pinned world, report only."
+task = "craftax"
+policy_kind = "python-code.craftax-choose-actions.v1"
+image = "craftax-eval-target"
+prerequisites = [
+    "Published craftax-eval-target image pinned by digest",
+    "OCI runtime available to the Desktop app",
+]
+scenarios = ["craftax-default-48x48"]
+screening_seeds = [101, 102]
+confirmation_seeds = []
+secrets = []
+
+[recipe.target]
+schema_version = "eval.target.v1"
+policy_kinds = ["python-code.craftax-choose-actions.v1"]
+trial_mode = "one-policy-one-seed"
+required_gates = ["policy_loaded", "verifier_completed"]
+required_artifacts = ["trace"]
+supports_live_events = true
+network = "none"
+
+[[recipe.target.metrics]]
+id = "reward"
+direction = "maximize"
+
+[[recipe.target.metrics]]
+id = "achievements"
+direction = "maximize"
+
+[recipe.selection]
+primary_metric = "reward"
+min_lift = 0.0
+min_valid_trials = 2
+decision_mode = "report_only"
+
+[recipe.selection.elimination]
+kind = "none"
+
+[recipe.limits]
+max_parallel_trials = 2
+timeout_seconds = 300
+cpus = 2.0
+memory_mb = 4096
+max_output_bytes = 268435456

synth_optimizers-0.2.13/src/synth_optimizers/eval/catalog/eval.craftax.llm-policy.smoke.v1.toml

@@ -0,0 +1,71 @@
+# Craftax LLM code-policy smoke. The candidate is data — an allowlisted model
+# and reasoning effort — so comparing two efforts is a candidate set, not a
+# code change.
+#
+# `network = "bridge"` and one declared secret are the minimum a paid policy
+# needs; the route, the rates, and the caps stay recipe-owned so a candidate
+# cannot bill against an endpoint the product did not name.
+
+[recipe]
+id = "eval.craftax.llm-policy.smoke.v1"
+title = "Craftax LLM policy smoke"
+description = "Score allowlisted LLM policies on one pinned Craftax world, report only."
+task = "craftax"
+policy_kind = "llm-policy.v1"
+image = "craftax-eval-target"
+prerequisites = [
+    "Published craftax-eval-target image pinned by digest",
+    "OPENAI_API_KEY in the eval home's secrets.toml",
+]
+scenarios = ["craftax-default-48x48"]
+screening_seeds = [101, 102]
+confirmation_seeds = []
+secrets = ["OPENAI_API_KEY"]
+
+[recipe.budget]
+max_llm_calls = 20
+max_usd = 0.30
+
+[[recipe.models]]
+id = "gpt-5.6-luna"
+route = "https://api.openai.com/v1/chat/completions"
+secret = "OPENAI_API_KEY"
+efforts = ["none", "low", "medium", "high"]
+usd_per_1m_input = 0.2
+usd_per_1m_output = 1.2
+usd_per_1m_cached_input = 0.02
+price_source = "https://developers.openai.com/api/docs/pricing (standard tier)"
+price_as_of = "2026-07-30"
+
+[recipe.target]
+schema_version = "eval.target.v1"
+policy_kinds = ["llm-policy.v1"]
+trial_mode = "one-policy-one-seed"
+required_gates = ["policy_loaded", "verifier_completed"]
+required_artifacts = ["trace"]
+supports_live_events = true
+network = "bridge"
+
+[[recipe.target.metrics]]
+id = "reward"
+direction = "maximize"
+
+[[recipe.target.metrics]]
+id = "achievements"
+direction = "maximize"
+
+[recipe.selection]
+primary_metric = "reward"
+min_lift = 0.0
+min_valid_trials = 2
+decision_mode = "report_only"
+
+[recipe.selection.elimination]
+kind = "none"
+
+[recipe.limits]
+max_parallel_trials = 4
+timeout_seconds = 1800
+cpus = 2.0
+memory_mb = 4096
+max_output_bytes = 268435456