agentinputlab 0.1.0a1__py3-none-any.whl

agentinputlab/recipes/skillopt_environment.py

@@ -0,0 +1,165 @@
+"""Explicit caller-owned EnvAdapter extension for the pinned SkillOpt trainer.
+
+The factory is trusted Python code, not a sandbox. Source and configuration are
+checked before import. Native training, reflection, selection and CLI controls
+remain in the caller-mounted upstream checkout.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import importlib.util
+import json
+import re
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from agentinputlab.artifacts import load_pinned_json_mapping
+from agentinputlab.contracts import ExternalDependencyError, ValidationError
+from agentinputlab.contracts.serialization import content_hash, to_jsonable
+
+ENVIRONMENT_RECEIPT = "agentinputlab_environment_binding.json"
+_SHA = re.compile(r"[0-9a-f]{64}")
+
+
+@dataclass(frozen=True, slots=True)
+class SkillOptEnvironmentBinding:
+    factory_path: str | Path
+    factory_sha256: str
+    configuration_path: str | Path
+    configuration_sha256: str
+    factory_name: str = "create"
+
+    def __post_init__(self) -> None:
+        for name in ("factory_sha256", "configuration_sha256"):
+            if not isinstance(getattr(self, name), str) or not _SHA.fullmatch(getattr(self, name)):
+                raise ValidationError(f"SkillOpt environment {name} must be a SHA-256")
+        for name in ("factory_path", "configuration_path"):
+            if not isinstance(getattr(self, name), (str, Path)) or not str(getattr(self, name)).strip():
+                raise ValidationError(f"SkillOpt environment {name} must be a path")
+        if (not isinstance(self.factory_name, str) or not self.factory_name.isidentifier()
+                or self.factory_name.startswith("_")):
+            raise ValidationError("SkillOpt environment factory_name must be a public identifier")
+
+    def verify(self) -> dict[str, Any]:
+        factory = Path(self.factory_path).expanduser().absolute()
+        if (factory.is_symlink() or not factory.is_file() or factory.suffix != ".py"
+                or factory.stat().st_size > 1024 * 1024):
+            raise ExternalDependencyError("SkillOpt environment factory must be a bounded regular Python file")
+        if hashlib.sha256(factory.read_bytes()).hexdigest() != self.factory_sha256:
+            raise ExternalDependencyError("SkillOpt environment factory checksum changed")
+        config = Path(self.configuration_path).expanduser().absolute()
+        load_pinned_json_mapping(config, expected_sha256=self.configuration_sha256,
+                                 max_bytes=8 * 1024 * 1024)
+        return dict(factory_path=str(factory.resolve()), factory_sha256=self.factory_sha256,
+                    configuration_path=str(config.resolve()),
+                    configuration_sha256=self.configuration_sha256, factory_name=self.factory_name)
+
+    def command_options(self) -> tuple[str, ...]:
+        return ("--factory", str(Path(self.factory_path).expanduser().absolute()),
+                "--factory-sha256", self.factory_sha256,
+                "--factory-name", self.factory_name,
+                "--factory-config", str(Path(self.configuration_path).expanduser().absolute()),
+                "--factory-config-sha256", self.configuration_sha256)
+
+
+def run_with_environment(*, checkout, binding: SkillOptEnvironmentBinding,
+                         receipt_path, native_args: list[str]) -> None:
+    from .skillopt import validate_skillopt_checkout
+
+    root = validate_skillopt_checkout(checkout)
+    identity = binding.verify()
+    configuration = load_pinned_json_mapping(binding.configuration_path,
+        expected_sha256=binding.configuration_sha256, max_bytes=8 * 1024 * 1024)
+    receipt = Path(receipt_path)
+    if receipt.exists() or receipt.is_symlink() or not receipt.parent.is_dir():
+        raise ValidationError("SkillOpt environment receipt must be a new file in the fit output")
+    prior_path, prior_argv = list(sys.path), list(sys.argv)
+    native_name = "_agentinputlab_skillopt_cli_" + content_hash(str(root))[:16]
+    factory_name = "_agentinputlab_skillopt_factory_" + binding.factory_sha256[:16]
+    previous = {name: sys.modules.get(name) for name in (native_name, factory_name)}
+    try:
+        sys.path.insert(0, str(root))
+        spec = importlib.util.spec_from_file_location(native_name, root / "scripts/train.py")
+        if spec is None or spec.loader is None:
+            raise ExternalDependencyError("SkillOpt native CLI cannot be imported")
+        native = importlib.util.module_from_spec(spec)
+        sys.modules[native_name] = native
+        spec.loader.exec_module(native)
+        spec = importlib.util.spec_from_file_location(factory_name, identity["factory_path"])
+        if spec is None or spec.loader is None:
+            raise ExternalDependencyError("SkillOpt environment factory cannot be imported")
+        factory_module = importlib.util.module_from_spec(spec)
+        sys.modules[factory_name] = factory_module
+        spec.loader.exec_module(factory_module)
+        factory = getattr(factory_module, binding.factory_name, None)
+        if not callable(factory):
+            raise ValidationError("SkillOpt environment factory is not callable")
+        from skillopt.envs.base import EnvAdapter
+
+        bound = False
+
+        def get_adapter(cfg):
+            nonlocal bound
+            if bound:
+                raise ValidationError("SkillOpt native CLI requested a second environment binding")
+            binding.verify()
+            adapter = factory(cfg=dict(cfg), configuration=to_jsonable(configuration))
+            if not isinstance(adapter, EnvAdapter) or not callable(getattr(adapter, "fingerprint", None)):
+                raise ValidationError("SkillOpt factory must return a native EnvAdapter with a fingerprint")
+            fingerprint = to_jsonable(adapter.fingerprint())
+            if not isinstance(fingerprint, dict) or not fingerprint:
+                raise ValidationError("SkillOpt environment fingerprint must be a nonempty mapping")
+            content_hash(fingerprint)
+            binding.verify()
+            value = dict(schema="agentinputlab.skillopt.environment_binding.v1",
+                         binding=identity, environment_fingerprint=fingerprint,
+                         native_training_algorithm_unchanged=True,
+                         profile_kind="caller_environment_adapter")
+            with receipt.open("x", encoding="utf-8") as stream:
+                json.dump(value, stream, sort_keys=True, indent=2, allow_nan=False)
+            bound = True
+            return adapter
+
+        native.get_adapter = get_adapter
+        sys.argv = [str(root / "scripts/train.py"), *native_args]
+        native.main()
+        if not bound:
+            raise ValidationError("SkillOpt native CLI did not instantiate the configured environment")
+        binding.verify()
+        validate_skillopt_checkout(root)
+    finally:
+        sys.path[:] = prior_path
+        sys.argv[:] = prior_argv
+        for name, value in previous.items():
+            if value is None:
+                sys.modules.pop(name, None)
+            else:
+                sys.modules[name] = value
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = list(sys.argv[1:] if argv is None else argv)
+    if "--" not in args:
+        raise ValidationError("SkillOpt environment wrapper requires explicit native arguments after --")
+    boundary = args.index("--")
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--checkout", required=True)
+    parser.add_argument("--factory", required=True)
+    parser.add_argument("--factory-sha256", required=True)
+    parser.add_argument("--factory-name", default="create")
+    parser.add_argument("--factory-config", required=True)
+    parser.add_argument("--factory-config-sha256", required=True)
+    parser.add_argument("--receipt", required=True)
+    a = parser.parse_args(args[:boundary])
+    binding = SkillOptEnvironmentBinding(a.factory, a.factory_sha256, a.factory_config,
+                                        a.factory_config_sha256, a.factory_name)
+    run_with_environment(checkout=a.checkout, binding=binding,
+                         receipt_path=a.receipt, native_args=args[boundary + 1:])
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agentinputlab/recipes/skillopt_environment_cli.py

@@ -0,0 +1,5 @@
+"""Child-process entry point for an explicitly bound SkillOpt EnvAdapter."""
+from .skillopt_environment import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agentinputlab/recipes/skillopt_retrieval.py

@@ -0,0 +1,188 @@
+"""IR environment for SkillOpt's source-native text-space optimizer.
+
+Create the adapter inside a trusted environment factory after mounting the
+pinned SkillOpt checkout. The trainer keeps its native six-stage update loop;
+this environment owns query generation, retrieval and scalar native feedback.
+"""
+from __future__ import annotations
+
+import json
+import math
+import random
+from dataclasses import replace
+from pathlib import Path
+
+from agentinputlab.backends.llm import LLMBackend
+from agentinputlab.backends.retrieval import Retriever
+from agentinputlab.benchmarks.base import BenchmarkAdapter, BenchmarkExample
+from agentinputlab.benchmarks.ir_query import ContextIRQueryPredictor, IRQueryOutputError
+from agentinputlab.benchmarks.retrieval import RetrievalPrediction
+from agentinputlab.contracts import AlgorithmOutputError, ContextBlock, ExternalDependencyError, ValidationError
+from agentinputlab.contracts.serialization import content_hash, to_jsonable
+
+
+def create_skillopt_retrieval_adapter(
+    *, adapter: BenchmarkAdapter, retriever: Retriever, target_backend: LLMBackend,
+    train, validation, cfg: dict, metric: str = "ndcg_at_10", top_k: int = 15,
+    per_call_output_tokens: int = 512, seed: int = 17, allow_json_fence: bool = False,
+    partition_label: str = "caller_declared_compatibility_partitions",
+):
+    """Return a native EnvAdapter with disjoint fit-only query groups.
+
+    The backend must share the fit's metered provider with the optimizer.
+    ``hard`` is native metric > 0 and ``soft`` is that unchanged metric. Invalid
+    query JSON receives zero training feedback with its paid output retained.
+    Held-out groups are evaluated separately and are never accepted here.
+    """
+    try:
+        from skillopt.envs.base import EnvAdapter
+    except ImportError as exc:
+        raise ExternalDependencyError("mount the pinned SkillOpt checkout before creating its IR environment") from exc
+    if not isinstance(adapter, BenchmarkAdapter) or not isinstance(retriever, Retriever):
+        raise ValidationError("SkillOpt IR requires an explicit dataset adapter and retriever")
+    if not isinstance(target_backend, LLMBackend):
+        raise ValidationError("SkillOpt IR requires an explicit metered target backend")
+    if not isinstance(cfg, dict) or not isinstance(metric, str) or not metric.strip():
+        raise ValidationError("SkillOpt IR requires native configuration and an explicit metric")
+    if not isinstance(partition_label, str) or not partition_label.strip():
+        raise ValidationError("SkillOpt IR partition label must be explicit")
+    groups = {"train":tuple(train), "validation":tuple(validation)}
+    examples = (*groups["train"], *groups["validation"])
+    if any(not group for group in groups.values()) or any(not isinstance(e, BenchmarkExample) for e in examples):
+        raise ValidationError("SkillOpt IR requires nonempty train and validation examples")
+    if len({e.example_id for e in examples}) != len(examples) or len({
+        content_hash(dict(query=e.query, instruction=e.instruction, context=e.visible_context,
+                          messages=e.visible_messages)) for e in examples
+    }) != len(examples):
+        raise ValidationError("SkillOpt IR train/validation IDs and visible inputs must be disjoint")
+    if any(e.visible_tool_names for e in examples):
+        raise ValidationError("SkillOpt IR environment does not execute tools")
+    dataset = to_jsonable(adapter.fingerprint())
+    corpus = to_jsonable(retriever.fingerprint())
+    # Validate endpoint controls before the trainer can start a paid call.
+    ContextIRQueryPredictor(target_backend, retriever, max_tokens=per_call_output_tokens,
+                            top_k=top_k, seed=seed, allow_json_fence=allow_json_fence)
+
+    def metric_value(metrics):
+        value = metrics.get(metric)
+        if type(value) not in (float,int) or not math.isfinite(value) or not 0 <= value <= 1:
+            raise ValidationError("SkillOpt IR native metric must be present and within [0,1]")
+        return float(value)
+
+    for e in examples:
+        metric_value(adapter.score(e, RetrievalPrediction({})))
+    owned = {content_hash(e.example_id): e for e in examples}
+
+    class RetrievalEnvAdapter(EnvAdapter):
+        def __init__(self):
+            self.analyst_workers = cfg.get("analyst_workers", 1)
+            self.failure_only = cfg.get("failure_only", False)
+            self.minibatch_size = cfg.get("minibatch_size", 2)
+            self.edit_budget = cfg.get("edit_budget", 4)
+            self._cfg = dict(cfg)
+
+        def fingerprint(self):
+            return dict(implementation="agentinputlab.skillopt.native_IR_environment.v1",
+                        dataset=dataset, retriever=corpus, target_backend=to_jsonable(target_backend.fingerprint()),
+                        metric=metric, hard_criterion="declared_native_metric_strictly_greater_than_zero",
+                        top_k=top_k, per_call_output_tokens=per_call_output_tokens, seed=seed,
+                        allow_json_fence=allow_json_fence, partition_label=partition_label,
+                        groups={split:[dict(case_id=e.example_id, state_sha256=e.visible_state().state_hash,
+                                gold_sha256=content_hash(e.evaluator_gold)) for e in cases]
+                                for split,cases in groups.items()},
+                        heldout_not_exposed=True, automatic_retry=False,
+                        native_reflection_patch_merge_and_validation_gate=True)
+
+        def _batch(self, split, count, batch_seed):
+            if type(count) is not int or count < 1 or count > len(groups[split]):
+                raise ValidationError("SkillOpt IR batch exceeds the declared group")
+            cases = random.Random(batch_seed).sample(list(groups[split]), count)
+            return [dict(id=content_hash(e.example_id), question=e.query,
+                         task_type=adapter.benchmark_id) for e in cases]
+
+        def build_train_env(self, batch_size, seed, **kwargs):
+            return self._batch("train", batch_size, seed)
+
+        def build_eval_env(self, env_num, split, seed, **kwargs):
+            if split not in {"val", "validation", "valid_seen"}:
+                raise ValidationError("SkillOpt IR fit environment exposes validation only; heldout evaluation is separate")
+            return self._batch("validation", env_num, seed)
+
+        def get_task_types(self):
+            return [adapter.benchmark_id]
+
+        def rollout(self, env_manager, skill_content, out_dir, **kwargs):
+            if not isinstance(skill_content, str) or not skill_content.strip():
+                raise ValidationError("SkillOpt IR candidate skill must be nonempty text")
+            if any(kwargs.get(name) for name in ("diagnostic_mode", "diagnostic_instruction", "diagnostic_trace_context_by_id")):
+                raise ValidationError("SkillOpt IR diagnostic extra attempts require a separate explicit protocol")
+            if (to_jsonable(adapter.fingerprint()) != dataset
+                    or to_jsonable(retriever.fingerprint()) != corpus):
+                raise ValidationError("SkillOpt IR source dataset or corpus changed")
+            root = Path(out_dir)
+            root.mkdir(parents=True, exist_ok=True)
+            results = []
+            rows = list(env_manager)
+            if any(not isinstance(item, dict) or item.get("id") not in owned for item in rows):
+                raise ValidationError("SkillOpt IR rollout contains an undeclared fit case")
+            if len({item["id"] for item in rows}) != len(rows):
+                raise ValidationError("SkillOpt IR rollout contains duplicate cases")
+            for item in rows:
+                e = owned[item["id"]]
+                if item != dict(id=content_hash(e.example_id), question=e.query, task_type=adapter.benchmark_id):
+                    raise ValidationError("SkillOpt IR visible fit item changed")
+                folder = root / "predictions" / item["id"]
+                folder.mkdir(parents=True, exist_ok=False)
+                state = e.visible_state()
+                candidate = ContextBlock(
+                    text="SKILLOPT CURRENT FITTING SKILL\n" + skill_content,
+                    block_id="skillopt-fit-skill:" + content_hash(skill_content),
+                    source="SkillOpt native fitting candidate")
+                state = replace(state, context=(*state.context, candidate))
+                row = dict(id=item["id"], source_case_id=e.example_id, question=e.query,
+                           task_type=adapter.benchmark_id, hard=0, soft=0.0, response="",
+                           agent_ok=False, n_turns=1, fail_reason="", native_metrics=None)
+                predictor = ContextIRQueryPredictor(target_backend, retriever, max_tokens=per_call_output_tokens,
+                    top_k=top_k, seed=seed, allow_json_fence=allow_json_fence)
+                try:
+                    reply = predictor.predict(state)
+                    metrics = dict(adapter.score(e, reply.prediction))
+                    value = metric_value(metrics)
+                    row.update(hard=int(value > 0), soft=value, agent_ok=True, native_metrics=metrics,
+                               response=reply.metadata["generation_response"]["text"],
+                               endpoint_metadata=to_jsonable(reply.metadata), usage=to_jsonable(reply.usage))
+                    if value == 0:
+                        row["fail_reason"] = f"native {metric}=0"
+                except IRQueryOutputError as exc:
+                    if not isinstance(exc.__cause__, AlgorithmOutputError):
+                        with (folder / "consumed-execution-error.json").open("x") as f:
+                            json.dump(dict(error_type=type(exc).__name__, response=to_jsonable(exc.response),
+                                           request=to_jsonable(exc.request), usage=to_jsonable(exc.usage),
+                                           automatic_retry=False), f, sort_keys=True, allow_nan=False)
+                        raise
+                    row.update(response=exc.response.text, agent_ok=True, soft=0.0, hard=0,
+                               fail_reason=str(exc), query_or_retrieval_failure=True,
+                               generation_request=to_jsonable(exc.request), generation_response=to_jsonable(exc.response),
+                               usage=to_jsonable(exc.usage))
+                except Exception as exc:
+                    with (folder / "execution-error.json").open("x") as f:
+                        json.dump(dict(error_type=type(exc).__name__, automatic_retry=False), f)
+                    raise
+                conversation = [dict(type="message", turn=1, content=row["response"]),
+                                dict(role="system", content="[NATIVE IR EVALUATION]\n" +
+                                     json.dumps(dict(metric=metric, score=row["soft"],
+                                                     valid_native_metric=row["native_metrics"] is not None)))]
+                with (folder / "conversation.json").open("x") as f:
+                    json.dump(conversation, f, sort_keys=True, indent=2, allow_nan=False)
+                with (folder / "result.json").open("x") as f:
+                    json.dump(row, f, sort_keys=True, indent=2, allow_nan=False)
+                results.append(row)
+            with (root / "results.jsonl").open("x") as f:
+                for row in results:
+                    f.write(json.dumps(row, sort_keys=True, allow_nan=False) + "\n")
+            return results
+
+    return RetrievalEnvAdapter()
+
+
+__all__ = ["create_skillopt_retrieval_adapter"]