agent-learning-kit 0.1.0__py3-none-any.whl

fi/opt/simulation.py

@@ -0,0 +1,260 @@
+from __future__ import annotations
+
+import asyncio
+import inspect
+from typing import Any, Callable, Dict, Iterable, Mapping, Optional
+
+from .evidence import score_simulation_evidence
+from .targets import AgentCandidate, CandidateEvaluation
+
+
+class SimulationEvaluator:
+    """
+    Evaluates an AgentCandidate through simulate-sdk and optionally ai-evaluation.
+
+    The bridge is dependency-light by design:
+    - pass fake runner/evaluate functions in tests,
+    - or install `agent-simulate` and `ai-evaluation` for real runs.
+    """
+
+    def __init__(
+        self,
+        *,
+        agent_factory: Optional[Callable[[AgentCandidate], Any]] = None,
+        scenario: Any = None,
+        topic: Optional[str] = None,
+        runner: Any = None,
+        runner_cls: Any = None,
+        runner_kwargs: Optional[
+            Mapping[str, Any] | Callable[[AgentCandidate], Mapping[str, Any]]
+        ] = None,
+        eval_specs: Optional[Iterable[Dict[str, Any]]] = None,
+        eval_templates: Optional[Iterable[str]] = None,
+        evaluate_report_fn: Optional[Callable[..., Any]] = None,
+        evaluate_report_kwargs: Optional[Dict[str, Any]] = None,
+        agent_report_config: Optional[
+            Mapping[str, Any] | Callable[[AgentCandidate], Mapping[str, Any]]
+        ] = None,
+        agent_report_threshold: float = 0.7,
+        use_agent_report_evaluator: bool = False,
+        report_scorer: Optional[Callable[[Any, AgentCandidate], float]] = None,
+        evidence_scorer_config: Optional[Mapping[str, Any]] = None,
+    ) -> None:
+        self.agent_factory = agent_factory
+        self.scenario = scenario
+        self.topic = topic
+        self.runner = runner
+        self.runner_cls = runner_cls
+        self.runner_kwargs = runner_kwargs or {}
+        self.eval_specs = list(eval_specs) if eval_specs is not None else None
+        self.eval_templates = list(eval_templates) if eval_templates is not None else None
+        self.evaluate_report_fn = evaluate_report_fn
+        self.evaluate_report_kwargs = evaluate_report_kwargs or {}
+        self.agent_report_config = agent_report_config
+        self.agent_report_threshold = agent_report_threshold
+        self.use_agent_report_evaluator = use_agent_report_evaluator
+        self.report_scorer = report_scorer
+        self.evidence_scorer_config = (
+            dict(evidence_scorer_config) if evidence_scorer_config is not None else None
+        )
+
+    def evaluate_candidate(self, candidate: AgentCandidate) -> CandidateEvaluation:
+        agent = self._build_agent(candidate)
+        runner = self._get_runner()
+        run_kwargs = {
+            **self._runner_kwargs(candidate),
+            **candidate.config.get("simulation", {}),
+        }
+        if self.scenario is not None:
+            run_kwargs["scenario"] = self.scenario
+        if self.topic is not None:
+            run_kwargs["topic"] = self.topic
+
+        report = _run_sync(
+            runner.run_test(
+                agent_callback=agent,
+                **run_kwargs,
+            )
+        )
+
+        if self.eval_specs is not None or self.eval_templates is not None:
+            report = self._evaluate_report(report)
+
+        agent_report_evaluation = None
+        if self.use_agent_report_evaluator or self.agent_report_config is not None:
+            agent_report_evaluation = self._evaluate_agent_report(report, candidate)
+
+        evidence_evaluation = None
+        if self.evidence_scorer_config is not None and agent_report_evaluation is None:
+            evidence_evaluation = score_simulation_evidence(
+                report,
+                candidate=candidate,
+                config=self.evidence_scorer_config,
+            )
+
+        score_source = (
+            agent_report_evaluation
+            if agent_report_evaluation is not None
+            else evidence_evaluation
+            if evidence_evaluation is not None
+            else report
+        )
+        score = self._score_report(score_source, candidate)
+        metadata = {"source": "simulate-sdk"}
+        if agent_report_evaluation is not None:
+            metadata["agent_report_evaluation"] = _dump_model(agent_report_evaluation)
+        if evidence_evaluation is not None:
+            metadata["simulation_evidence_score"] = evidence_evaluation.metadata.get(
+                "simulation_evidence_score"
+            )
+        return CandidateEvaluation(
+            candidate=candidate,
+            score=score,
+            report=report,
+            metadata=metadata,
+        )
+
+    def _build_agent(self, candidate: AgentCandidate) -> Any:
+        if self.agent_factory is not None:
+            return self.agent_factory(candidate)
+        agent = candidate.config.get("agent_callback") or candidate.config.get("agent")
+        if agent is None:
+            raise ValueError(
+                "SimulationEvaluator needs an agent_factory or candidate.config['agent_callback']."
+            )
+        return agent
+
+    def _get_runner(self) -> Any:
+        if self.runner is not None:
+            return self.runner
+        if self.runner_cls is not None:
+            return self.runner_cls()
+        try:
+            from fi.simulate import TestRunner
+        except Exception as exc:  # pragma: no cover - import clarity
+            raise RuntimeError(
+                "agent-simulate is required for SimulationEvaluator unless runner/runner_cls is provided."
+            ) from exc
+        return TestRunner()
+
+    def _evaluate_report(self, report: Any) -> Any:
+        evaluate_report = self.evaluate_report_fn
+        if evaluate_report is None:
+            try:
+                from fi.simulate.evaluation import evaluate_report as imported
+            except Exception as exc:  # pragma: no cover - import clarity
+                raise RuntimeError(
+                    "simulate-sdk evaluate_report is required unless evaluate_report_fn is provided."
+                ) from exc
+            evaluate_report = imported
+
+        kwargs = dict(self.evaluate_report_kwargs)
+        if self.eval_specs is not None:
+            kwargs["eval_specs"] = self.eval_specs
+        if self.eval_templates is not None:
+            kwargs["eval_templates"] = self.eval_templates
+        return evaluate_report(report, **kwargs)
+
+    def _evaluate_agent_report(
+        self,
+        report: Any,
+        candidate: AgentCandidate,
+    ) -> Any:
+        config = self._agent_report_config(candidate)
+        try:
+            from fi.simulate.evaluation import evaluate_agent_report
+        except Exception:
+            try:
+                from fi.evals.metrics.agents import evaluate_agent_report
+            except Exception as exc:  # pragma: no cover - import clarity
+                raise RuntimeError(
+                    "SimulationEvaluator local agent report scoring requires "
+                    "simulate-sdk with evaluate_agent_report or ai-evaluation>=1.1."
+                ) from exc
+
+        return evaluate_agent_report(
+            report,
+            config=config,
+            threshold=self.agent_report_threshold,
+        )
+
+    def _agent_report_config(self, candidate: AgentCandidate) -> Dict[str, Any]:
+        config = self.agent_report_config
+        if callable(config):
+            return dict(config(candidate))
+        return dict(config or {})
+
+    def _runner_kwargs(self, candidate: AgentCandidate) -> Dict[str, Any]:
+        config = self.runner_kwargs
+        if callable(config):
+            return dict(config(candidate))
+        return dict(config or {})
+
+    def _score_report(self, report: Any, candidate: AgentCandidate) -> float:
+        if self.report_scorer is not None:
+            return float(self.report_scorer(report, candidate))
+        direct_score = _coerce_score(getattr(report, "score", None))
+        if direct_score is not None:
+            return direct_score
+        scores = list(_iter_report_scores(report))
+        if not scores:
+            return 0.0
+        return sum(scores) / len(scores)
+
+
+def _run_sync(value: Any) -> Any:
+    if not inspect.isawaitable(value):
+        return value
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(value)
+    raise RuntimeError(
+        "SimulationEvaluator.evaluate_candidate() was called from a running event loop. "
+        "Pass a synchronous runner or call it outside the event loop for now."
+    )
+
+
+def _iter_report_scores(report: Any):
+    for result in getattr(report, "results", []) or []:
+        evaluation = getattr(result, "evaluation", None)
+        if not isinstance(evaluation, dict):
+            continue
+        for item in evaluation.values():
+            if isinstance(item, dict):
+                for key in ("score", "output", "value"):
+                    value = item.get(key)
+                    score = _coerce_score(value)
+                    if score is not None:
+                        yield score
+                        break
+            else:
+                score = _coerce_score(item)
+                if score is not None:
+                    yield score
+
+
+def _coerce_score(value: Any) -> Optional[float]:
+    if isinstance(value, bool):
+        return 1.0 if value else 0.0
+    if isinstance(value, (int, float)):
+        return max(0.0, min(1.0, float(value)))
+    if isinstance(value, str):
+        lowered = value.strip().lower()
+        if lowered in {"pass", "passed", "true", "yes"}:
+            return 1.0
+        if lowered in {"fail", "failed", "false", "no"}:
+            return 0.0
+        try:
+            return max(0.0, min(1.0, float(lowered)))
+        except ValueError:
+            return None
+    return None
+
+
+def _dump_model(value: Any) -> Any:
+    if hasattr(value, "model_dump"):
+        return value.model_dump()
+    if hasattr(value, "dict"):
+        return value.dict()
+    return value

fi/opt/targets.py

@@ -0,0 +1,232 @@
+from __future__ import annotations
+
+import copy
+import hashlib
+import itertools
+import json
+from typing import Any, Dict, Iterable, List, Literal, Optional
+
+from pydantic import BaseModel, Field
+
+
+OptimizationLayer = Literal[
+    "objective",
+    "harness",
+    "integration",
+    "framework",
+    "streaming",
+    "world",
+    "security",
+    "perception",
+    "prompt",
+    "planner",
+    "autonomy",
+    "policy",
+    "tools",
+    "memory",
+    "router",
+    "graph",
+    "retrieval",
+    "retriever",
+    "model",
+    "voice",
+    "browser",
+    "cua",
+    "multi_agent",
+    "orchestration",
+    "action",
+    "environment",
+    "implementation",
+    "evaluator",
+    "custom",
+]
+
+
+class AgentCandidate(BaseModel):
+    """
+    A concrete agent/workflow configuration to evaluate.
+
+    `config` is intentionally framework-neutral. It can represent a LangGraph
+    graph config, CrewAI crew inputs, LiveKit voice settings, Pipecat pipeline
+    parameters, browser/CUA policy, tool schemas, memory settings, or a plain
+    prompt template.
+    """
+
+    id: str
+    config: Dict[str, Any]
+    target_name: Optional[str] = None
+    layers: List[OptimizationLayer] = Field(default_factory=list)
+    parent_id: Optional[str] = None
+    patch: Dict[str, Any] = Field(default_factory=dict)
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+    @classmethod
+    def from_config(
+        cls,
+        config: Dict[str, Any],
+        *,
+        target_name: Optional[str] = None,
+        layers: Optional[List[OptimizationLayer]] = None,
+        parent_id: Optional[str] = None,
+        patch: Optional[Dict[str, Any]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> "AgentCandidate":
+        payload = {
+            "target_name": target_name,
+            "config": config,
+            "patch": patch or {},
+        }
+        digest = hashlib.sha256(
+            json.dumps(payload, sort_keys=True, default=str).encode("utf-8")
+        ).hexdigest()[:16]
+        return cls(
+            id=f"candidate_{digest}",
+            config=copy.deepcopy(config),
+            target_name=target_name,
+            layers=list(layers or []),
+            parent_id=parent_id,
+            patch=copy.deepcopy(patch or {}),
+            metadata=copy.deepcopy(metadata or {}),
+        )
+
+    def get_path(self, path: str, default: Any = None) -> Any:
+        current: Any = self.config
+        for part in _split_path(path):
+            if isinstance(current, dict) and part in current:
+                current = current[part]
+            elif isinstance(current, list) and part.isdigit() and int(part) < len(current):
+                current = current[int(part)]
+            else:
+                return default
+        return current
+
+    def with_patch(
+        self,
+        patch: Dict[str, Any],
+        *,
+        layers: Optional[List[OptimizationLayer]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> "AgentCandidate":
+        new_config = copy.deepcopy(self.config)
+        for path, value in patch.items():
+            set_path(new_config, path, value)
+        merged_metadata = {**self.metadata, **(metadata or {})}
+        return AgentCandidate.from_config(
+            new_config,
+            target_name=self.target_name,
+            layers=layers or self.layers,
+            parent_id=self.id,
+            patch=patch,
+            metadata=merged_metadata,
+        )
+
+
+class OptimizationTarget(BaseModel):
+    """
+    Framework-neutral optimization target.
+
+    `search_space` maps dot paths to candidate values. Examples:
+    - `prompt.system`: ["Be concise", "Ask one clarifying question first"]
+    - `tools.0.description`: ["Search orders by id", "Search orders by id and email"]
+    - `memory.strategy`: ["buffer", "summary", "vector"]
+    - `router.default_model`: ["gpt-4o-mini", "claude-haiku"]
+    - `voice.vad.min_silence_duration`: [0.1, 0.3, 0.5]
+    - `browser.policy.allow_cross_origin`: [False, True]
+    """
+
+    name: str
+    base_config: Dict[str, Any]
+    layers: List[OptimizationLayer] = Field(default_factory=lambda: ["prompt"])
+    search_space: Dict[str, List[Any]] = Field(default_factory=dict)
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+    def seed_candidate(self) -> AgentCandidate:
+        return AgentCandidate.from_config(
+            self.base_config,
+            target_name=self.name,
+            layers=self.layers,
+            metadata={"kind": "seed", **self.metadata},
+        )
+
+    def iter_candidates(
+        self,
+        *,
+        include_seed: bool = True,
+        max_candidates: Optional[int] = None,
+    ) -> Iterable[AgentCandidate]:
+        count = 0
+        if include_seed:
+            yield self.seed_candidate()
+            count += 1
+            if max_candidates is not None and count >= max_candidates:
+                return
+
+        if not self.search_space:
+            return
+
+        paths = list(self.search_space.keys())
+        value_lists = [self.search_space[path] for path in paths]
+        for values in itertools.product(*value_lists):
+            patch = dict(zip(paths, values))
+            # Avoid duplicating the seed when every patch value equals base config.
+            if all(self.seed_candidate().get_path(path) == value for path, value in patch.items()):
+                continue
+            yield self.seed_candidate().with_patch(
+                patch,
+                metadata={"kind": "search", "search_paths": paths, **self.metadata},
+            )
+            count += 1
+            if max_candidates is not None and count >= max_candidates:
+                return
+
+
+class CandidateEvaluation(BaseModel):
+    """Score and evidence for one evaluated candidate."""
+
+    candidate: AgentCandidate
+    score: float
+    reason: str = ""
+    individual_results: List[Any] = Field(default_factory=list)
+    report: Any = None
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+
+def set_path(config: Dict[str, Any], path: str, value: Any) -> None:
+    parts = _split_path(path)
+    if not parts:
+        raise ValueError("Path cannot be empty.")
+
+    current: Any = config
+    for index, part in enumerate(parts[:-1]):
+        next_part = parts[index + 1]
+        if isinstance(current, list):
+            if not part.isdigit():
+                raise ValueError(f"Expected numeric list index in path '{path}'.")
+            list_index = int(part)
+            _ensure_list_size(current, list_index)
+            if current[list_index] is None:
+                current[list_index] = [] if next_part.isdigit() else {}
+            current = current[list_index]
+        else:
+            if part not in current or current[part] is None:
+                current[part] = [] if next_part.isdigit() else {}
+            current = current[part]
+
+    final = parts[-1]
+    if isinstance(current, list):
+        if not final.isdigit():
+            raise ValueError(f"Expected numeric list index in path '{path}'.")
+        list_index = int(final)
+        _ensure_list_size(current, list_index)
+        current[list_index] = value
+    else:
+        current[final] = value
+
+
+def _split_path(path: str) -> List[str]:
+    return [part for part in path.split(".") if part]
+
+
+def _ensure_list_size(items: List[Any], index: int) -> None:
+    while len(items) <= index:
+        items.append(None)

fi/opt/types.py

@@ -0,0 +1,66 @@
+from pydantic import BaseModel, Field
+from typing import Any, List, Dict, Optional
+
+
+class LLMMessage(BaseModel):
+    """Every message sent and received by the LLM MUST follow this format."""
+
+    role: str
+    content: str
+    name: Optional[str] = None
+    function_call: Optional[str] = None
+    tool_call_id: Optional[str] = None
+
+
+class EvaluationResult(BaseModel):
+    """
+    A standardized  result from a single evaluation.
+    """
+
+    score: float = Field(..., description="The normalized score (0.0 to 1.0).")
+    reason: str = Field("", description="The explanation for the score.")
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict, description="Any other metadata from the evaluator."
+    )
+
+
+class IterationHistory(BaseModel):
+    """
+    A detailed record of a single optimization iteration.
+    """
+
+    prompt: str
+    average_score: float
+    individual_results: List[EvaluationResult]
+    candidate_id: Optional[str] = None
+    candidate_config: Optional[Dict[str, Any]] = None
+    layers: List[str] = Field(default_factory=list)
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+
+class OptimizationResult(BaseModel):
+    """Output Model to hold the results of an optimization run."""
+
+    best_generator: Any
+    best_candidate: Any = None
+    history: List[IterationHistory]
+    final_score: float = 0.0
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+    # Early stopping metadata
+    early_stopped: bool = Field(
+        default=False,
+        description="Whether optimization was terminated early by a stopping criterion"
+    )
+    stop_reason: Optional[str] = Field(
+        default=None,
+        description="Explanation for early stopping (if applicable)"
+    )
+    total_iterations: int = Field(
+        default=0,
+        description="Total number of iterations completed"
+    )
+    total_evaluations: int = Field(
+        default=0,
+        description="Total number of dataset evaluations performed"
+    )

fi/opt/utils/__init__.py

@@ -0,0 +1,4 @@
+from .setup_logging import setup_logging
+from .early_stopping import EarlyStoppingConfig, EarlyStoppingChecker, EarlyStoppingException
+
+__all__ = ["setup_logging", "EarlyStoppingConfig", "EarlyStoppingChecker", "EarlyStoppingException"]