agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/voice_loop.py

@@ -0,0 +1,174 @@
+"""Phase 9A unit 4 — the voice improvement loop (the 13D Practice Loop on
+``world.kind = voice_telephony``).
+
+ARCH §2.3 / decisions 9A-D5 (the 13D loop on voice_telephony; NO new optimizer),
+9A-A4 (multi-objective-mandatory voice loss; single-timing-term rejection),
+9A-A14 (``V1_VOICE_FAILURE_SUBLAYERS`` voice sub-attribution).
+
+This module invents NO optimizer, NO artifact kind, NO loss machinery. It is a
+thin composition layer over verbatim engines:
+
+  * the multi-objective voice loss compiles via ``loss.compile_objective`` (the
+    Goodhart guard at ``loss.py:106-116`` is reused VERBATIM — "There is no
+    override."); the 9A-A4 composition rule (≥2 terms, ≥1 non-timing quality
+    term) is a thin validator on top, raising ``voice_loss_guard_missing``;
+  * the whole voice-agent config is the search space, assembled by
+    ``optimize.build_practice_loop_manifest`` (the same ``base_agent`` +
+    ``search_space`` whole-agent contract) with ``world.kind=voice_telephony``;
+  * the voice sub-attribution is an additive tag stamped alongside the base
+    ``FAILURE_LAYERS`` tag (the existing ``practice/_diagnose.py`` machinery is
+    consumed, not rewritten).
+
+The canon constants below are this module's home; ``trinity.py`` carries literal
+mirrors that the milestone test cross-pins (the GUNA_AXES cross-pin pattern —
+trinity never imports this module so the gate runs even if this is broken).
+"""
+
+from __future__ import annotations
+
+from typing import Any, Mapping, Optional, Sequence
+
+# --- canon (ARCH §4 voice-loss term refs + §2.3 sub-attribution) ------------
+# the gate-pinned eval refs (ARCH §4 names are binding in code/gate; the UI-UX
+# display names — tool_arg_correctness/selectivity_jir/asr_wer_delta/
+# perturbation_robustness_delta — are presentation aliases only).
+V1_VOICE_LOSS_TERM_REFS = (
+    "task_success",
+    "tool_argument_correctness",
+    "barge_in_latency",
+    "ttfb",
+    "wer_delta",
+    "recovery",
+    "selectivity",
+    "codec_survival",
+    "perturbation_robustness",
+)
+# the non-timing quality anchors — a voice loss MUST carry >= 1 of these (9A-A4).
+V1_VOICE_LOSS_NON_TIMING_QUALITY_TERMS = ("task_success", "tool_argument_correctness")
+# the timing-only terms (a single-timing-term objective is reward-hackable —
+# ASPIRin R§2.3 — and is structurally rejected).
+V1_VOICE_LOSS_TIMING_TERMS = ("barge_in_latency", "ttfb", "recovery")
+# the four-token voice sub-attribution closed set (9A-A14), stamped alongside
+# the base FAILURE_LAYERS tag.
+V1_VOICE_FAILURE_SUBLAYERS = ("acoustic_codec", "asr_mishear", "llm", "tts_endpointing")
+
+
+class VoiceLossCompositionError(ValueError):
+    """Raised when a voice objective violates the 9A-A4 composition rule
+    (the ``voice_loss_guard_missing`` finding — a voice specialization of
+    ``objective_guards_missing``)."""
+
+
+def _term_refs(objective: Mapping[str, Any]) -> list[str]:
+    return [
+        str(term.get("eval"))
+        for term in (objective.get("evals") or [])
+        if isinstance(term, Mapping) and term.get("eval")
+    ]
+
+
+def compile_voice_objective(payload: Mapping[str, Any]) -> dict:
+    """Compile a multi-objective voice loss (9A-A4). Enforces, ON TOP of the
+    verbatim ``loss.compile_objective`` Goodhart guard:
+
+      (a) >= 2 terms,
+      (b) >= 1 non-timing quality term
+          (``task_success`` / ``tool_argument_correctness``),
+      (c) a populated guard block (delegated to ``compile_objective``).
+
+    A single-timing-term voice objective is structurally rejected
+    (``VoiceLossCompositionError`` / the ``voice_loss_guard_missing`` finding).
+    The underlying guard is the UNEDITED ``loss.py`` enforcement."""
+
+    from . import loss as _loss  # downward facade import (legal)
+
+    refs = _term_refs(payload)
+    if len(refs) < 2:
+        raise VoiceLossCompositionError(
+            "voice_loss_guard_missing: a voice objective is reward-hackable as a "
+            "single term (ASPIRin); it MUST be multi-objective (>= 2 terms). "
+            f"got {refs}"
+        )
+    if not any(ref in V1_VOICE_LOSS_NON_TIMING_QUALITY_TERMS for ref in refs):
+        raise VoiceLossCompositionError(
+            "voice_loss_guard_missing: a voice objective MUST carry >= 1 "
+            "non-timing quality term "
+            f"({V1_VOICE_LOSS_NON_TIMING_QUALITY_TERMS}); a timing-only loss is "
+            f"reward-hackable and is structurally rejected. got {refs}"
+        )
+    for ref in refs:
+        if ref not in V1_VOICE_LOSS_TERM_REFS:
+            raise VoiceLossCompositionError(
+                f"voice_loss_guard_missing: unknown voice loss term {ref!r}; "
+                f"expected members of {V1_VOICE_LOSS_TERM_REFS}"
+            )
+    # the verbatim Goodhart guard (loss.py:106-116) — "There is no override."
+    return _loss.compile_objective(payload)
+
+
+def attribute_voice_sublayer(
+    *,
+    failure_layer: str,
+    deficit: Mapping[str, Any] | None = None,
+    signal: str | None = None,
+) -> str:
+    """Map a weak voice cell to a ``V1_VOICE_FAILURE_SUBLAYERS`` token, stamped
+    ALONGSIDE the base ``FAILURE_LAYERS`` tag (a weak cell carries both). The
+    base attribution rides the existing ``practice/_diagnose.py`` machinery; this
+    is the thin sublayer helper.
+
+    Mapping (ARCH §2.3): a weak ``selectivity`` / endpointing signal →
+    ``tts_endpointing`` (not ``llm``); a mis-heard value under clean audio →
+    ``asr_mishear``; a claim that died through the codec → ``acoustic_codec``;
+    otherwise the reasoning/policy layer → ``llm``."""
+
+    sig = str(signal or (deficit or {}).get("signal") or "").lower()
+    if any(k in sig for k in ("selectivity", "endpoint", "barge", "vad", "interrupt", "recovery", "ttfb")):
+        return "tts_endpointing"
+    if any(k in sig for k in ("codec", "survival", "packet", "band_energy", "perturbation")):
+        return "acoustic_codec"
+    if any(k in sig for k in ("wer", "mishear", "asr", "tool_argument", "transcription")):
+        return "asr_mishear"
+    # default to the reasoning/policy layer unless infra clearly implicated.
+    if failure_layer in ("lane_infra", "framework_runtime", "provider"):
+        return "acoustic_codec"
+    return "llm"
+
+
+def build_voice_practice_loop_manifest(
+    *,
+    name: str,
+    base_agent: Mapping[str, Any],
+    search_space: Mapping[str, Sequence[Any]],
+    objective: Mapping[str, Any],
+    eval_budget: int,
+    seed: int,
+    scenario_inline: Optional[Mapping[str, Any]] = None,
+    max_rounds: int = 8,
+) -> dict[str, Any]:
+    """Assemble the voice improvement-loop manifest: the 13D Practice Loop on
+    ``world.kind=voice_telephony`` with the multi-objective voice loss + the
+    whole voice-agent search space (9A-D5). Delegates to
+    ``optimize.build_practice_loop_manifest`` so its validators hold VERBATIM.
+    The objective is compiled by ``compile_voice_objective`` (the 9A-A4 rule)
+    before it rides the simulation."""
+
+    from . import optimize as _optimize  # downward facade import (legal)
+
+    compiled = compile_voice_objective(objective)
+    inline = dict(scenario_inline or {})
+    inline.setdefault("version", "agent-learning.simulation.v1")
+    inline["objective"] = compiled
+    world = dict(inline.get("world") or {})
+    world["kind"] = "voice_telephony"
+    inline["world"] = world
+
+    return _optimize.build_practice_loop_manifest(
+        name=name,
+        simulation={"version": inline["version"], "inline": inline},
+        base_agent=base_agent,
+        search_space=search_space,
+        eval_budget=eval_budget,
+        seed=seed,
+        max_rounds=max_rounds,
+    )

fi/api/__init__.py

@@ -0,0 +1 @@
+"""Low-level HTTP primitives used by the cloud eval clients."""

fi/api/auth.py

@@ -0,0 +1,137 @@
+"""Auth + HTTP client primitives.
+
+``APIKeyAuth`` is the common base for every cloud client in the SDK. It
+reads credentials from env vars (``FI_API_KEY`` / ``FI_SECRET_KEY``)
+unless explicit keys are passed.
+"""
+import os
+import time
+from abc import ABC, abstractmethod
+from typing import Dict, Generic, Optional, TypeVar, Union
+
+from requests import Response
+from requests_futures.sessions import FuturesSession
+
+from fi.api.types import RequestConfig
+from fi.utils.constants import (
+    API_KEY_ENVVAR_NAME,
+    DEFAULT_MAX_QUEUE,
+    DEFAULT_MAX_WORKERS,
+    DEFAULT_TIMEOUT,
+    SECRET_KEY_ENVVAR_NAME,
+    get_base_url,
+)
+from fi.utils.errors import DatasetNotFoundError, MissingAuthError
+from fi.utils.executor import BoundedExecutor
+
+T = TypeVar("T")
+U = TypeVar("U")
+
+
+class ResponseHandler(Generic[T, U], ABC):
+    """Parses + validates a requests.Response into a typed result."""
+
+    @classmethod
+    def parse(cls, response: Response) -> Union[T, U]:
+        if not response.ok or response.status_code != 200:
+            cls._handle_error(response)
+        return cls._parse_success(response)
+
+    @classmethod
+    @abstractmethod
+    def _parse_success(cls, response: Response) -> Union[T, U]:
+        ...
+
+    @classmethod
+    @abstractmethod
+    def _handle_error(cls, response: Response) -> None:
+        ...
+
+
+class HttpClient:
+    """Thin async-capable HTTP client with retries."""
+
+    def __init__(
+        self,
+        base_url: Optional[str] = None,
+        session: Optional[FuturesSession] = None,
+        default_headers: Optional[Dict[str, str]] = None,
+        **kwargs,
+    ):
+        self._base_url = (base_url or get_base_url()).rstrip("/")
+        self._session = session or FuturesSession(
+            executor=BoundedExecutor(
+                bound=kwargs.get("max_queue", DEFAULT_MAX_QUEUE),
+                max_workers=kwargs.get("max_workers", DEFAULT_MAX_WORKERS),
+            ),
+        )
+        self._default_headers = default_headers or {}
+        self._default_timeout = kwargs.get("timeout", DEFAULT_TIMEOUT)
+
+    def request(
+        self,
+        config: RequestConfig,
+        response_handler: Optional[ResponseHandler[T, U]] = None,
+    ) -> Union[Response, T]:
+        url = config.url
+        headers = {**self._default_headers, **(config.headers or {})}
+        params = config.params or {}
+        json_body = config.json or {}
+        timeout = config.timeout or self._default_timeout
+        files = config.files or {}
+        data = config.data or {}
+
+        for attempt in range(config.retry_attempts):
+            try:
+                response = self._session.request(
+                    method=config.method.value,
+                    url=url,
+                    headers=headers,
+                    params=params,
+                    json=json_body,
+                    data=data,
+                    files=files,
+                    timeout=timeout,
+                ).result()
+
+                if response_handler:
+                    return response_handler.parse(response=response)
+                return response
+
+            except Exception as exc:
+                if isinstance(exc, DatasetNotFoundError):
+                    raise
+                if attempt == config.retry_attempts - 1:
+                    raise
+                time.sleep(config.retry_delay)
+
+    def close(self) -> None:
+        self._session.close()
+
+
+class APIKeyAuth(HttpClient):
+    """HTTP client that injects FutureAGI API + secret key headers."""
+
+    _fi_api_key: Optional[str] = None
+    _fi_secret_key: Optional[str] = None
+
+    def __init__(
+        self,
+        fi_api_key: Optional[str] = None,
+        fi_secret_key: Optional[str] = None,
+        fi_base_url: Optional[str] = None,
+        **kwargs,
+    ) -> None:
+        self.__class__._fi_api_key = fi_api_key or os.environ.get(API_KEY_ENVVAR_NAME)
+        self.__class__._fi_secret_key = fi_secret_key or os.environ.get(SECRET_KEY_ENVVAR_NAME)
+        if self._fi_api_key is None or self._fi_secret_key is None:
+            raise MissingAuthError(self._fi_api_key, self._fi_secret_key)
+
+        super().__init__(
+            base_url=fi_base_url,
+            default_headers={
+                "X-Api-Key": self._fi_api_key,
+                "X-Secret-Key": self._fi_secret_key,
+            },
+            **kwargs,
+        )

fi/api/types.py

@@ -0,0 +1,29 @@
+from enum import Enum
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+
+class HttpMethod(Enum):
+    GET = "GET"
+    POST = "POST"
+    PUT = "PUT"
+    DELETE = "DELETE"
+    PATCH = "PATCH"
+
+
+class RequestConfig(BaseModel):
+    """Configuration for an HTTP request."""
+
+    method: HttpMethod
+    url: str
+    headers: Optional[Dict[str, str]] = {}
+    params: Optional[Dict[str, Any]] = {}
+    files: Optional[Dict[str, Any]] = {}
+    data: Optional[Dict[str, Any]] = {}
+    json: Optional[Dict[str, Any]] = {}
+    timeout: Optional[int] = None
+    retry_attempts: int = 3
+    retry_delay: float = 1.0
+
+    model_config = ConfigDict(protected_namespaces=())

fi/cli/__init__.py

@@ -0,0 +1,9 @@
+"""
+AI Evaluation CLI
+
+Command-line interface for running LLM evaluations with 60+ templates.
+"""
+
+from fi.cli.main import app, main
+
+__all__ = ["app", "main"]

fi/cli/assertions/__init__.py

@@ -0,0 +1,25 @@
+"""Assertions module for CLI evaluation result validation."""
+
+from .conditions import Condition, Operator, MetricType
+from .parser import ConditionParser
+from .evaluator import (
+    AssertionEvaluator,
+    AssertionResult,
+    AssertionOutcome,
+    AssertionReport,
+)
+from .reporter import AssertionReporter
+from .exit_codes import ExitCode
+
+__all__ = [
+    "Condition",
+    "Operator",
+    "MetricType",
+    "ConditionParser",
+    "AssertionEvaluator",
+    "AssertionResult",
+    "AssertionOutcome",
+    "AssertionReport",
+    "AssertionReporter",
+    "ExitCode",
+]

fi/cli/assertions/conditions.py

@@ -0,0 +1,76 @@
+"""Condition types and operators for assertions."""
+
+from enum import Enum
+from typing import Union, Optional
+from dataclasses import dataclass
+
+
+class Operator(Enum):
+    """Comparison operators for assertion conditions."""
+    GTE = ">="      # Greater than or equal
+    LTE = "<="      # Less than or equal
+    GT = ">"        # Greater than
+    LT = "<"        # Less than
+    EQ = "=="       # Equal
+    NEQ = "!="      # Not equal
+    BETWEEN = "between"  # Between two values
+
+
+class MetricType(Enum):
+    """Types of metrics that can be evaluated in assertions."""
+    PASS_RATE = "pass_rate"           # Percentage of passing evaluations
+    AVG_SCORE = "avg_score"           # Average score (for numeric outputs)
+    MIN_SCORE = "min_score"           # Minimum score
+    MAX_SCORE = "max_score"           # Maximum score
+    FAILED_COUNT = "failed_count"     # Number of failures
+    PASSED_COUNT = "passed_count"     # Number of passes
+    TOTAL_COUNT = "total_count"       # Total evaluations
+    P50_SCORE = "p50_score"           # 50th percentile
+    P90_SCORE = "p90_score"           # 90th percentile
+    P95_SCORE = "p95_score"           # 95th percentile
+    RUNTIME_AVG = "runtime_avg"       # Average runtime in ms
+    RUNTIME_P95 = "runtime_p95"       # 95th percentile runtime
+    # Global metrics (across all templates)
+    TOTAL_PASS_RATE = "total_pass_rate"
+
+
+@dataclass
+class Condition:
+    """A single assertion condition."""
+    metric: MetricType
+    operator: Operator
+    value: Union[float, int]
+    value2: Optional[Union[float, int]] = None  # For BETWEEN operator
+
+    def evaluate(self, actual_value: float) -> bool:
+        """Evaluate if the condition passes.
+
+        Args:
+            actual_value: The actual metric value to compare against.
+
+        Returns:
+            True if the condition passes, False otherwise.
+        """
+        if self.operator == Operator.GTE:
+            return actual_value >= self.value
+        elif self.operator == Operator.LTE:
+            return actual_value <= self.value
+        elif self.operator == Operator.GT:
+            return actual_value > self.value
+        elif self.operator == Operator.LT:
+            return actual_value < self.value
+        elif self.operator == Operator.EQ:
+            return actual_value == self.value
+        elif self.operator == Operator.NEQ:
+            return actual_value != self.value
+        elif self.operator == Operator.BETWEEN:
+            if self.value2 is None:
+                return False
+            return self.value <= actual_value <= self.value2
+        return False
+
+    def __str__(self) -> str:
+        """String representation of the condition."""
+        if self.operator == Operator.BETWEEN:
+            return f"{self.value} <= {self.metric.value} <= {self.value2}"
+        return f"{self.metric.value} {self.operator.value} {self.value}"