agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/types.py

@@ -0,0 +1,156 @@
+import warnings
+from enum import Enum
+from typing import Any, Dict, List, Optional, Union
+
+from pydantic import BaseModel, Field
+
+warnings.filterwarnings("ignore", message='Field name "schema" in .* shadows an attribute in parent "BaseModel"')
+
+
+class OutputType(Enum):
+    SCORE = "score"
+    BOOLEAN = "boolean"
+    JSON = "json"
+    TEXT = "text"
+
+
+class RequiredKeys(Enum):
+    text = "text"
+    response = "response"
+    query = "query"
+    context = "context"
+    expected_response = "expected_response"
+    expected_text = "expected_text"
+    document = "document"
+    input = "input"
+    output = "output"
+    prompt = "prompt"
+    image_url = "image_url"
+    input_image_url = "input_image_url"
+    output_image_url = "output_image_url"
+    actual_json = "actual_json"
+    expected_json = "expected_json"
+    messages = "messages"
+
+
+class BaseMetricInput(BaseModel):
+    pass
+
+
+class BaseMetricOutput(BaseModel):
+    pass
+
+
+class TextMetricInput(BaseMetricInput):
+    response: str = Field(..., description="The text response to be evaluated.")
+    expected_response: Optional[Union[str, List[str]]] = Field(
+        None, description="The ground truth reference text(s)."
+    )
+
+
+class JsonMetricInput(BaseMetricInput):
+    response: Union[str, Dict, List] = Field(
+        ..., description="The JSON output from the model."
+    )
+    expected_response: Optional[Union[str, Dict, List]] = Field(
+        None, description="The ground truth JSON."
+    )
+    schema: Optional[Union[str, Dict]] = Field(
+        None, description="The JSON schema for validation."
+    )
+
+
+class ConfigParam(BaseModel):
+    type: str
+    default: Optional[Any] = None
+
+
+class ConfigPossibleValues(BaseModel):
+    min_length: Optional[int] = None
+    validations: Optional[List[str]] = None
+    eval_prompt: Optional[str] = None
+    substring: Optional[str] = None
+    model: Optional[str] = None
+    code: Optional[str] = None
+    keywords: Optional[List[str]] = None
+    keyword: Optional[str] = None
+    failure_threshold: Optional[float] = None
+    headers: Optional[Dict[str, str]] = None
+    case_sensitive: Optional[bool] = None
+    comparator: Optional[str] = None
+    payload: Optional[Dict[str, Any]] = None
+    url: Optional[str] = None
+    input: Optional[str] = None
+    max_length: Optional[int] = None
+    multi_choice: Optional[bool] = None
+    system_prompt: Optional[str] = None
+    pattern: Optional[str] = None
+    grading_criteria: Optional[str] = None
+    _schema: Optional[str] = None
+    rule_prompt: Optional[str] = None
+    choices: Optional[List[str]] = None
+
+
+class DatapointFieldAnnotation(BaseModel):
+    """
+    The annotations to be logged for the datapoint field.
+    """
+
+    field_name: str
+    text: str
+    annotation_type: str
+    annotation_note: str
+
+
+class EvalResultMetric(BaseModel):
+    """
+    Represents the LLM evaluation result metric.
+    """
+
+    id: Union[str, int, float]
+    value: Union[str, int, float, List[Any]]
+
+
+class EvalResult(BaseModel):
+    """
+    Represents the LLM evaluation result.
+    """
+
+    name: str
+    output: Optional[Any] = None
+    reason: Optional[str] = None
+    runtime: int = 0
+    output_type: Optional[str] = None
+    eval_id: Optional[str] = None
+    model: Optional[str] = None
+    error_localizer_enabled: Optional[bool] = None
+    error_localizer: Optional[Dict[str, Any]] = None
+
+
+class BatchRunResult(BaseModel):
+    """
+    Represents the result of a batch run of LLM evaluation.
+    """
+
+    eval_results: List[Optional[EvalResult]]
+
+
+class EvalTags(Enum):
+    CONVERSATION = "CONVERSATION"
+    HALLUCINATION = "HALLUCINATION"
+    RAG = "RAG"
+    FUTURE_EVALS = "FUTURE_EVALS"
+    LLMS = "LLMS"
+    CUSTOM = "CUSTOM"
+    FUNCTION = "FUNCTION"
+    IMAGE = "IMAGE"
+    SAFETY = "SAFETY"
+    TEXT = "TEXT"
+
+
+class Comparator(Enum):
+    COSINE = "CosineSimilarity"
+    LEVENSHTEIN = "NormalisedLevenshteinSimilarity"
+    JARO_WINKLER = "JaroWincklerSimilarity"
+    JACCARD = "JaccardSimilarity"
+    SORENSEN_DICE = "SorensenDiceSimilarity"

fi/opt/__init__.py

@@ -0,0 +1,221 @@
+from .components import (
+    AgentComponent,
+    AgentComponentSpec,
+    COMPONENT_SPECS,
+    ComponentDiagnosis,
+    FailureMode,
+    FAILURE_ROUTES,
+    HARNESS_LAYER_PATH_PREFIXES,
+    HARNESS_LAYERS,
+    diagnose_agent_report_evaluation,
+    diagnose_report,
+    diagnose_text,
+    relevant_search_paths,
+)
+from .evidence import (
+    DEFAULT_SIMULATION_EVIDENCE_WEIGHTS,
+    score_simulation_evidence,
+)
+from .targets import (
+    AgentCandidate,
+    CandidateEvaluation,
+    OptimizationLayer,
+    OptimizationTarget,
+    set_path,
+)
+from .simulation import SimulationEvaluator
+from .integrations.simulate import (
+    EvalSuiteOptimizationProblem,
+    ManifestOptimizationProblem,
+    SimulateEvalSuiteOptimizationProblem,
+    SimulateManifestOptimizationProblem,
+    SimulateSuiteOptimizationProblem,
+    SuiteOptimizationProblem,
+    deep_merge,
+    optimize_agent_learning_suite,
+    optimize_agent_learning_suite_file,
+    optimize_eval_suite,
+    optimize_eval_suite_file,
+    optimize_simulate_manifest,
+    optimize_simulate_manifest_file,
+    problem_from_agent_learning_suite,
+    problem_from_agent_learning_suite_file,
+    problem_from_eval_suite,
+    problem_from_eval_suite_file,
+    problem_from_simulate_manifest,
+    problem_from_simulate_manifest_file,
+)
+from .optimizers.agent_feedback import (
+    AgentMultiInteractionAblationReport,
+    AgentMultiInteractionBackendLineage,
+    AgentMultiInteractionBackendPlan,
+    AgentMultiInteractionBackendRun,
+    AgentMultiInteractionOptimizationResult,
+    AgentMultiInteractionOptimizer,
+)
+from .deployment import (
+    AgentDeploymentExport,
+    AgentPromotionCheck,
+    AgentRollbackDecision,
+    PromotionMetricCheck,
+    RollbackObservation,
+    check_agent_deployment_rollback,
+    check_agent_deployment_promotion,
+    export_agent_deployment,
+)
+from .mutations import (
+    AgentMutationBundle,
+    AgentMutationLibrary,
+    DEFAULT_AGENT_MUTATION_LIBRARY,
+    FrameworkMutationRule,
+)
+from .observability import (
+    AgentDatasetSinkResult,
+    AgentObservabilityRecord,
+    AgentObservabilityWindow,
+    AgentRegistryReplayPackLineageEntry,
+    AgentRegistryReplayPackLineageReport,
+    AgentRegistryReplayPackLineageTransition,
+    AgentRegistryReplayPackManifest,
+    AgentRegistryReplayPackPromotionCheck,
+    AgentRegistryReplayPackTriageReport,
+    AgentRegressionCase,
+    AgentRegressionDatasetCoverageReport,
+    AgentRegressionDataset,
+    build_agent_regression_dataset,
+    build_agent_regression_dataset_coverage_report,
+    build_futureagi_registry_replay_pack_manifest,
+    check_futureagi_registry_replay_pack_promotion,
+    compare_futureagi_registry_replay_pack_lineage,
+    load_agent_report_replay_cases,
+    load_futureagi_experiment_history,
+    load_futureagi_regression_dataset,
+    load_agent_observability_feedback,
+    publish_futureagi_regression_dataset,
+    triage_futureagi_registry_replay_pack_regression,
+)
+from .optimizer_trace import build_optimizer_society_trace, optimizer_trajectory_profile
+from .research import (
+    ResearchCorpusSummary,
+    ResearchPaper,
+    build_agent_research_corpus,
+    build_deep_read_queue,
+    infer_red_team_signals,
+    infer_research_themes,
+    load_research_papers,
+    map_research_to_red_team_campaign,
+    normalize_research_paper,
+    research_note_for,
+    research_summary_markdown,
+)
+from .optimizers.futureagi_replay import (
+    FutureAGIExperimentHistoryOptimizer,
+    FutureAGIReplayOptimizerSchedule,
+    FutureAGIRegressionReplayOptimizer,
+    schedule_futureagi_registry_replay_optimization,
+)
+from .types import EvaluationResult, IterationHistory, LLMMessage, OptimizationResult
+
+__all__ = [
+    "AgentComponent",
+    "AgentComponentSpec",
+    "AgentCandidate",
+    "AgentDatasetSinkResult",
+    "AgentDeploymentExport",
+    "AgentMutationBundle",
+    "AgentMutationLibrary",
+    "AgentMultiInteractionAblationReport",
+    "AgentMultiInteractionBackendLineage",
+    "AgentMultiInteractionBackendPlan",
+    "AgentMultiInteractionBackendRun",
+    "AgentMultiInteractionOptimizationResult",
+    "AgentMultiInteractionOptimizer",
+    "AgentObservabilityRecord",
+    "AgentObservabilityWindow",
+    "AgentRegistryReplayPackLineageEntry",
+    "AgentRegistryReplayPackLineageReport",
+    "AgentRegistryReplayPackLineageTransition",
+    "AgentRegistryReplayPackManifest",
+    "AgentRegistryReplayPackPromotionCheck",
+    "AgentRegistryReplayPackTriageReport",
+    "AgentRegressionCase",
+    "AgentRegressionDataset",
+    "AgentRegressionDatasetCoverageReport",
+    "AgentPromotionCheck",
+    "AgentRollbackDecision",
+    "COMPONENT_SPECS",
+    "CandidateEvaluation",
+    "ComponentDiagnosis",
+    "DEFAULT_AGENT_MUTATION_LIBRARY",
+    "DEFAULT_SIMULATION_EVIDENCE_WEIGHTS",
+    "FailureMode",
+    "FAILURE_ROUTES",
+    "HARNESS_LAYER_PATH_PREFIXES",
+    "HARNESS_LAYERS",
+    "EvalSuiteOptimizationProblem",
+    "FrameworkMutationRule",
+    "FutureAGIExperimentHistoryOptimizer",
+    "FutureAGIReplayOptimizerSchedule",
+    "FutureAGIRegressionReplayOptimizer",
+    "PromotionMetricCheck",
+    "ResearchCorpusSummary",
+    "ResearchPaper",
+    "RollbackObservation",
+    "check_agent_deployment_rollback",
+    "check_agent_deployment_promotion",
+    "check_futureagi_registry_replay_pack_promotion",
+    "compare_futureagi_registry_replay_pack_lineage",
+    "build_agent_regression_dataset",
+    "build_agent_regression_dataset_coverage_report",
+    "build_agent_research_corpus",
+    "build_deep_read_queue",
+    "build_futureagi_registry_replay_pack_manifest",
+    "build_optimizer_society_trace",
+    "optimizer_trajectory_profile",
+    "load_agent_report_replay_cases",
+    "ManifestOptimizationProblem",
+    "SuiteOptimizationProblem",
+    "diagnose_agent_report_evaluation",
+    "export_agent_deployment",
+    "load_agent_observability_feedback",
+    "load_futureagi_experiment_history",
+    "load_research_papers",
+    "load_futureagi_regression_dataset",
+    "map_research_to_red_team_campaign",
+    "normalize_research_paper",
+    "publish_futureagi_regression_dataset",
+    "research_note_for",
+    "research_summary_markdown",
+    "score_simulation_evidence",
+    "triage_futureagi_registry_replay_pack_regression",
+    "OptimizationLayer",
+    "OptimizationTarget",
+    "optimize_agent_learning_suite",
+    "optimize_agent_learning_suite_file",
+    "optimize_eval_suite",
+    "optimize_eval_suite_file",
+    "optimize_simulate_manifest",
+    "optimize_simulate_manifest_file",
+    "problem_from_agent_learning_suite",
+    "problem_from_agent_learning_suite_file",
+    "problem_from_eval_suite",
+    "problem_from_eval_suite_file",
+    "problem_from_simulate_manifest",
+    "problem_from_simulate_manifest_file",
+    "diagnose_report",
+    "diagnose_text",
+    "infer_red_team_signals",
+    "infer_research_themes",
+    "relevant_search_paths",
+    "set_path",
+    "SimulationEvaluator",
+    "SimulateEvalSuiteOptimizationProblem",
+    "SimulateManifestOptimizationProblem",
+    "SimulateSuiteOptimizationProblem",
+    "schedule_futureagi_registry_replay_optimization",
+    "deep_merge",
+    "EvaluationResult",
+    "IterationHistory",
+    "LLMMessage",
+    "OptimizationResult",
+]

fi/opt/_objective_scoring.py

@@ -0,0 +1,85 @@
+"""Canonical objective-anchored scoring — the ONE source of truth for the
+benchmark score (B2), the reward-hack detector (B6), the Task objective (B1), AND
+the optimizer's candidate scoring (bug #2 fix).
+
+Lives in ``fi`` (the vendored engine) so the engine's optimizer integration can
+use it WITHOUT importing the public ``fi.alk`` package — respecting the
+vendored_engine_boundary. ``fi.alk.tasks`` re-exports these so its public
+call sites are unchanged. Pure functions, no kit/engine cross-deps.
+
+Why this exists: the engine's all-metrics mean (``summary.evaluation_score``) pins
+~30/38 metrics at 1.0 and has near-zero dynamic range — a terrible agent and a
+good one both score ~0.95. Scoring only the task's DECLARED objective terms gives
+real dynamic range (a failing ground-truth anchor actually drops the score).
+"""
+
+from __future__ import annotations
+
+from typing import Any, Mapping
+
+# eval-ref -> engine metric_averages key. Exact match wins; this only fills gaps
+# where the objective's declared ref differs from the engine's metric name.
+METRIC_ALIASES = {
+    "task_success": "task_completion",
+    "artifact_grounding": "source_grounding",
+    "tool_argument_correctness": "tool_argument_schema",
+}
+
+
+def resolve_metric(metrics: Mapping[str, Any], eval_ref: str) -> float | None:
+    """Resolve an objective eval-ref to its value in the engine metric averages,
+    via exact match then the known alias map. Returns None if unresolved."""
+
+    if eval_ref in metrics:
+        try:
+            return float(metrics[eval_ref])
+        except (TypeError, ValueError):
+            return None
+    alias = METRIC_ALIASES.get(eval_ref)
+    if alias and alias in metrics:
+        try:
+            return float(metrics[alias])
+        except (TypeError, ValueError):
+            return None
+    return None
+
+
+def objective_score(metrics: Mapping[str, Any], objective: Mapping[str, Any]) -> dict[str, Any]:
+    """Weighted mean over the objective's DECLARED terms, each resolved to a real
+    engine metric. Returns {score, terms_resolved, terms_total, per_term}; score
+    is None when no declared term resolves to a metric (caller falls back)."""
+
+    terms = [t for t in (objective.get("evals") or []) if isinstance(t, Mapping) and t.get("eval")]
+    per_term: dict[str, dict[str, Any]] = {}
+    num = 0.0
+    den = 0.0
+    for term in terms:
+        ref = str(term["eval"])
+        weight = float(term.get("weight", 1.0))
+        val = resolve_metric(metrics, ref)
+        per_term[ref] = {"weight": weight, "value": val, "anchor": bool(term.get("anchor"))}
+        if val is not None and weight > 0:
+            num += weight * val
+            den += weight
+    resolved = sum(1 for v in per_term.values() if v["value"] is not None)
+    score = (num / den) if den > 0 else None
+    return {
+        "score": round(score, 6) if score is not None else None,
+        "terms_resolved": resolved,
+        "terms_total": len(terms),
+        "per_term": per_term,
+    }
+
+
+def has_declared_anchor_objective(objective: Mapping[str, Any] | None) -> bool:
+    """True iff ``objective`` declares >=1 term explicitly marked ``anchor: true``.
+    The gate for objective-anchored optimizer scoring: only manifests that DECLARE
+    a real anchored objective opt in; structural/hook/legacy manifests (no declared
+    objective) keep the engine's existing all-metrics score — no regression."""
+
+    if not isinstance(objective, Mapping):
+        return False
+    return any(
+        isinstance(t, Mapping) and t.get("eval") and t.get("anchor") is True
+        for t in (objective.get("evals") or [])
+    )

fi/opt/base/__init__.py

@@ -0,0 +1,11 @@
+from .base_generator import BaseGenerator
+from .base_optimizer import BaseOptimizer
+from .base_mapper import BaseDataMapper
+from .evaluator import Evaluator
+
+__all__ = [
+    "BaseGenerator",
+    "BaseOptimizer",
+    "BaseDataMapper",
+    "Evaluator",
+]

fi/opt/base/base_generator.py

@@ -0,0 +1,33 @@
+from abc import ABC, abstractmethod
+from typing import Dict
+
+
+class BaseGenerator(ABC):
+    """
+    Abstract base class for all Generators. A Generator is a callable entity
+    (like an LLM) that executes a prompt and returns a result. It also manages
+    an internal prompt template that can be modified by an optimizer.
+    """
+
+    @abstractmethod
+    def generate(self, prompt_vars: Dict[str, str], **kwargs) -> str:
+        """
+        Executes the generator with a given set of input variables.
+
+        Args:
+            prompt_vars: A dictionary of variables to fill in the prompt template.
+
+        Returns:
+            The string output from the language model.
+        """
+        pass
+
+    @abstractmethod
+    def get_prompt_template(self) -> str:
+        """Returns the current internal prompt template."""
+        pass
+
+    @abstractmethod
+    def set_prompt_template(self, template: str):
+        """Updates the internal prompt template."""
+        pass

fi/opt/base/base_mapper.py

@@ -0,0 +1,26 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Any
+
+
+class BaseDataMapper(ABC):
+    """
+    Abstract base class for Data Mappers. A Data Mapper is responsible for
+    transforming data into the format expected by an evaluator.
+    """
+
+    @abstractmethod
+    def map(
+        self, generated_output: str, ground_truth_example: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """
+        Maps the generated output and a ground truth example to the format
+        expected by the evaluator's `inputs`.
+
+        Args:
+            generated_output: The output from the Generator.
+            ground_truth_example: A single example from the dataset.
+
+        Returns:
+            A dictionary formatted for the evaluator's `inputs` argument.
+        """
+        pass

fi/opt/base/base_optimizer.py

@@ -0,0 +1,45 @@
+from abc import ABC, abstractmethod
+from typing import List, Dict, Any, Callable
+from ..types import OptimizationResult
+
+
+class BaseOptimizer(ABC):
+    """
+    Abstract base class for all optimization algorithms.
+    Each concrete optimizer will implement its own `optimize` method,
+    containing the full logic for its optimization loop.
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    @abstractmethod
+    def optimize(
+        self,
+        evaluator: Any,
+        data_mapper: Any,  # We'll refine this later
+        dataset: List[Dict[str, Any]],
+        metric: Callable,
+        **kwargs: Any,
+    ) -> OptimizationResult:
+        """
+        Runs the full optimization process.
+
+        Args:
+            evaluator: The user-provided evaluator instance.
+            data_mapper: The user-provided data mapper.
+            dataset: The dataset to use for evaluation.
+            metric: The metric function to use for evaluation.
+            **kwargs: Additional, optimizer-specific arguments. Common optional
+                arguments include:
+                - early_stopping (EarlyStoppingConfig): Configuration for early
+                    stopping criteria. Supports patience-based stopping, score
+                    thresholds, minimum improvement deltas, and cost budgets.
+                    When configured, optimization may terminate before reaching
+                    the maximum number of iterations.
+
+        Returns:
+            An OptimizationResult object with the best generator, iteration
+            history, final score, and early stopping metadata (if applicable).
+        """
+        pass