agentinputlab 0.1.0a1__py3-none-any.whl

agentinputlab/connectors/tools/schema.py

@@ -0,0 +1,102 @@
+"""Immutable, schema-only tool connector for planning interventions.
+
+The connector exposes an audited snapshot of an arena's tool declarations but
+cannot execute them.  Execution remains inside the official arena after the
+input intervention has been materialized.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+from agentinputlab.contracts.errors import ValidationError
+from agentinputlab.contracts.serialization import content_hash
+
+from .base import ToolCall, ToolConnector, ToolResult, ToolSpec
+
+
+@dataclass(frozen=True, slots=True)
+class SchemaOnlyToolConnector(ToolConnector):
+    """Expose frozen tool schemas while denying every execution request."""
+
+    specs: tuple[ToolSpec, ...]
+    connector_id: str = "schema-only-tools"
+
+    def __post_init__(self) -> None:
+        if not self.connector_id.strip():
+            raise ValidationError("schema-only connector requires a non-empty ID")
+        object.__setattr__(self, "specs", tuple(self.specs))
+        if not self.specs:
+            raise ValidationError("schema-only connector requires at least one tool")
+        if any(not isinstance(spec, ToolSpec) for spec in self.specs):
+            raise TypeError("schema-only connector specs must contain ToolSpec values")
+        names = tuple(spec.name for spec in self.specs)
+        if len(names) != len(set(names)):
+            raise ValidationError("schema-only connector tool names must be unique")
+
+    def list_tools(self) -> tuple[ToolSpec, ...]:
+        return self.specs
+
+    def call(self, request: ToolCall) -> ToolResult:
+        if not isinstance(request, ToolCall):
+            raise TypeError("schema-only connector requires ToolCall")
+        return ToolResult(
+            call_id=request.call_id,
+            error=(
+                "schema-only connector cannot execute tools; execution belongs to "
+                "the official downstream arena"
+            ),
+            metadata={"denied": True, "connector_id": self.connector_id},
+        )
+
+    def fingerprint(self) -> Mapping[str, Any]:
+        return {
+            "implementation": "agentinputlab.schema_only_tool_connector.v1",
+            "connector_id": self.connector_id,
+            "execution_policy": "deny_all.v1",
+            "tool_schema_hash": content_hash(self.specs),
+        }
+
+    @classmethod
+    def from_openai_definitions(
+        cls,
+        definitions: Sequence[Mapping[str, Any]],
+        *,
+        connector_id: str = "schema-only-tools",
+    ) -> SchemaOnlyToolConnector:
+        """Normalize OpenAI-style or bare function declarations."""
+
+        if isinstance(definitions, (str, bytes)) or not isinstance(definitions, Sequence):
+            raise ValidationError("tool definitions must be a sequence")
+        specs: list[ToolSpec] = []
+        for index, raw in enumerate(definitions):
+            if not isinstance(raw, Mapping):
+                raise ValidationError(f"tool definition {index} must be an object")
+            function = raw.get("function", raw)
+            if not isinstance(function, Mapping):
+                raise ValidationError(f"tool definition {index} function must be an object")
+            name = function.get("name")
+            description = function.get("description")
+            parameters = function.get("parameters", function.get("input_schema", {}))
+            if not isinstance(name, str) or not name.strip():
+                raise ValidationError(f"tool definition {index} requires a name")
+            if not isinstance(description, str) or not description.strip():
+                raise ValidationError(f"tool definition {index} requires a description")
+            if not isinstance(parameters, Mapping):
+                raise ValidationError(f"tool definition {index} parameters must be an object")
+            specs.append(
+                ToolSpec(
+                    name=name,
+                    description=description,
+                    input_schema=dict(parameters),
+                    # Schema-only snapshots do not assert mutability or networking.
+                    read_only=False,
+                    network=False,
+                )
+            )
+        return cls(tuple(specs), connector_id=connector_id)
+
+
+__all__ = ["SchemaOnlyToolConnector"]

agentinputlab/contracts/__init__.py

@@ -0,0 +1,91 @@
+"""Core state, patch, contract, provenance, materialization, and trace APIs."""
+
+from .availability import RunnerKind, RuntimeProbe
+from .dependency import DependencyBinding, DependencyRole, TreatmentDefinition
+from .enums import (
+    ExecutionKind,
+    ImplementationKind,
+    PatchOp,
+    ProfileKind,
+    ReleaseTier,
+    SemanticFamily,
+    SupportStatus,
+    TracePhase,
+)
+from .errors import (
+    AgentInputLabError,
+    AlgorithmOutputError,
+    AlgorithmUnavailableError,
+    ArmExecutionError,
+    ArtifactError,
+    BalanceError,
+    BudgetExceededError,
+    CapabilityError,
+    ExternalDependencyError,
+    IntentViolationError,
+    RegistryError,
+    StateConflictError,
+    ValidationError,
+)
+from .evidence import Evidence, EvidenceKind
+from .intent import IntentCheck, IntentContract, check_intent_contract
+from .lifecycle import (
+    CompositeIntervention,
+    CompositeResult,
+    Intervention,
+    InterventionContext,
+    OfflineRecipe,
+)
+from .patch import InputPatch, PatchOperation
+from .state import ContextBlock, ExecutionBudget, FrozenAgentState, Message
+from .trace import ExecutionTrace, TraceEvent
+from .worker import WorkerRequest, WorkerResponse
+
+__all__ = [
+    "AgentInputLabError",
+    "AlgorithmOutputError",
+    "AlgorithmUnavailableError",
+    "ArmExecutionError",
+    "ArtifactError",
+    "BalanceError",
+    "BudgetExceededError",
+    "CapabilityError",
+    "CompositeIntervention",
+    "CompositeResult",
+    "ContextBlock",
+    "DependencyBinding",
+    "DependencyRole",
+    "Evidence",
+    "EvidenceKind",
+    "ExecutionBudget",
+    "ExecutionKind",
+    "ExecutionTrace",
+    "ExternalDependencyError",
+    "FrozenAgentState",
+    "ImplementationKind",
+    "InputPatch",
+    "IntentCheck",
+    "IntentContract",
+    "IntentViolationError",
+    "Intervention",
+    "InterventionContext",
+    "Message",
+    "OfflineRecipe",
+    "PatchOp",
+    "PatchOperation",
+    "ProfileKind",
+    "RegistryError",
+    "ReleaseTier",
+    "RunnerKind",
+    "RuntimeProbe",
+    "SemanticFamily",
+    "StateConflictError",
+    "SupportStatus",
+    "TraceEvent",
+    "TracePhase",
+    "TreatmentDefinition",
+    "ValidationError",
+    "WorkerRequest",
+    "WorkerResponse",
+    "check_intent_contract",
+]

agentinputlab/contracts/availability.py

@@ -0,0 +1,24 @@
+"""Runtime availability is separate from source-audit status."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .enums import StrEnum
+
+
+class RunnerKind(StrEnum):
+    CORE = "core"
+    OPTIONAL_EXTRA = "optional_extra"
+    EXTERNAL_WORKER = "external_worker"
+    REPLAY_ONLY = "replay_only"
+
+
+@dataclass(frozen=True, slots=True)
+class RuntimeProbe:
+    available: bool
+    runner_kind: RunnerKind
+    missing_extras: tuple[str, ...] = ()
+    missing_artifacts: tuple[str, ...] = ()
+    upstream_revision: str | None = None
+    reason: str = ""

agentinputlab/contracts/dependency.py

@@ -0,0 +1,58 @@
+"""Dependency-slot identities used by controlled and factorial comparisons."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+from .enums import StrEnum
+from .errors import ValidationError
+from .serialization import content_hash, freeze_mapping
+
+
+class DependencyRole(StrEnum):
+    TREATMENT = "treatment"
+    NUISANCE = "nuisance"
+    CROSSED = "crossed"
+
+
+@dataclass(frozen=True, slots=True)
+class DependencyBinding:
+    slot: str
+    role: DependencyRole
+    implementation_id: str
+    revision: str
+    config: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.slot or not self.implementation_id:
+            raise ValidationError("dependency binding requires slot and implementation_id")
+        try:
+            object.__setattr__(self, "config", freeze_mapping(self.config))
+        except TypeError as exc:
+            raise ValidationError(
+                f"dependency binding config must be canonically serializable: {exc}"
+            ) from exc
+
+    @property
+    def resolved_hash(self) -> str:
+        return content_hash(self)
+
+
+@dataclass(frozen=True, slots=True)
+class TreatmentDefinition:
+    method_id: str
+    release_id: str
+    treatment_slots: tuple[str, ...]
+    nuisance_slots: tuple[str, ...] = ()
+    crossed_slots: tuple[str, ...] = ()
+
+    def __post_init__(self) -> None:
+        groups = [
+            set(self.treatment_slots),
+            set(self.nuisance_slots),
+            set(self.crossed_slots),
+        ]
+        if any(groups[i] & groups[j] for i in range(3) for j in range(i + 1, 3)):
+            raise ValidationError("treatment, nuisance, and crossed slots must be disjoint")

agentinputlab/contracts/enums.py

@@ -0,0 +1,82 @@
+"""Stable public enums used by manifests, traces, and result files."""
+
+from __future__ import annotations
+
+from enum import Enum
+
+
+class StrEnum(str, Enum):
+    """Python 3.10-compatible string enum."""
+
+    def __str__(self) -> str:
+        return self.value
+
+
+class SemanticFamily(StrEnum):
+    REFORMULATE = "reformulate"
+    DECOMPOSE = "decompose"
+    ACQUIRE = "acquire"
+    CLARIFY = "clarify"
+    CURATE = "curate"
+    COMPOSITE = "composite"
+    NO_OP = "no_op"
+
+
+class ReleaseTier(StrEnum):
+    CORE = "core"
+    OPTIONAL = "optional"
+
+
+class ExecutionKind(StrEnum):
+    RUNTIME = "runtime"
+    COMPONENT = "component"
+    COMPOSITE = "composite"
+    TRAINING_RECIPE = "training_recipe"
+    ARTIFACT_REPLAY = "artifact_replay"
+
+
+class ImplementationKind(StrEnum):
+    OFFICIAL_ADAPTER = "official_adapter"
+    COMMUNITY_IMPLEMENTATION = "community_implementation"
+    PAPER_REIMPLEMENTATION = "paper_reimplementation"
+    THIN_EXTERNAL_WRAPPER = "thin_external_wrapper"
+    ARTIFACT_REPLAY = "artifact_replay"
+
+
+class ProfileKind(StrEnum):
+    OFFICIAL_NATIVE = "official_native"
+    CONTROLLED_ADAPTER = "controlled_adapter"
+    FACTORIAL_ADAPTER = "factorial_adapter"
+
+
+class SupportStatus(StrEnum):
+    SOURCE_AUDITED = "source_audited"
+    PLANNED = "planned"
+    EXTERNAL_ONLY = "external_only"
+    NATIVE_ONLY = "native_only"
+    RETRAIN_REQUIRED = "retrain_required"
+    BLOCKED = "blocked"
+    EXPERIMENTAL = "experimental"
+    SUPPORTED = "supported"
+
+
+class PatchOp(StrEnum):
+    REPLACE_QUERY = "replace_query"
+    APPEND_QUERY = "append_query"
+    REPLACE_INSTRUCTION = "replace_instruction"
+    APPEND_INSTRUCTION = "append_instruction"
+    ADD_CONTEXT = "add_context"
+    REPLACE_CONTEXT = "replace_context"
+    ADD_MESSAGE = "add_message"
+    REPLACE_MESSAGES = "replace_messages"
+    SET_METADATA = "set_metadata"
+    REQUEST_CLARIFICATION = "request_clarification"
+    NO_OP = "no_op"
+
+
+class TracePhase(StrEnum):
+    PREPARE = "prepare"
+    PROPOSE = "propose"
+    OBSERVE = "observe"
+    FINALIZE = "finalize"
+    ERROR = "error"

agentinputlab/contracts/errors.py

@@ -0,0 +1,63 @@
+"""Public exception hierarchy.
+
+The library uses typed errors instead of silent fallbacks.  A benchmark run must
+be able to distinguish an unavailable upstream asset from an algorithm deciding
+to return ``NO_OP``.
+"""
+
+
+class AgentInputLabError(Exception):
+    """Base class for all expected library errors."""
+
+
+class ValidationError(AgentInputLabError, ValueError):
+    """A state, patch, manifest, or configuration violates its schema."""
+
+
+class ArmExecutionError(ValidationError):
+    """An expected execution failure invalidates one experimental arm only.
+
+    This marker remains a :class:`ValidationError` for API compatibility, but
+    paired-study orchestration can distinguish it from invalid experiment
+    configuration without inspecting exception messages or traceback text.
+    """
+
+
+class AlgorithmOutputError(ArmExecutionError):
+    """A model, embedder, retriever, or algorithm emitted an invalid result."""
+
+
+class StateConflictError(AgentInputLabError):
+    """A patch was applied to a state other than the one it was proposed for."""
+
+
+class CapabilityError(AgentInputLabError):
+    """A backend or connector lacks a capability required by a profile."""
+
+
+class AlgorithmUnavailableError(AgentInputLabError):
+    """A registered profile cannot run in the current environment."""
+
+
+class ExternalDependencyError(AlgorithmUnavailableError):
+    """An optional upstream checkout, package, model, or service is unavailable."""
+
+
+class ArtifactError(AgentInputLabError):
+    """An artifact is absent, corrupt, unsafe, or has the wrong lineage."""
+
+
+class RegistryError(AgentInputLabError):
+    """The algorithm registry is inconsistent or cannot be resolved."""
+
+
+class BalanceError(AgentInputLabError):
+    """A claimed controlled comparison differs on an undeclared dependency."""
+
+
+class BudgetExceededError(AgentInputLabError):
+    """A profile attempted to exceed a declared execution budget."""
+
+
+class IntentViolationError(AgentInputLabError):
+    """An intervention violated a runner-enforced observable intent contract."""

agentinputlab/contracts/evidence.py

@@ -0,0 +1,43 @@
+"""Evidence types that keep synthetic retrieval surrogates distinct from sources."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+from .enums import StrEnum
+from .errors import ValidationError
+from .serialization import freeze_mapping
+
+
+class EvidenceKind(StrEnum):
+    RETRIEVED_DOCUMENT = "retrieved_document"
+    SYNTHETIC_QUERY_SURROGATE = "synthetic_query_surrogate"
+    TOOL_OBSERVATION = "tool_observation"
+
+
+@dataclass(frozen=True, slots=True)
+class Evidence:
+    kind: EvidenceKind
+    text: str
+    evidence_id: str
+    source_revision: str = ""
+    score: float | None = None
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "metadata", freeze_mapping(self.metadata))
+        if not self.text:
+            raise ValidationError("evidence text must not be empty")
+        if self.kind is EvidenceKind.RETRIEVED_DOCUMENT and (
+            not self.evidence_id or not self.source_revision
+        ):
+            raise ValidationError("retrieved evidence requires document id and source revision")
+
+    @property
+    def citable(self) -> bool:
+        return self.kind in {
+            EvidenceKind.RETRIEVED_DOCUMENT,
+            EvidenceKind.TOOL_OBSERVATION,
+        }

agentinputlab/contracts/intent.py

@@ -0,0 +1,110 @@
+"""Intent constraints and evaluator-safe state projection."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from .enums import PatchOp
+from .patch import InputPatch
+from .state import FrozenAgentState
+
+
+@dataclass(frozen=True, slots=True)
+class IntentContract:
+    """Observable constraints an intervention must preserve.
+
+    The contract intentionally contains no evaluator answer or hidden simulator
+    state.  Gold data belongs to the benchmark evaluator process.
+    """
+
+    required_entities: tuple[str, ...] = ()
+    required_constraints: tuple[str, ...] = ()
+    forbidden_assumptions: tuple[str, ...] = ()
+    may_request_clarification: bool = True
+
+    def __post_init__(self) -> None:
+        for field_name in (
+            "required_entities",
+            "required_constraints",
+            "forbidden_assumptions",
+        ):
+            values = tuple(getattr(self, field_name))
+            if not all(isinstance(value, str) and value.strip() for value in values):
+                raise ValueError(f"{field_name} must contain only non-empty strings")
+            if len({value.casefold() for value in values}) != len(values):
+                raise ValueError(f"{field_name} must not contain duplicate clauses")
+            object.__setattr__(self, field_name, values)
+        if not isinstance(self.may_request_clarification, bool):
+            raise TypeError("may_request_clarification must be a bool")
+
+
+@dataclass(frozen=True, slots=True)
+class IntentCheck:
+    preserved: bool
+    missing_entities: tuple[str, ...] = ()
+    missing_constraints: tuple[str, ...] = ()
+    introduced_assumptions: tuple[str, ...] = ()
+
+
+def check_intent_contract(
+    contract: IntentContract,
+    state: FrozenAgentState,
+    patch: InputPatch,
+    output: FrozenAgentState,
+) -> IntentCheck:
+    """Apply conservative, observable checks without consulting benchmark gold.
+
+    This is a hard safety gate, not a semantic judge.  It only enforces explicit
+    strings supplied by the benchmark adapter and the clarification permission.
+    Learned or LLM-based intent metrics belong in secondary evaluation.
+    """
+
+    del state
+    visible = "\n".join(
+        (
+            output.query,
+            output.instruction,
+            *(block.text for block in output.context),
+            *(message.content for message in output.messages),
+        )
+    )
+    missing_entities = tuple(
+        entity for entity in contract.required_entities if not _contains_phrase(visible, entity)
+    )
+    missing_constraints = tuple(
+        constraint
+        for constraint in contract.required_constraints
+        if not _contains_phrase(visible, constraint)
+    )
+    introduced_assumptions = tuple(
+        assumption
+        for assumption in contract.forbidden_assumptions
+        if _contains_phrase(visible, assumption)
+    )
+    requested_clarification = any(
+        operation.op is PatchOp.REQUEST_CLARIFICATION for operation in patch.operations
+    )
+    clarification_violation = requested_clarification and not contract.may_request_clarification
+    if clarification_violation:
+        introduced_assumptions = (*introduced_assumptions, "<clarification-not-permitted>")
+    return IntentCheck(
+        preserved=not missing_entities and not missing_constraints and not introduced_assumptions,
+        missing_entities=missing_entities,
+        missing_constraints=missing_constraints,
+        introduced_assumptions=introduced_assumptions,
+    )
+
+
+def _contains_phrase(text: str, phrase: str) -> bool:
+    """Match a normalized contiguous token phrase, never an arbitrary substring."""
+
+    visible_tokens = tuple(re.findall(r"\w+", text.casefold(), flags=re.UNICODE))
+    phrase_tokens = tuple(re.findall(r"\w+", phrase.casefold(), flags=re.UNICODE))
+    if not phrase_tokens:
+        return False
+    width = len(phrase_tokens)
+    return any(
+        visible_tokens[index : index + width] == phrase_tokens
+        for index in range(len(visible_tokens) - width + 1)
+    )