atlas-agent-evaluation 1.0.1__tar.gz

atlas_agent_evaluation-1.0.1/.gitignore

@@ -0,0 +1,35 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.coverage
+coverage.xml
+reports/release/*coverage.xml
+reports/release/junit.xml
+reports/release/bandit.json
+reports/release/dependency-audit.json
+reports/release/requirements-audit.txt
+.mypy_cache/
+.pytest_cache/
+.tmp-pytest-*/
+.tmp-pytest/
+.ruff_cache/
+.uv-cache/
+.venv/
+build/
+dist/
+/release/
+bin/
+obj/
+htmlcov/
+.env
+.env.*
+!.env.example
+*.key
+*.log
+*.pem
+*.tmp
+*.swp
+.DS_Store
+Thumbs.db
+.idea/
+.vscode/

atlas_agent_evaluation-1.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jorge Medina
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

atlas_agent_evaluation-1.0.1/PKG-INFO

@@ -0,0 +1,29 @@
+Metadata-Version: 2.5
+Name: atlas-agent-evaluation
+Version: 1.0.1
+Summary: Avaliação provider-neutral para o Atlas Agent Framework
+Project-URL: Repository, https://github.com/Medicode/ai-agent-framework
+Project-URL: Documentation, https://github.com/Medicode/ai-agent-framework/tree/main/docs/reference
+Project-URL: Issues, https://github.com/Medicode/ai-agent-framework/issues
+Author: Jorge Medina
+Maintainer: Jorge Medina
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: atlas-agent-core~=1.0.1
+Requires-Dist: pydantic<3,>=2.13
+Description-Content-Type: text/markdown
+
+# atlas-agent-evaluation
+
+Framework de avaliação provider-neutral.
+
+Este pacote integra o ecossistema modular Atlas Agent Framework. Consulte a
+[documentação oficial](https://github.com/Medicode/ai-agent-framework/tree/main/docs)
+para instalação, compatibilidade e APIs públicas.

atlas_agent_evaluation-1.0.1/README.md

@@ -0,0 +1,7 @@
+# atlas-agent-evaluation
+
+Framework de avaliação provider-neutral.
+
+Este pacote integra o ecossistema modular Atlas Agent Framework. Consulte a
+[documentação oficial](https://github.com/Medicode/ai-agent-framework/tree/main/docs)
+para instalação, compatibilidade e APIs públicas.

atlas_agent_evaluation-1.0.1/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "atlas-agent-evaluation"
+dynamic = ["version"]
+description = "Avaliação provider-neutral para o Atlas Agent Framework"
+requires-python = ">=3.12"
+license = "MIT"
+license-files = ["LICENSE"]
+readme = "README.md"
+authors = [{ name = "Jorge Medina" }]
+maintainers = [{ name = "Jorge Medina" }]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Typing :: Typed",
+]
+dependencies = [
+    "atlas-agent-core~=1.0.1",
+    "pydantic>=2.13,<3",
+]
+
+[project.urls]
+Repository = "https://github.com/Medicode/ai-agent-framework"
+Documentation = "https://github.com/Medicode/ai-agent-framework/tree/main/docs/reference"
+Issues = "https://github.com/Medicode/ai-agent-framework/issues"
+
+[tool.hatch.version]
+path = "src/atlas_agents/evaluation/_version.py"
+
+[tool.hatch.build]
+dev-mode-dirs = ["src"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/atlas_agents"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/atlas_agents/evaluation", "LICENSE", "README.md", "pyproject.toml"]

atlas_agent_evaluation-1.0.1/src/atlas_agents/evaluation/__init__.py

@@ -0,0 +1,132 @@
+"""Public runtime-independent evaluation framework."""
+
+from atlas_agents.evaluation._version import __version__
+from atlas_agents.evaluation.case import (
+    EvaluationCase,
+    EvaluationExpectation,
+    EvaluationInput,
+)
+from atlas_agents.evaluation.dataset import EvaluationDataset
+from atlas_agents.evaluation.errors import (
+    DuplicateEvaluationCaseError,
+    DuplicateEvaluatorError,
+    DuplicateExpectationError,
+    EvaluationConfigurationError,
+    EvaluationDatasetError,
+    EvaluationError,
+    EvaluationExecutionError,
+    EvaluationMetricError,
+    EvaluationProtocolError,
+    EvaluatorNotRegisteredError,
+    EvaluatorRegistryError,
+    IncompatibleEvaluationMetricError,
+)
+from atlas_agents.evaluation.evaluator import (
+    EvaluationContext,
+    EvaluationContextFactory,
+    EvaluationExecutor,
+    Evaluator,
+    EvaluatorRegistry,
+)
+from atlas_agents.evaluation.evaluators import (
+    CitationEvaluator,
+    ContainsEvaluator,
+    ExactMatchEvaluator,
+    ExecutionStatusEvaluator,
+    GuardrailOutcomeEvaluator,
+    JudgeEvaluator,
+    ToolUsageEvaluator,
+)
+from atlas_agents.evaluation.executor import AgentRuntimeEvaluationExecutor
+from atlas_agents.evaluation.judge import (
+    EvaluationJudge,
+    JudgeRequest,
+    JudgeResponse,
+)
+from atlas_agents.evaluation.metric import EvaluationMetric, MetricDirection
+from atlas_agents.evaluation.observation import (
+    EvaluationCapturePolicy,
+    EvaluationObservation,
+    EvaluationObservationSummary,
+    ObservedToolCall,
+)
+from atlas_agents.evaluation.report import (
+    EvaluationCaseOutcome,
+    EvaluationCaseResult,
+    EvaluationMetricSummary,
+    EvaluationReport,
+    EvaluationReportOutcome,
+    EvaluationSummary,
+)
+from atlas_agents.evaluation.result import (
+    EvaluationErrorInfo,
+    EvaluationFinding,
+    EvaluationFindingSeverity,
+    EvaluationResult,
+    EvaluationResultStatus,
+    EvaluationScore,
+)
+from atlas_agents.evaluation.runner import (
+    EvaluationClock,
+    EvaluationIdFactory,
+    EvaluationRunner,
+)
+from atlas_agents.evaluation.summary import build_summary
+
+__all__ = [
+    "AgentRuntimeEvaluationExecutor",
+    "CitationEvaluator",
+    "ContainsEvaluator",
+    "DuplicateEvaluationCaseError",
+    "DuplicateEvaluatorError",
+    "DuplicateExpectationError",
+    "EvaluationCapturePolicy",
+    "EvaluationCase",
+    "EvaluationCaseOutcome",
+    "EvaluationCaseResult",
+    "EvaluationClock",
+    "EvaluationConfigurationError",
+    "EvaluationContext",
+    "EvaluationContextFactory",
+    "EvaluationDataset",
+    "EvaluationDatasetError",
+    "EvaluationError",
+    "EvaluationErrorInfo",
+    "EvaluationExecutionError",
+    "EvaluationExecutor",
+    "EvaluationExpectation",
+    "EvaluationFinding",
+    "EvaluationFindingSeverity",
+    "EvaluationIdFactory",
+    "EvaluationInput",
+    "EvaluationJudge",
+    "EvaluationMetric",
+    "EvaluationMetricError",
+    "EvaluationMetricSummary",
+    "EvaluationObservation",
+    "EvaluationObservationSummary",
+    "EvaluationProtocolError",
+    "EvaluationReport",
+    "EvaluationReportOutcome",
+    "EvaluationResult",
+    "EvaluationResultStatus",
+    "EvaluationRunner",
+    "EvaluationScore",
+    "EvaluationSummary",
+    "Evaluator",
+    "EvaluatorNotRegisteredError",
+    "EvaluatorRegistry",
+    "EvaluatorRegistryError",
+    "ExactMatchEvaluator",
+    "ExecutionStatusEvaluator",
+    "GuardrailOutcomeEvaluator",
+    "IncompatibleEvaluationMetricError",
+    "JudgeEvaluator",
+    "JudgeRequest",
+    "JudgeResponse",
+    "MetricDirection",
+    "ObservedToolCall",
+    "ToolUsageEvaluator",
+    "__version__",
+    "build_summary",
+]

atlas_agent_evaluation-1.0.1/src/atlas_agents/evaluation/_models.py

@@ -0,0 +1,39 @@
+"""Shared immutable and JSON-safe evaluation model helpers."""
+
+from collections.abc import Mapping
+from copy import deepcopy
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
+
+_JSON_MAPPING = TypeAdapter(dict[str, JsonValue])
+
+
+class FrozenEvaluationModel(BaseModel):
+    """Provide strict immutable value semantics to evaluation contracts."""
+
+    model_config = ConfigDict(
+        frozen=True,
+        extra="forbid",
+        arbitrary_types_allowed=True,
+    )
+
+
+def non_empty(value: str) -> str:
+    """Normalize outer whitespace and reject blank public identifiers."""
+    normalized = value.strip()
+    if not normalized:
+        raise ValueError("O valor não pode estar vazio")
+    return normalized
+
+
+def json_mapping(value: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
+    """Validate and isolate a JSON-compatible mapping."""
+    return deepcopy(_JSON_MAPPING.validate_python(dict(value)))
+
+
+def timezone_aware(value: datetime) -> datetime:
+    """Reject timestamps without a usable timezone offset."""
+    if value.tzinfo is None or value.utcoffset() is None:
+        raise ValueError("O timestamp deve possuir fuso horário")
+    return value

atlas_agent_evaluation-1.0.1/src/atlas_agents/evaluation/_version.py

@@ -0,0 +1,3 @@
+"""Package version generated from the repository VERSION file."""
+
+__version__ = "1.0.1"

atlas_agent_evaluation-1.0.1/src/atlas_agents/evaluation/case.py

@@ -0,0 +1,79 @@
+"""Immutable evaluation inputs, expectations, and cases."""
+
+from typing import Self
+
+from pydantic import Field, JsonValue, field_validator, model_validator
+
+from atlas_agents import AgentInput
+from atlas_agents.evaluation._models import (
+    FrozenEvaluationModel,
+    json_mapping,
+    non_empty,
+)
+from atlas_agents.evaluation.errors import DuplicateExpectationError
+
+
+class EvaluationInput(FrozenEvaluationModel):
+    """Describe optional production input used when executing an evaluation case."""
+
+    agent_input: AgentInput | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+    @field_validator("metadata")
+    @classmethod
+    def validate_metadata(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
+        """Keep metadata JSON-compatible and isolated."""
+        return json_mapping(value)
+
+
+class EvaluationExpectation(FrozenEvaluationModel):
+    """Bind an explicit expected value to one configured evaluator."""
+
+    expectation_id: str
+    evaluator_id: str
+    expected: JsonValue
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+    @field_validator("expectation_id", "evaluator_id")
+    @classmethod
+    def validate_ids(cls, value: str) -> str:
+        """Reject blank stable identifiers."""
+        return non_empty(value)
+
+    @field_validator("metadata")
+    @classmethod
+    def validate_metadata(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
+        """Keep metadata JSON-compatible and isolated."""
+        return json_mapping(value)
+
+
+class EvaluationCase(FrozenEvaluationModel):
+    """Describe one reproducible input and its ordered expectations."""
+
+    case_id: str
+    name: str
+    input: EvaluationInput = EvaluationInput()
+    expectations: tuple[EvaluationExpectation, ...] = ()
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+    @field_validator("case_id", "name")
+    @classmethod
+    def validate_text(cls, value: str) -> str:
+        """Reject blank case identifiers and names."""
+        return non_empty(value)
+
+    @field_validator("metadata")
+    @classmethod
+    def validate_metadata(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
+        """Keep metadata JSON-compatible and isolated."""
+        return json_mapping(value)
+
+    @model_validator(mode="after")
+    def validate_expectation_ids(self) -> Self:
+        """Require unique expectation IDs while preserving order."""
+        identifiers = tuple(item.expectation_id for item in self.expectations)
+        if len(set(identifiers)) != len(identifiers):
+            raise DuplicateExpectationError(
+                "Os identificadores de expectativa não podem se repetir."
+            )
+        return self

atlas_agent_evaluation-1.0.1/src/atlas_agents/evaluation/dataset.py

@@ -0,0 +1,45 @@
+"""Versioned ordered evaluation dataset contracts."""
+
+from typing import Self
+
+from pydantic import Field, JsonValue, field_validator, model_validator
+
+from atlas_agents.evaluation._models import (
+    FrozenEvaluationModel,
+    json_mapping,
+    non_empty,
+)
+from atlas_agents.evaluation.case import EvaluationCase
+from atlas_agents.evaluation.errors import DuplicateEvaluationCaseError
+
+
+class EvaluationDataset(FrozenEvaluationModel):
+    """Group ordered cases under a stable dataset identity and version."""
+
+    dataset_id: str
+    name: str
+    version: str
+    cases: tuple[EvaluationCase, ...] = ()
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+    @field_validator("dataset_id", "name", "version")
+    @classmethod
+    def validate_text(cls, value: str) -> str:
+        """Reject blank dataset identity fields."""
+        return non_empty(value)
+
+    @field_validator("metadata")
+    @classmethod
+    def validate_metadata(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
+        """Keep metadata JSON-compatible and isolated."""
+        return json_mapping(value)
+
+    @model_validator(mode="after")
+    def validate_case_ids(self) -> Self:
+        """Reject duplicate cases while allowing an empty dataset."""
+        identifiers = tuple(case.case_id for case in self.cases)
+        if len(set(identifiers)) != len(identifiers):
+            raise DuplicateEvaluationCaseError(
+                "Os identificadores de casos não podem se repetir."
+            )
+        return self

atlas_agent_evaluation-1.0.1/src/atlas_agents/evaluation/errors.py

@@ -0,0 +1,49 @@
+"""Typed failures for evaluation configuration and orchestration."""
+
+
+class EvaluationError(Exception):
+    """Base class for expected evaluation framework failures."""
+
+
+class EvaluationConfigurationError(EvaluationError):
+    """Report invalid evaluation configuration."""
+
+
+class EvaluationDatasetError(EvaluationConfigurationError):
+    """Report an invalid dataset contract."""
+
+
+class DuplicateEvaluationCaseError(EvaluationDatasetError):
+    """Report a repeated case identifier."""
+
+
+class DuplicateExpectationError(EvaluationDatasetError):
+    """Report a repeated expectation identifier within one case."""
+
+
+class EvaluatorRegistryError(EvaluationConfigurationError):
+    """Base class for evaluator registry failures."""
+
+
+class DuplicateEvaluatorError(EvaluatorRegistryError):
+    """Report a repeated evaluator identifier."""
+
+
+class EvaluatorNotRegisteredError(EvaluatorRegistryError):
+    """Report an expectation referencing an unknown evaluator."""
+
+
+class EvaluationProtocolError(EvaluationError):
+    """Report an evaluator or observation protocol violation."""
+
+
+class EvaluationMetricError(EvaluationError):
+    """Report an invalid metric, score, threshold, or bound."""
+
+
+class IncompatibleEvaluationMetricError(EvaluationMetricError):
+    """Report conflicting definitions sharing a metric identifier."""
+
+
+class EvaluationExecutionError(EvaluationError):
+    """Report failure to produce an observation for one case."""

atlas_agent_evaluation-1.0.1/src/atlas_agents/evaluation/evaluator.py

@@ -0,0 +1,125 @@
+"""Async evaluator, executor, registry, and context contracts."""
+
+from collections.abc import Callable
+from typing import Protocol
+
+from pydantic import Field, JsonValue, field_validator
+
+from atlas_agents import AgentContext
+from atlas_agents.evaluation._models import (
+    FrozenEvaluationModel,
+    json_mapping,
+    non_empty,
+)
+from atlas_agents.evaluation.case import EvaluationCase, EvaluationExpectation
+from atlas_agents.evaluation.errors import (
+    DuplicateEvaluatorError,
+    EvaluatorNotRegisteredError,
+)
+from atlas_agents.evaluation.metric import EvaluationMetric
+from atlas_agents.evaluation.observation import EvaluationObservation
+from atlas_agents.evaluation.result import EvaluationResult
+
+
+class EvaluationContext(FrozenEvaluationModel):
+    """Carry immutable correlation facts for one evaluator invocation."""
+
+    evaluation_run_id: str
+    dataset_id: str
+    case_id: str
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+    @field_validator("evaluation_run_id", "dataset_id", "case_id")
+    @classmethod
+    def validate_ids(cls, value: str) -> str:
+        """Reject blank evaluation correlation identifiers."""
+        return non_empty(value)
+
+    @field_validator("metadata")
+    @classmethod
+    def validate_metadata(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
+        """Keep context metadata JSON-compatible and isolated."""
+        return json_mapping(value)
+
+
+class Evaluator(Protocol):
+    """Evaluate one explicit expectation against an immutable observation."""
+
+    @property
+    def evaluator_id(self) -> str:
+        """Return the stable evaluator identifier."""
+        ...
+
+    @property
+    def metric(self) -> EvaluationMetric:
+        """Return the metric definition emitted by this evaluator."""
+        ...
+
+    async def evaluate(
+        self,
+        case: EvaluationCase,
+        expectation: EvaluationExpectation,
+        observation: EvaluationObservation,
+        context: EvaluationContext,
+    ) -> EvaluationResult:
+        """Measure the observation without mutating production artifacts."""
+        ...
+
+
+class EvaluationExecutor(Protocol):
+    """Produce an immutable observation through an explicitly configured path."""
+
+    async def execute(self, case: EvaluationCase) -> EvaluationObservation:
+        """Execute or retrieve one case observation."""
+        ...
+
+
+type EvaluationContextFactory = Callable[[EvaluationCase], AgentContext]
+
+
+class EvaluatorRegistry:
+    """Store evaluators by exact identifier in deterministic registration order."""
+
+    def __init__(self, evaluators: tuple[Evaluator, ...] = ()) -> None:
+        """Initialize an isolated registry and register initial evaluators."""
+        self._evaluators: dict[str, Evaluator] = {}
+        for evaluator in evaluators:
+            self.register(evaluator)
+
+    @property
+    def evaluators(self) -> tuple[Evaluator, ...]:
+        """Return evaluators in registration order."""
+        return tuple(self._evaluators.values())
+
+    def register(self, evaluator: Evaluator) -> None:
+        """Register one evaluator and reject duplicate IDs."""
+        identifier = non_empty(evaluator.evaluator_id)
+        if identifier in self._evaluators:
+            raise DuplicateEvaluatorError(
+                f"O evaluator '{identifier}' já está registrado."
+            )
+        self._evaluators[identifier] = evaluator
+
+    def unregister(self, evaluator_id: str) -> Evaluator:
+        """Remove and return an evaluator by exact identifier."""
+        identifier = non_empty(evaluator_id)
+        evaluator = self._evaluators.pop(identifier, None)
+        if evaluator is None:
+            raise EvaluatorNotRegisteredError(
+                f"O evaluator '{identifier}' não está registrado."
+            )
+        return evaluator
+
+    def get(self, evaluator_id: str) -> Evaluator:
+        """Resolve an evaluator or raise a typed error."""
+        identifier = non_empty(evaluator_id)
+        evaluator = self._evaluators.get(identifier)
+        if evaluator is None:
+            raise EvaluatorNotRegisteredError(
+                f"O evaluator '{identifier}' não está registrado."
+            )
+        return evaluator
+
+    def try_get(self, evaluator_id: str) -> Evaluator | None:
+        """Resolve an evaluator without raising when it is absent."""
+        return self._evaluators.get(non_empty(evaluator_id))

atlas_agent_evaluation-1.0.1/src/atlas_agents/evaluation/evaluators/__init__.py

@@ -0,0 +1,19 @@
+"""Built-in deterministic and opt-in evaluation implementations."""
+
+from atlas_agents.evaluation.evaluators.citation import CitationEvaluator
+from atlas_agents.evaluation.evaluators.contains import ContainsEvaluator
+from atlas_agents.evaluation.evaluators.exact_match import ExactMatchEvaluator
+from atlas_agents.evaluation.evaluators.guardrail import GuardrailOutcomeEvaluator
+from atlas_agents.evaluation.evaluators.judge import JudgeEvaluator
+from atlas_agents.evaluation.evaluators.status import ExecutionStatusEvaluator
+from atlas_agents.evaluation.evaluators.tool_usage import ToolUsageEvaluator
+
+__all__ = [
+    "CitationEvaluator",
+    "ContainsEvaluator",
+    "ExactMatchEvaluator",
+    "ExecutionStatusEvaluator",
+    "GuardrailOutcomeEvaluator",
+    "JudgeEvaluator",
+    "ToolUsageEvaluator",
+]