agentic-evals 0.1.0__py3-none-any.whl

agentic_evals/__init__.py

@@ -0,0 +1,69 @@
+"""A standalone, framework-agnostic evaluation and scoring engine.
+
+Score whatever trace-shaped data you give it (see `EvalTrace`/`EvalSpan`) —
+this package has no dependency on any specific tracing/observability tool.
+Extracted from AgenticLens's evaluation module (github.com/DeepAgentLabs/agenticlens).
+"""
+
+from agentic_evals.evaluators import (
+    BusinessRuleEvaluator,
+    CallableEvaluator,
+    EvaluationContext,
+    Evaluator,
+    EvaluatorRegistry,
+    LLMJudgeEvaluator,
+)
+from agentic_evals.gate import GateConfig, GateDecision, evaluate_gate
+from agentic_evals.models import (
+    CaseEvaluation,
+    EvalSpan,
+    EvalTrace,
+    EvaluationReport,
+    EvaluationSample,
+    EvaluationSummary,
+    EvaluatorConfig,
+    HTTPTarget,
+    LiveTarget,
+    PythonTarget,
+    Score,
+    TestCase,
+    TestSuite,
+)
+from agentic_evals.runner import (
+    evaluate_suite,
+    load_samples,
+    load_suite,
+    run_live_suite,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "BusinessRuleEvaluator",
+    "CallableEvaluator",
+    "CaseEvaluation",
+    "EvalSpan",
+    "EvalTrace",
+    "EvaluationContext",
+    "EvaluationReport",
+    "EvaluationSample",
+    "EvaluationSummary",
+    "Evaluator",
+    "EvaluatorConfig",
+    "EvaluatorRegistry",
+    "GateConfig",
+    "GateDecision",
+    "HTTPTarget",
+    "LLMJudgeEvaluator",
+    "LiveTarget",
+    "PythonTarget",
+    "Score",
+    "TestCase",
+    "TestSuite",
+    "__version__",
+    "evaluate_gate",
+    "evaluate_suite",
+    "load_samples",
+    "load_suite",
+    "run_live_suite",
+]

agentic_evals/evaluators.py

@@ -0,0 +1,95 @@
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Protocol
+
+from agentic_evals.models import (
+    EvaluationSample,
+    EvaluatorConfig,
+    Score,
+    TestCase,
+)
+
+
+@dataclass(frozen=True)
+class EvaluationContext:
+    case: TestCase
+    sample: EvaluationSample
+    config: EvaluatorConfig
+
+
+class Evaluator(Protocol):
+    @property
+    def name(self) -> str: ...
+
+    def evaluate(self, context: EvaluationContext) -> list[Score]: ...
+
+
+EvaluationFunction = Callable[[EvaluationContext], Score | list[Score]]
+
+
+class CallableEvaluator:
+    """Adapt a trusted Python function to the unified evaluator contract."""
+
+    def __init__(
+        self,
+        name: str,
+        function: EvaluationFunction,
+        *,
+        evaluator_type: str = "custom",
+    ) -> None:
+        if not name:
+            raise ValueError("evaluator name must not be empty")
+        self._name = name
+        self._function = function
+        self._evaluator_type = evaluator_type
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def evaluate(self, context: EvaluationContext) -> list[Score]:
+        result = self._function(context)
+        scores = result if isinstance(result, list) else [result]
+        return [
+            score.model_copy(
+                update={
+                    "evaluator_type": self._evaluator_type,
+                    "passed": score.value >= context.config.threshold,
+                    "required": context.config.required,
+                }
+            )
+            for score in scores
+        ]
+
+
+class LLMJudgeEvaluator(CallableEvaluator):
+    """Provider-neutral adapter for an application-supplied LLM judge function."""
+
+    def __init__(self, name: str, judge: EvaluationFunction) -> None:
+        super().__init__(name, judge, evaluator_type="llm_judge")
+
+
+class BusinessRuleEvaluator(CallableEvaluator):
+    """Named helper for application-specific pass/fail logic."""
+
+    def __init__(self, name: str, rule: EvaluationFunction) -> None:
+        super().__init__(name, rule, evaluator_type="business_rule")
+
+
+class EvaluatorRegistry:
+    def __init__(self) -> None:
+        self._evaluators: dict[str, Evaluator] = {}
+
+    def register(self, evaluator: Evaluator, *, replace: bool = False) -> None:
+        if evaluator.name in self._evaluators and not replace:
+            raise ValueError(f"evaluator {evaluator.name!r} is already registered")
+        self._evaluators[evaluator.name] = evaluator
+
+    def get(self, name: str) -> Evaluator:
+        try:
+            return self._evaluators[name]
+        except KeyError as exc:
+            raise ValueError(f"evaluator {name!r} is not registered") from exc
+
+    def names(self) -> tuple[str, ...]:
+        return tuple(sorted(self._evaluators))

agentic_evals/gate.py

@@ -0,0 +1,61 @@
+from pydantic import BaseModel, Field
+
+from agentic_evals.models import EvaluationReport
+
+
+class GateConfig(BaseModel):
+    min_pass_rate: float = Field(default=1.0, ge=0, le=1)
+    min_average_score: float = Field(default=1.0, ge=0, le=1)
+    max_failed_cases: int = Field(default=0, ge=0)
+    max_average_latency_ms: float | None = Field(default=None, gt=0)
+    max_total_cost_usd: float | None = Field(default=None, ge=0)
+
+
+class GateDecision(BaseModel):
+    passed: bool
+    reasons: list[str]
+    observed: dict[str, float | int | None]
+
+
+def evaluate_gate(report: EvaluationReport, config: GateConfig) -> GateDecision:
+    summary = report.summary
+    reasons: list[str] = []
+    if summary.pass_rate < config.min_pass_rate:
+        reasons.append(f"Pass rate {summary.pass_rate:.1%} is below {config.min_pass_rate:.1%}.")
+    if summary.average_score < config.min_average_score:
+        reasons.append(
+            f"Average score {summary.average_score:.3f} is below {config.min_average_score:.3f}."
+        )
+    if summary.failed_cases > config.max_failed_cases:
+        reasons.append(f"Failed cases {summary.failed_cases} exceed {config.max_failed_cases}.")
+    if (
+        config.max_average_latency_ms is not None
+        and summary.average_latency_ms > config.max_average_latency_ms
+    ):
+        reasons.append(
+            f"Average latency {summary.average_latency_ms:.1f} ms exceeds "
+            f"{config.max_average_latency_ms:.1f} ms."
+        )
+    if config.max_total_cost_usd is not None:
+        if (
+            summary.total_cost_usd is None
+            or len(report.cases) != summary.total_cases
+            or any(case.cost_usd is None for case in report.cases)
+        ):
+            reasons.append("Total cost is unavailable or incomplete.")
+        elif summary.total_cost_usd > config.max_total_cost_usd:
+            reasons.append(
+                f"Total cost ${summary.total_cost_usd:.6f} exceeds "
+                f"${config.max_total_cost_usd:.6f}."
+            )
+    return GateDecision(
+        passed=not reasons,
+        reasons=reasons,
+        observed={
+            "pass_rate": summary.pass_rate,
+            "average_score": summary.average_score,
+            "failed_cases": summary.failed_cases,
+            "average_latency_ms": summary.average_latency_ms,
+            "total_cost_usd": summary.total_cost_usd,
+        },
+    )

agentic_evals/models.py

@@ -0,0 +1,154 @@
+from datetime import datetime, timezone
+from typing import Any
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class EvalSpan(BaseModel):
+    """The minimal per-span shape the evaluation engine actually inspects.
+
+    Deliberately not AgenticLens's `Span` — a much richer model with agent
+    names, model/provider, retries, timestamps, and more. This package
+    never needs any of that, so it doesn't ask for it. Anyone with any
+    trace-shaped data can produce an `EvalTrace` without depending on a
+    specific tracing tool's schema.
+    """
+
+    tool_name: str | None = None
+    attributes: dict[str, Any] = Field(default_factory=dict)
+
+
+class EvalTrace(BaseModel):
+    """The minimal per-run shape the evaluation engine actually inspects."""
+
+    trace_id: str = ""
+    spans: list[EvalSpan] = Field(default_factory=list)
+    total_latency_ms: float = 0.0
+    estimated_cost_usd: float | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class EvaluatorConfig(BaseModel):
+    name: str = Field(min_length=1)
+    threshold: float = Field(default=1.0, ge=0, le=1)
+    required: bool = True
+    config: dict[str, Any] = Field(default_factory=dict)
+
+
+class TestCase(BaseModel):
+    id: str
+    name: str
+    input: Any = None
+    expected_output: str | None = None
+    expected_contains: list[str] = Field(default_factory=list)
+    output_json_schema: dict[str, Any] | None = None
+    required_output_fields: list[str] = Field(default_factory=list)
+    required_tools: list[str] = Field(default_factory=list)
+    forbidden_tools: list[str] = Field(default_factory=list)
+    required_tool_arguments: dict[str, list[str]] = Field(default_factory=dict)
+    max_latency_ms: float | None = Field(default=None, gt=0)
+    max_cost_usd: float | None = Field(default=None, ge=0)
+    max_turns: int | None = Field(default=None, gt=0)
+    evaluators: list[EvaluatorConfig] = Field(default_factory=list)
+    tags: list[str] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def require_expectation(self) -> "TestCase":
+        if not any(
+            (
+                self.expected_output is not None,
+                self.expected_contains,
+                self.output_json_schema is not None,
+                self.required_output_fields,
+                self.required_tools,
+                self.forbidden_tools,
+                self.required_tool_arguments,
+                self.max_latency_ms is not None,
+                self.max_cost_usd is not None,
+                self.max_turns is not None,
+                self.evaluators,
+            )
+        ):
+            raise ValueError("test case must define at least one expectation")
+        return self
+
+
+class TestSuite(BaseModel):
+    name: str
+    version: str
+    description: str = ""
+    cases: list[TestCase]
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def unique_case_ids(self) -> "TestSuite":
+        ids = [case.id for case in self.cases]
+        if len(ids) != len(set(ids)):
+            raise ValueError("test case IDs must be unique")
+        if not ids:
+            raise ValueError("test suite must contain at least one case")
+        return self
+
+
+class EvaluationSample(BaseModel):
+    case_id: str
+    output: str
+    trace: EvalTrace
+
+
+class Score(BaseModel):
+    name: str
+    value: float = Field(ge=0, le=1)
+    passed: bool
+    required: bool = True
+    explanation: str
+    evaluator_type: str = "deterministic"
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class CaseEvaluation(BaseModel):
+    case_id: str
+    case_name: str
+    passed: bool
+    scores: list[Score]
+    output: str
+    trace_id: str
+    latency_ms: float
+    cost_usd: float | None = None
+
+
+class EvaluationSummary(BaseModel):
+    total_cases: int
+    passed_cases: int
+    failed_cases: int
+    pass_rate: float = Field(ge=0, le=1)
+    average_score: float = Field(ge=0, le=1)
+    total_cost_usd: float | None = None
+    average_latency_ms: float
+
+
+class EvaluationReport(BaseModel):
+    schema_version: str = "1.0"
+    suite_name: str
+    suite_version: str
+    created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    summary: EvaluationSummary
+    cases: list[CaseEvaluation]
+
+
+class LiveTarget(BaseModel):
+    kind: str = Field(pattern="^(python|http)$")
+    timeout_seconds: float = Field(default=30.0, gt=0)
+
+
+class PythonTarget(LiveTarget):
+    kind: str = "python"
+    callable_path: str
+
+
+class HTTPTarget(LiveTarget):
+    kind: str = "http"
+    url: str
+    method: str = "POST"
+    headers: dict[str, str] = Field(default_factory=dict)

agentic_evals/runner.py

@@ -0,0 +1,409 @@
+import json
+import urllib.error
+import urllib.request
+from importlib import import_module
+from importlib.util import module_from_spec, spec_from_file_location
+from pathlib import Path
+from typing import Any
+
+import yaml
+from jsonschema import Draft202012Validator, SchemaError
+from referencing import Registry
+from referencing.exceptions import Unresolvable
+
+from agentic_evals.evaluators import EvaluationContext, EvaluatorRegistry
+from agentic_evals.models import (
+    CaseEvaluation,
+    EvaluationReport,
+    EvaluationSample,
+    EvaluationSummary,
+    HTTPTarget,
+    LiveTarget,
+    PythonTarget,
+    Score,
+    TestCase,
+    TestSuite,
+)
+
+
+def _load_data(path: Path) -> Any:
+    text = path.read_text(encoding="utf-8")
+    return json.loads(text) if path.suffix.lower() == ".json" else yaml.safe_load(text)
+
+
+def load_suite(path: Path) -> TestSuite:
+    return TestSuite.model_validate(_load_data(path))
+
+
+def load_samples(path: Path) -> list[EvaluationSample]:
+    data = _load_data(path)
+    items = data["samples"] if isinstance(data, dict) and "samples" in data else data
+    return [EvaluationSample.model_validate(item) for item in items]
+
+
+def _lookup_path(payload: Any, dotted_path: str) -> tuple[bool, Any]:
+    current = payload
+    for part in dotted_path.split("."):
+        if isinstance(current, dict) and part in current:
+            current = current[part]
+        elif isinstance(current, list) and part.isdigit() and int(part) < len(current):
+            current = current[int(part)]
+        else:
+            return False, None
+    return True, current
+
+
+def _schema_validator(schema: dict[str, Any]) -> Any:
+    dialect = schema.get("$schema", "https://json-schema.org/draft/2020-12/schema")
+    if dialect != "https://json-schema.org/draft/2020-12/schema":
+        raise ValueError("output_json_schema supports JSON Schema Draft 2020-12 only")
+    try:
+        Draft202012Validator.check_schema(schema)
+    except SchemaError as exc:
+        raise ValueError(f"Invalid output_json_schema: {exc.message}") from exc
+    # Resolve embedded references only; never fetch schemas from the network.
+    return Draft202012Validator(schema, registry=Registry())
+
+
+def _validate_json_schema(payload: Any, schema: dict[str, Any]) -> tuple[bool, str]:
+    validator = _schema_validator(schema)
+    try:
+        error = next(validator.iter_errors(payload), None)
+    except Unresolvable as exc:
+        raise ValueError(f"Unresolvable output_json_schema reference: {exc}") from exc
+    if error is not None:
+        return False, f"Output violates JSON Schema at {error.json_path}: {error.message}"
+    return True, "Output matches the configured JSON Schema Draft 2020-12."
+
+
+def _reject_json_constant(value: str) -> Any:
+    raise ValueError(f"Non-finite JSON constant {value} is not allowed")
+
+
+def _turn_count(sample: EvaluationSample) -> int | None:
+    metadata_turns = sample.trace.metadata.get("turn_count")
+    if isinstance(metadata_turns, int) and metadata_turns > 0:
+        return metadata_turns
+    return None
+
+
+def _score_case(
+    case: TestCase,
+    sample: EvaluationSample,
+    registry: EvaluatorRegistry | None,
+) -> list[Score]:
+    scores: list[Score] = []
+    output = sample.output.strip()
+    if case.expected_output is not None:
+        passed = output == case.expected_output.strip()
+        scores.append(
+            Score(
+                name="exact_match",
+                value=float(passed),
+                passed=passed,
+                explanation="Output exactly matches the reference."
+                if passed
+                else "Output does not exactly match the reference.",
+            )
+        )
+    for expected in case.expected_contains:
+        passed = expected.casefold() in output.casefold()
+        scores.append(
+            Score(
+                name=f"contains:{expected}",
+                value=float(passed),
+                passed=passed,
+                explanation=f"Output contains required text: {expected!r}."
+                if passed
+                else f"Output is missing required text: {expected!r}.",
+            )
+        )
+    parsed_output: Any | None = None
+    output_json_error: str | None = None
+    if case.output_json_schema is not None or case.required_output_fields:
+        try:
+            parsed_output = json.loads(output, parse_constant=_reject_json_constant)
+        except ValueError as exc:
+            output_json_error = str(exc)
+    if case.output_json_schema is not None:
+        if output_json_error is not None:
+            scores.append(
+                Score(
+                    name="json_schema",
+                    value=0.0,
+                    passed=False,
+                    explanation=f"Output is not valid JSON: {output_json_error}.",
+                )
+            )
+        else:
+            valid, reason = _validate_json_schema(parsed_output, case.output_json_schema)
+            scores.append(
+                Score(
+                    name="json_schema",
+                    value=float(valid),
+                    passed=valid,
+                    explanation=reason,
+                )
+            )
+    for field_path in case.required_output_fields:
+        exists = (
+            output_json_error is None
+            and parsed_output is not None
+            and _lookup_path(parsed_output, field_path)[0]
+        )
+        scores.append(
+            Score(
+                name=f"required_field:{field_path}",
+                value=float(exists),
+                passed=exists,
+                explanation=f"Output contains required field {field_path!r}."
+                if exists
+                else (
+                    "Output is not valid JSON, so required field "
+                    f"{field_path!r} could not be checked."
+                    if output_json_error is not None
+                    else f"Output is missing required field {field_path!r}."
+                ),
+            )
+        )
+
+    tools = {span.tool_name for span in sample.trace.spans if span.tool_name}
+    for tool in case.required_tools:
+        passed = tool in tools
+        scores.append(
+            Score(
+                name=f"required_tool:{tool}",
+                value=float(passed),
+                passed=passed,
+                explanation=f"Required tool {tool!r} was called."
+                if passed
+                else f"Required tool {tool!r} was not called.",
+            )
+        )
+    for tool in case.forbidden_tools:
+        passed = tool not in tools
+        scores.append(
+            Score(
+                name=f"forbidden_tool:{tool}",
+                value=float(passed),
+                passed=passed,
+                explanation=f"Forbidden tool {tool!r} was not called."
+                if passed
+                else f"Forbidden tool {tool!r} was called.",
+            )
+        )
+    for tool_name, required_args in case.required_tool_arguments.items():
+        matching_spans = [span for span in sample.trace.spans if span.tool_name == tool_name]
+        args_present = False
+        for span in matching_spans:
+            tool_args = span.attributes.get("tool_args")
+            if isinstance(tool_args, dict) and all(arg in tool_args for arg in required_args):
+                args_present = True
+                break
+        scores.append(
+            Score(
+                name=f"tool_args:{tool_name}",
+                value=float(args_present),
+                passed=args_present,
+                explanation=f"Tool {tool_name!r} included required arguments {required_args}."
+                if args_present
+                else f"Tool {tool_name!r} did not include required arguments {required_args}.",
+            )
+        )
+    if case.max_latency_ms is not None:
+        passed = sample.trace.total_latency_ms <= case.max_latency_ms
+        scores.append(
+            Score(
+                name="latency_threshold",
+                value=float(passed),
+                passed=passed,
+                explanation=(
+                    f"Latency {sample.trace.total_latency_ms:.1f} ms "
+                    f"{'meets' if passed else 'exceeds'} the "
+                    f"{case.max_latency_ms:.1f} ms limit."
+                ),
+            )
+        )
+    if case.max_cost_usd is not None:
+        cost = sample.trace.estimated_cost_usd
+        passed = cost is not None and cost <= case.max_cost_usd
+        scores.append(
+            Score(
+                name="cost_threshold",
+                value=float(passed),
+                passed=passed,
+                explanation=(
+                    f"Cost ${cost:.6f} meets the ${case.max_cost_usd:.6f} limit."
+                    if passed and cost is not None
+                    else "Cost is unavailable or exceeds the configured limit."
+                ),
+            )
+        )
+    if case.max_turns is not None:
+        turns = _turn_count(sample)
+        passed = turns is not None and turns <= case.max_turns
+        scores.append(
+            Score(
+                name="turn_count_threshold",
+                value=float(passed),
+                passed=passed,
+                explanation=(
+                    f"Turn count {turns} {'meets' if passed else 'exceeds'} the "
+                    f"{case.max_turns} turn limit."
+                    if turns is not None
+                    else (
+                        "Trace metadata is missing a positive integer turn_count, "
+                        "so the max_turns check could not be evaluated."
+                    )
+                ),
+            )
+        )
+    for evaluator_config in case.evaluators:
+        if registry is None:
+            raise ValueError(
+                f"test case {case.id!r} requires evaluator {evaluator_config.name!r}, "
+                "but no evaluator registry was supplied"
+            )
+        evaluator = registry.get(evaluator_config.name)
+        custom_scores = evaluator.evaluate(
+            EvaluationContext(case=case, sample=sample, config=evaluator_config)
+        )
+        scores.extend(custom_scores)
+    return scores
+
+
+def _load_python_callable(callable_path: str) -> Any:
+    """Load a trusted Python live target from module:function or path.py:function."""
+    module_name, _, attr_path = callable_path.rpartition(":")
+    if not module_name or not attr_path:
+        raise ValueError("python target callable_path must be in module:function format")
+    try:
+        target: Any = import_module(module_name)
+    except ModuleNotFoundError as exc:
+        file_path = Path(module_name)
+        if not file_path.exists():
+            raise
+        spec = spec_from_file_location(file_path.stem, file_path)
+        if spec is None or spec.loader is None:
+            raise ValueError(f"Unable to load Python target from {file_path}") from exc
+        module = module_from_spec(spec)
+        spec.loader.exec_module(module)
+        target = module
+    for part in attr_path.split("."):
+        target = getattr(target, part)
+    return target
+
+
+def run_live_suite(
+    suite: TestSuite,
+    target: LiveTarget,
+    *,
+    registry: EvaluatorRegistry | None = None,
+) -> EvaluationReport:
+    """Run a trusted live target against every case in a suite.
+
+    Live targets are intentionally powerful developer-facing integrations:
+    Python targets execute local code and HTTP targets can reach arbitrary URLs.
+    Only use trusted suite files and trusted target definitions.
+    """
+    samples: list[EvaluationSample] = []
+    if isinstance(target, PythonTarget):
+        callable_target = _load_python_callable(target.callable_path)
+        for case in suite.cases:
+            result = callable_target(case.input, case=case)
+            sample = EvaluationSample.model_validate({**result, "case_id": case.id})
+            samples.append(sample)
+    elif isinstance(target, HTTPTarget):
+        for case in suite.cases:
+            request = urllib.request.Request(
+                target.url,
+                data=json.dumps({"input": case.input, "case_id": case.id}).encode("utf-8"),
+                headers={"Content-Type": "application/json", **target.headers},
+                method=target.method.upper(),
+            )
+            try:
+                with urllib.request.urlopen(request, timeout=target.timeout_seconds) as response:
+                    payload = json.loads(response.read().decode("utf-8"))
+            except urllib.error.URLError as exc:
+                raise ValueError(f"HTTP target request failed for case {case.id!r}: {exc}") from exc
+            sample = EvaluationSample.model_validate({**payload, "case_id": case.id})
+            samples.append(sample)
+    else:
+        raise ValueError(f"Unsupported live target kind: {target.kind}")
+    return evaluate_suite(suite, samples, registry=registry)
+
+
+def evaluate_suite(
+    suite: TestSuite,
+    samples: list[EvaluationSample],
+    *,
+    registry: EvaluatorRegistry | None = None,
+) -> EvaluationReport:
+    case_ids = [case.id for case in suite.cases]
+    if not case_ids or len(case_ids) != len(set(case_ids)):
+        raise ValueError("suite must contain nonempty, unique case IDs")
+    for case in suite.cases:
+        if case.output_json_schema is not None:
+            _schema_validator(case.output_json_schema)
+    by_case: dict[str, EvaluationSample] = {}
+    expected = set(case_ids)
+    for supplied in samples:
+        if supplied.case_id in by_case:
+            raise ValueError(f"Duplicate sample case ID: {supplied.case_id!r}")
+        if supplied.case_id not in expected:
+            raise ValueError(f"Unknown sample case ID: {supplied.case_id!r}")
+        by_case[supplied.case_id] = supplied
+    results: list[CaseEvaluation] = []
+    for case in suite.cases:
+        sample = by_case.get(case.id)
+        if sample is None:
+            scores = [
+                Score(
+                    name="sample_available",
+                    value=0,
+                    passed=False,
+                    explanation="No evaluation sample was supplied for this test case.",
+                )
+            ]
+            results.append(
+                CaseEvaluation(
+                    case_id=case.id,
+                    case_name=case.name,
+                    passed=False,
+                    scores=scores,
+                    output="",
+                    trace_id="",
+                    latency_ms=0,
+                )
+            )
+            continue
+        scores = _score_case(case, sample, registry)
+        results.append(
+            CaseEvaluation(
+                case_id=case.id,
+                case_name=case.name,
+                passed=all(score.passed or not score.required for score in scores),
+                scores=scores,
+                output=sample.output,
+                trace_id=sample.trace.trace_id,
+                latency_ms=sample.trace.total_latency_ms,
+                cost_usd=sample.trace.estimated_cost_usd,
+            )
+        )
+    passed = sum(result.passed for result in results)
+    all_scores = [score.value for result in results for score in result.scores]
+    costs = [result.cost_usd for result in results if result.cost_usd is not None]
+    return EvaluationReport(
+        suite_name=suite.name,
+        suite_version=suite.version,
+        summary=EvaluationSummary(
+            total_cases=len(results),
+            passed_cases=passed,
+            failed_cases=len(results) - passed,
+            pass_rate=passed / len(results),
+            average_score=sum(all_scores) / len(all_scores),
+            total_cost_usd=sum(costs) if len(costs) == len(results) else None,
+            average_latency_ms=sum(result.latency_ms for result in results) / len(results),
+        ),
+        cases=results,
+    )

agentic_evals-0.1.0.dist-info/METADATA

@@ -0,0 +1,237 @@
+Metadata-Version: 2.5
+Name: agentic-evals
+Version: 0.1.0
+Summary: A standalone, framework-agnostic evaluation and scoring engine for LLM/agent outputs — extracted from AgenticLens's evaluation module.
+Project-URL: Homepage, https://github.com/DeepAgentLabs/agentic-evals
+Project-URL: Repository, https://github.com/DeepAgentLabs/agentic-evals
+Project-URL: Issues, https://github.com/DeepAgentLabs/agentic-evals/issues
+Project-URL: Changelog, https://github.com/DeepAgentLabs/agentic-evals/blob/main/CHANGELOG.md
+Author: agentic-evals Contributors
+License: MIT License
+        
+        Copyright (c) 2026 DeepAgentLabs
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: agents,evaluation,llm,llm-as-judge,scoring,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Requires-Dist: jsonschema>=4.23
+Requires-Dist: pydantic<3,>=2.0
+Requires-Dist: pyyaml<7,>=6.0
+Requires-Dist: referencing>=0.35
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Requires-Dist: twine>=5.1; extra == 'dev'
+Requires-Dist: types-pyyaml>=6.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# agentic-evals
+
+**A standalone, framework-agnostic evaluation and scoring engine for LLM and
+agent outputs.**
+
+Extracted from [AgenticLens](https://github.com/DeepAgentLabs/agenticlens)'s
+proven evaluation module — the same engine, usable on its own. It scores
+whatever trace-shaped data you give it (see `EvalTrace`/`EvalSpan` below);
+it has no dependency on any specific tracing/observability tool, and no
+dependency on AgenticLens itself.
+
+## Status
+
+**Alpha.** The engine (deterministic checks, LLM-as-judge and custom
+evaluators, a release gate, live Python/HTTP targets) is real, tested code
+lifted directly from AgenticLens's evaluation module. No PyPI release yet.
+
+## Why a separate package
+
+AgenticLens's evaluation module was never actually AgenticLens-specific —
+scoring an LLM/agent output against expectations doesn't need AgenticLens's
+full trace schema, CLI, or dashboards. Pulling it out means:
+
+- `agentic-sidecar`, `agentic-chaos`, or any other project can score outputs
+  without depending on all of AgenticLens.
+- Anyone with *any* trace-shaped data — not just AgenticLens users — can use
+  it, the same way Braintrust's `autoevals` doesn't care what produced the
+  string it's scoring.
+
+## Install
+
+```bash
+pip install agentic-evals
+```
+
+## Core concepts
+
+- **`EvalTrace`/`EvalSpan`** — the minimal trace shape the engine inspects:
+  a trace id, spans (each optionally naming a `tool_name` and carrying
+  arbitrary `attributes`), total latency, estimated cost, and metadata.
+  Deliberately not tied to any specific instrumentation format — build one
+  from whatever you already have.
+- **`Score`** — a single named judgment (0-1 value, pass/fail, explanation).
+- **`Evaluator`** — anything with a `.name` and an `.evaluate(context) ->
+  list[Score]`. `CallableEvaluator` adapts a plain Python function;
+  `LLMJudgeEvaluator` and `BusinessRuleEvaluator` are named convenience
+  subclasses for readability/reporting.
+- **`TestCase`/`TestSuite`** — declarative expectations (exact match,
+  substring, JSON Schema, required fields, required/forbidden tool calls,
+  required tool arguments, latency/cost/turn-count thresholds, or a named
+  custom evaluator) plus the cases that make up a suite.
+- **`evaluate_suite`** — runs a suite against supplied `EvaluationSample`s
+  and returns an `EvaluationReport` (per-case scores plus a pass-rate/cost/
+  latency summary).
+- **`GateConfig`/`evaluate_gate`** — turn an `EvaluationReport` into a
+  pass/fail release decision on configurable thresholds.
+
+## Quickstart
+
+```python
+from agentic_evals import (
+    EvalSpan,
+    EvalTrace,
+    EvaluationSample,
+    TestCase,
+    TestSuite,
+    evaluate_suite,
+)
+
+suite = TestSuite(
+    name="support-answers",
+    version="1",
+    cases=[
+        TestCase(
+            id="case-1",
+            name="Answer contains the right total",
+            expected_contains=["42"],
+            required_tools=["calculator"],
+            max_latency_ms=2000,
+        )
+    ],
+)
+
+sample = EvaluationSample(
+    case_id="case-1",
+    output="The combined total is 42.",
+    trace=EvalTrace(
+        trace_id="trace-1",
+        total_latency_ms=350,
+        spans=[EvalSpan(tool_name="calculator")],
+    ),
+)
+
+report = evaluate_suite(suite, [sample])
+print(report.summary.pass_rate)  # 1.0
+```
+
+## LLM-as-judge
+
+```python
+from agentic_evals import (
+    EvaluationContext,
+    EvaluatorConfig,
+    EvaluatorRegistry,
+    LLMJudgeEvaluator,
+    Score,
+    TestCase,
+)
+
+
+def judge(context: EvaluationContext) -> Score:
+    # Call whatever model/provider you like here.
+    correct = "42" in context.sample.output
+    return Score(
+        name="answer_quality",
+        value=0.95 if correct else 0.1,
+        passed=correct,
+        explanation="Judged against the rubric in context.config.config.",
+    )
+
+
+registry = EvaluatorRegistry()
+registry.register(LLMJudgeEvaluator("answer_quality_judge", judge))
+
+case = TestCase(
+    id="case-1",
+    name="Answer quality",
+    evaluators=[EvaluatorConfig(name="answer_quality_judge", threshold=0.8)],
+)
+```
+
+## Release gates
+
+```python
+from agentic_evals import GateConfig, evaluate_gate
+
+decision = evaluate_gate(
+    report,
+    GateConfig(min_pass_rate=0.95, max_average_latency_ms=1500, max_total_cost_usd=0.25),
+)
+if not decision.passed:
+    raise SystemExit(f"Release gate failed: {decision.reasons}")
+```
+
+Never fabricates a value it can't back up: `total_cost_usd` on a summary or
+gate decision stays `None` unless every case in scope has a known cost —
+an incomplete cost picture is reported as unavailable, not `$0.00`.
+
+## Live targets
+
+Point a suite at a real running system (a trusted Python callable, or an
+HTTP endpoint) instead of pre-recorded samples:
+
+```python
+from agentic_evals import PythonTarget, run_live_suite
+
+report = run_live_suite(suite, PythonTarget(callable_path="my_module:run_case"))
+```
+
+Live targets are intentionally powerful developer-facing integrations —
+Python targets execute local code and HTTP targets can reach arbitrary
+URLs. Only point them at trusted suite files and trusted target
+definitions.
+
+## Using it with AgenticLens's own traces
+
+If you already have an AgenticLens `Run` (from its instrumentation API or
+OTLP ingestion), AgenticLens itself provides the adapter —
+`agenticlens.evaluation.to_eval_trace(run)` — so you don't have to hand-build
+an `EvalTrace`. This package has no dependency in the other direction.
+
+## What's deliberately not here
+
+Dataset versioning/splitting, judge calibration, and HTML report rendering
+stay in AgenticLens for now — those are product features built *on top of*
+this engine (the same way Braintrust's dataset/experiment platform is
+separate from the `autoevals` library itself), not the engine.
+
+## License
+
+MIT

agentic_evals-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+agentic_evals/__init__.py,sha256=F-ZnWxm5P89v86MKkBf3oaEWtdw4gJmLKK0hoNGRKS8,1531
+agentic_evals/evaluators.py,sha256=gughdCUxseDsu5MqMepG2UKs-0kBiHP9On_mpcQL-W8,2791
+agentic_evals/gate.py,sha256=0w0_uLILEeDn9GIsQ-xyav3TvNX5WoYlB7i34SzPit8,2384
+agentic_evals/models.py,sha256=kh8ukzCuE9FAX9gEktOD4heicF0b3DdzCqDfBJ-rCMc,4700
+agentic_evals/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+agentic_evals/runner.py,sha256=8z8la58EPw4EJvg-1yPIYMZdznicSdYU_uvomoJYJbE,15710
+agentic_evals-0.1.0.dist-info/METADATA,sha256=irdrO4xK855W-vMz-CtrMWIAryDORuPOn558hsSChU8,8627
+agentic_evals-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+agentic_evals-0.1.0.dist-info/licenses/LICENSE,sha256=MDZ9UDelXt06pT2XN55Fz1XHj4Tb4whEhX8hjBRnySg,1070
+agentic_evals-0.1.0.dist-info/RECORD,,

agentic_evals-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

agentic_evals-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 DeepAgentLabs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.