agentic-evals 0.1.0__tar.gz

agentic_evals-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,56 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --extra dev --python ${{ matrix.python-version }}
+
+      - name: Lint (ruff check)
+        run: uv run ruff check .
+
+      - name: Format check (ruff format)
+        run: uv run ruff format --check .
+
+      - name: Type check (mypy)
+        run: uv run mypy
+
+      - name: Test (pytest)
+        run: uv run pytest
+
+  package:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+      - name: Set up Python
+        run: uv python install 3.12
+      - name: Install dependencies
+        run: uv sync --extra dev --python 3.12
+      - name: Build distributions
+        run: |
+          uv run python -m build
+          uv run python -m twine check dist/*

agentic_evals-0.1.0/.github/workflows/release-pypi.yml

@@ -0,0 +1,91 @@
+name: publish-pypi
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+      - name: Build distributions
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install build twine
+          python -m build
+          python -m twine check dist/*
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+  publish-pypi:
+    runs-on: ubuntu-latest
+    needs: build
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - name: Download distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  create-github-release:
+    runs-on: ubuntu-latest
+    needs: publish-pypi
+    steps:
+      - uses: actions/checkout@v4
+      - name: Extract changelog section
+        env:
+          RELEASE_TAG: ${{ github.ref_name }}
+        run: |
+          python - <<'PY'
+          from pathlib import Path
+          import os
+
+          tag = os.environ["RELEASE_TAG"]
+          version = tag.removeprefix("v")
+          changelog = Path("CHANGELOG.md").read_text(encoding="utf-8").splitlines()
+
+          start = None
+          end = None
+          heading = f"## {version} - "
+          for index, line in enumerate(changelog):
+              if line.startswith(heading):
+                  start = index
+                  continue
+              if start is not None and line.startswith("## "):
+                  end = index
+                  break
+
+          if start is None:
+              raise SystemExit(
+                  f"Could not find CHANGELOG.md section for version {version}."
+              )
+
+          if end is None:
+              end = len(changelog)
+
+          section = "\n".join(changelog[start:end]).strip() + "\n"
+          Path("release-notes.md").write_text(section, encoding="utf-8")
+          PY
+      - name: Create GitHub release
+        uses: softprops/action-gh-release@v3
+        with:
+          tag_name: ${{ github.ref_name }}
+          name: ${{ github.ref_name }}
+          body_path: release-notes.md
+          generate_release_notes: false

agentic_evals-0.1.0/.github/workflows/release-testpypi.yml

@@ -0,0 +1,46 @@
+name: publish-testpypi
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+      - name: Build distributions
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install build twine
+          python -m build
+          python -m twine check dist/*
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+  publish-testpypi:
+    runs-on: ubuntu-latest
+    needs: build
+    environment: testpypi
+    permissions:
+      id-token: write
+    steps:
+      - name: Download distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+      - name: Publish to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          skip-existing: true

agentic_evals-0.1.0/.gitignore

@@ -0,0 +1,13 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+.venv/
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+htmlcov/
+.coverage
+dist/
+build/
+.env

agentic_evals-0.1.0/CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+
+All notable changes to this project will be documented here.
+
+This project follows [Semantic Versioning](https://semver.org/).
+
+## 0.1.0 - 2026-09-13
+
+### Added
+
+- Initial extraction of AgenticLens's evaluation engine into a standalone,
+  framework-agnostic package: `EvalTrace`/`EvalSpan` (a minimal, tool-agnostic
+  trace shape), `Score`, `Evaluator`/`EvaluatorRegistry`/`CallableEvaluator`/
+  `LLMJudgeEvaluator`/`BusinessRuleEvaluator`, `TestCase`/`TestSuite`,
+  `evaluate_suite` (deterministic JSON Schema/field/tool/latency/cost/
+  turn-count checks plus custom evaluators), `run_live_suite` (Python/HTTP
+  live targets), and `GateConfig`/`evaluate_gate`.
+- Zero dependency on AgenticLens or any other DeepAgentLabs project —
+  depends only on `pydantic`, `pyyaml`, `jsonschema`, and `referencing`.

agentic_evals-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 DeepAgentLabs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

agentic_evals-0.1.0/PKG-INFO

@@ -0,0 +1,237 @@
+Metadata-Version: 2.5
+Name: agentic-evals
+Version: 0.1.0
+Summary: A standalone, framework-agnostic evaluation and scoring engine for LLM/agent outputs — extracted from AgenticLens's evaluation module.
+Project-URL: Homepage, https://github.com/DeepAgentLabs/agentic-evals
+Project-URL: Repository, https://github.com/DeepAgentLabs/agentic-evals
+Project-URL: Issues, https://github.com/DeepAgentLabs/agentic-evals/issues
+Project-URL: Changelog, https://github.com/DeepAgentLabs/agentic-evals/blob/main/CHANGELOG.md
+Author: agentic-evals Contributors
+License: MIT License
+        
+        Copyright (c) 2026 DeepAgentLabs
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: agents,evaluation,llm,llm-as-judge,scoring,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Requires-Dist: jsonschema>=4.23
+Requires-Dist: pydantic<3,>=2.0
+Requires-Dist: pyyaml<7,>=6.0
+Requires-Dist: referencing>=0.35
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Requires-Dist: twine>=5.1; extra == 'dev'
+Requires-Dist: types-pyyaml>=6.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# agentic-evals
+
+**A standalone, framework-agnostic evaluation and scoring engine for LLM and
+agent outputs.**
+
+Extracted from [AgenticLens](https://github.com/DeepAgentLabs/agenticlens)'s
+proven evaluation module — the same engine, usable on its own. It scores
+whatever trace-shaped data you give it (see `EvalTrace`/`EvalSpan` below);
+it has no dependency on any specific tracing/observability tool, and no
+dependency on AgenticLens itself.
+
+## Status
+
+**Alpha.** The engine (deterministic checks, LLM-as-judge and custom
+evaluators, a release gate, live Python/HTTP targets) is real, tested code
+lifted directly from AgenticLens's evaluation module. No PyPI release yet.
+
+## Why a separate package
+
+AgenticLens's evaluation module was never actually AgenticLens-specific —
+scoring an LLM/agent output against expectations doesn't need AgenticLens's
+full trace schema, CLI, or dashboards. Pulling it out means:
+
+- `agentic-sidecar`, `agentic-chaos`, or any other project can score outputs
+  without depending on all of AgenticLens.
+- Anyone with *any* trace-shaped data — not just AgenticLens users — can use
+  it, the same way Braintrust's `autoevals` doesn't care what produced the
+  string it's scoring.
+
+## Install
+
+```bash
+pip install agentic-evals
+```
+
+## Core concepts
+
+- **`EvalTrace`/`EvalSpan`** — the minimal trace shape the engine inspects:
+  a trace id, spans (each optionally naming a `tool_name` and carrying
+  arbitrary `attributes`), total latency, estimated cost, and metadata.
+  Deliberately not tied to any specific instrumentation format — build one
+  from whatever you already have.
+- **`Score`** — a single named judgment (0-1 value, pass/fail, explanation).
+- **`Evaluator`** — anything with a `.name` and an `.evaluate(context) ->
+  list[Score]`. `CallableEvaluator` adapts a plain Python function;
+  `LLMJudgeEvaluator` and `BusinessRuleEvaluator` are named convenience
+  subclasses for readability/reporting.
+- **`TestCase`/`TestSuite`** — declarative expectations (exact match,
+  substring, JSON Schema, required fields, required/forbidden tool calls,
+  required tool arguments, latency/cost/turn-count thresholds, or a named
+  custom evaluator) plus the cases that make up a suite.
+- **`evaluate_suite`** — runs a suite against supplied `EvaluationSample`s
+  and returns an `EvaluationReport` (per-case scores plus a pass-rate/cost/
+  latency summary).
+- **`GateConfig`/`evaluate_gate`** — turn an `EvaluationReport` into a
+  pass/fail release decision on configurable thresholds.
+
+## Quickstart
+
+```python
+from agentic_evals import (
+    EvalSpan,
+    EvalTrace,
+    EvaluationSample,
+    TestCase,
+    TestSuite,
+    evaluate_suite,
+)
+
+suite = TestSuite(
+    name="support-answers",
+    version="1",
+    cases=[
+        TestCase(
+            id="case-1",
+            name="Answer contains the right total",
+            expected_contains=["42"],
+            required_tools=["calculator"],
+            max_latency_ms=2000,
+        )
+    ],
+)
+
+sample = EvaluationSample(
+    case_id="case-1",
+    output="The combined total is 42.",
+    trace=EvalTrace(
+        trace_id="trace-1",
+        total_latency_ms=350,
+        spans=[EvalSpan(tool_name="calculator")],
+    ),
+)
+
+report = evaluate_suite(suite, [sample])
+print(report.summary.pass_rate)  # 1.0
+```
+
+## LLM-as-judge
+
+```python
+from agentic_evals import (
+    EvaluationContext,
+    EvaluatorConfig,
+    EvaluatorRegistry,
+    LLMJudgeEvaluator,
+    Score,
+    TestCase,
+)
+
+
+def judge(context: EvaluationContext) -> Score:
+    # Call whatever model/provider you like here.
+    correct = "42" in context.sample.output
+    return Score(
+        name="answer_quality",
+        value=0.95 if correct else 0.1,
+        passed=correct,
+        explanation="Judged against the rubric in context.config.config.",
+    )
+
+
+registry = EvaluatorRegistry()
+registry.register(LLMJudgeEvaluator("answer_quality_judge", judge))
+
+case = TestCase(
+    id="case-1",
+    name="Answer quality",
+    evaluators=[EvaluatorConfig(name="answer_quality_judge", threshold=0.8)],
+)
+```
+
+## Release gates
+
+```python
+from agentic_evals import GateConfig, evaluate_gate
+
+decision = evaluate_gate(
+    report,
+    GateConfig(min_pass_rate=0.95, max_average_latency_ms=1500, max_total_cost_usd=0.25),
+)
+if not decision.passed:
+    raise SystemExit(f"Release gate failed: {decision.reasons}")
+```
+
+Never fabricates a value it can't back up: `total_cost_usd` on a summary or
+gate decision stays `None` unless every case in scope has a known cost —
+an incomplete cost picture is reported as unavailable, not `$0.00`.
+
+## Live targets
+
+Point a suite at a real running system (a trusted Python callable, or an
+HTTP endpoint) instead of pre-recorded samples:
+
+```python
+from agentic_evals import PythonTarget, run_live_suite
+
+report = run_live_suite(suite, PythonTarget(callable_path="my_module:run_case"))
+```
+
+Live targets are intentionally powerful developer-facing integrations —
+Python targets execute local code and HTTP targets can reach arbitrary
+URLs. Only point them at trusted suite files and trusted target
+definitions.
+
+## Using it with AgenticLens's own traces
+
+If you already have an AgenticLens `Run` (from its instrumentation API or
+OTLP ingestion), AgenticLens itself provides the adapter —
+`agenticlens.evaluation.to_eval_trace(run)` — so you don't have to hand-build
+an `EvalTrace`. This package has no dependency in the other direction.
+
+## What's deliberately not here
+
+Dataset versioning/splitting, judge calibration, and HTML report rendering
+stay in AgenticLens for now — those are product features built *on top of*
+this engine (the same way Braintrust's dataset/experiment platform is
+separate from the `autoevals` library itself), not the engine.
+
+## License
+
+MIT

agentic_evals-0.1.0/README.md

@@ -0,0 +1,181 @@
+# agentic-evals
+
+**A standalone, framework-agnostic evaluation and scoring engine for LLM and
+agent outputs.**
+
+Extracted from [AgenticLens](https://github.com/DeepAgentLabs/agenticlens)'s
+proven evaluation module — the same engine, usable on its own. It scores
+whatever trace-shaped data you give it (see `EvalTrace`/`EvalSpan` below);
+it has no dependency on any specific tracing/observability tool, and no
+dependency on AgenticLens itself.
+
+## Status
+
+**Alpha.** The engine (deterministic checks, LLM-as-judge and custom
+evaluators, a release gate, live Python/HTTP targets) is real, tested code
+lifted directly from AgenticLens's evaluation module. No PyPI release yet.
+
+## Why a separate package
+
+AgenticLens's evaluation module was never actually AgenticLens-specific —
+scoring an LLM/agent output against expectations doesn't need AgenticLens's
+full trace schema, CLI, or dashboards. Pulling it out means:
+
+- `agentic-sidecar`, `agentic-chaos`, or any other project can score outputs
+  without depending on all of AgenticLens.
+- Anyone with *any* trace-shaped data — not just AgenticLens users — can use
+  it, the same way Braintrust's `autoevals` doesn't care what produced the
+  string it's scoring.
+
+## Install
+
+```bash
+pip install agentic-evals
+```
+
+## Core concepts
+
+- **`EvalTrace`/`EvalSpan`** — the minimal trace shape the engine inspects:
+  a trace id, spans (each optionally naming a `tool_name` and carrying
+  arbitrary `attributes`), total latency, estimated cost, and metadata.
+  Deliberately not tied to any specific instrumentation format — build one
+  from whatever you already have.
+- **`Score`** — a single named judgment (0-1 value, pass/fail, explanation).
+- **`Evaluator`** — anything with a `.name` and an `.evaluate(context) ->
+  list[Score]`. `CallableEvaluator` adapts a plain Python function;
+  `LLMJudgeEvaluator` and `BusinessRuleEvaluator` are named convenience
+  subclasses for readability/reporting.
+- **`TestCase`/`TestSuite`** — declarative expectations (exact match,
+  substring, JSON Schema, required fields, required/forbidden tool calls,
+  required tool arguments, latency/cost/turn-count thresholds, or a named
+  custom evaluator) plus the cases that make up a suite.
+- **`evaluate_suite`** — runs a suite against supplied `EvaluationSample`s
+  and returns an `EvaluationReport` (per-case scores plus a pass-rate/cost/
+  latency summary).
+- **`GateConfig`/`evaluate_gate`** — turn an `EvaluationReport` into a
+  pass/fail release decision on configurable thresholds.
+
+## Quickstart
+
+```python
+from agentic_evals import (
+    EvalSpan,
+    EvalTrace,
+    EvaluationSample,
+    TestCase,
+    TestSuite,
+    evaluate_suite,
+)
+
+suite = TestSuite(
+    name="support-answers",
+    version="1",
+    cases=[
+        TestCase(
+            id="case-1",
+            name="Answer contains the right total",
+            expected_contains=["42"],
+            required_tools=["calculator"],
+            max_latency_ms=2000,
+        )
+    ],
+)
+
+sample = EvaluationSample(
+    case_id="case-1",
+    output="The combined total is 42.",
+    trace=EvalTrace(
+        trace_id="trace-1",
+        total_latency_ms=350,
+        spans=[EvalSpan(tool_name="calculator")],
+    ),
+)
+
+report = evaluate_suite(suite, [sample])
+print(report.summary.pass_rate)  # 1.0
+```
+
+## LLM-as-judge
+
+```python
+from agentic_evals import (
+    EvaluationContext,
+    EvaluatorConfig,
+    EvaluatorRegistry,
+    LLMJudgeEvaluator,
+    Score,
+    TestCase,
+)
+
+
+def judge(context: EvaluationContext) -> Score:
+    # Call whatever model/provider you like here.
+    correct = "42" in context.sample.output
+    return Score(
+        name="answer_quality",
+        value=0.95 if correct else 0.1,
+        passed=correct,
+        explanation="Judged against the rubric in context.config.config.",
+    )
+
+
+registry = EvaluatorRegistry()
+registry.register(LLMJudgeEvaluator("answer_quality_judge", judge))
+
+case = TestCase(
+    id="case-1",
+    name="Answer quality",
+    evaluators=[EvaluatorConfig(name="answer_quality_judge", threshold=0.8)],
+)
+```
+
+## Release gates
+
+```python
+from agentic_evals import GateConfig, evaluate_gate
+
+decision = evaluate_gate(
+    report,
+    GateConfig(min_pass_rate=0.95, max_average_latency_ms=1500, max_total_cost_usd=0.25),
+)
+if not decision.passed:
+    raise SystemExit(f"Release gate failed: {decision.reasons}")
+```
+
+Never fabricates a value it can't back up: `total_cost_usd` on a summary or
+gate decision stays `None` unless every case in scope has a known cost —
+an incomplete cost picture is reported as unavailable, not `$0.00`.
+
+## Live targets
+
+Point a suite at a real running system (a trusted Python callable, or an
+HTTP endpoint) instead of pre-recorded samples:
+
+```python
+from agentic_evals import PythonTarget, run_live_suite
+
+report = run_live_suite(suite, PythonTarget(callable_path="my_module:run_case"))
+```
+
+Live targets are intentionally powerful developer-facing integrations —
+Python targets execute local code and HTTP targets can reach arbitrary
+URLs. Only point them at trusted suite files and trusted target
+definitions.
+
+## Using it with AgenticLens's own traces
+
+If you already have an AgenticLens `Run` (from its instrumentation API or
+OTLP ingestion), AgenticLens itself provides the adapter —
+`agenticlens.evaluation.to_eval_trace(run)` — so you don't have to hand-build
+an `EvalTrace`. This package has no dependency in the other direction.
+
+## What's deliberately not here
+
+Dataset versioning/splitting, judge calibration, and HTML report rendering
+stay in AgenticLens for now — those are product features built *on top of*
+this engine (the same way Braintrust's dataset/experiment platform is
+separate from the `autoevals` library itself), not the engine.
+
+## License
+
+MIT