evaldeck 0.1.1__tar.gz → 0.1.3__tar.gz

{evaldeck-0.1.1 → evaldeck-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evaldeck
-Version: 0.1.1
+Version: 0.1.3
 Summary: The evaluation framework for AI agents. Pytest for agents.
 Project-URL: Homepage, https://github.com/tantra-run/evaldeck-py
 Project-URL: Documentation, https://tantra-run.github.io/evaldeck-py/
@@ -29,6 +29,7 @@ Requires-Dist: rich>=13.0
 Provides-Extra: all
 Requires-Dist: anthropic>=0.18; extra == 'all'
 Requires-Dist: openai>=1.0; extra == 'all'
+Requires-Dist: openinference-instrumentation-langchain>=0.1; extra == 'all'
 Provides-Extra: anthropic
 Requires-Dist: anthropic>=0.18; extra == 'anthropic'
 Provides-Extra: dev
@@ -45,6 +46,8 @@ Requires-Dist: mkdocs-autorefs>=0.5; extra == 'docs'
 Requires-Dist: mkdocs-material>=9.5; extra == 'docs'
 Requires-Dist: mkdocs>=1.5; extra == 'docs'
 Requires-Dist: mkdocstrings[python]>=0.24; extra == 'docs'
+Provides-Extra: langchain
+Requires-Dist: openinference-instrumentation-langchain>=0.1; extra == 'langchain'
 Provides-Extra: openai
 Requires-Dist: openai>=1.0; extra == 'openai'
 Description-Content-Type: text/markdown

{evaldeck-0.1.1 → evaldeck-0.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "evaldeck"
-version = "0.1.1"
+version = "0.1.3"
 description = "The evaluation framework for AI agents. Pytest for agents."
 readme = "README.md"
 license = "Apache-2.0"
@@ -43,7 +43,8 @@ dependencies = [
 [project.optional-dependencies]
 openai = ["openai>=1.0"]
 anthropic = ["anthropic>=0.18"]
-all = ["evaldeck[openai,anthropic]"]
+langchain = ["openinference-instrumentation-langchain>=0.1"]
+all = ["evaldeck[openai,anthropic,langchain]"]
 dev = [
     "pytest>=7.0",
     "pytest-asyncio>=0.21",
@@ -73,6 +74,18 @@ Issues = "https://github.com/tantra-run/evaldeck-py/issues"
 [tool.hatch.build.targets.wheel]
 packages = ["src/evaldeck"]
 
+[tool.hatch.build.targets.sdist]
+exclude = [
+    "tests/",
+    "docs/",
+    ".github/",
+    ".devcontainer/",
+    ".evaldeck/",
+    "internal/",
+    "examples/",
+    ".claude/",
+]
+
 [tool.ruff]
 target-version = "py310"
 line-length = 100

{evaldeck-0.1.1 → evaldeck-0.1.3}/src/evaldeck/config.py

@@ -15,6 +15,7 @@ class AgentConfig(BaseModel):
     module: str | None = None
     function: str | None = None
     class_name: str | None = None
+    framework: str | None = None  # "langchain", "crewai", etc.
 
 
 class GraderDefaults(BaseModel):

{evaldeck-0.1.1 → evaldeck-0.1.3}/src/evaldeck/evaluator.py

@@ -571,4 +571,17 @@ class EvaluationRunner:
 
         module = importlib.import_module(agent_config.module)
         func = getattr(module, agent_config.function)
+
+        # Handle framework-specific integration
+        if agent_config.framework:
+            framework = agent_config.framework.lower()
+
+            if framework == "langchain":
+                from evaldeck.integrations.langchain import create_langchain_runner
+
+                return create_langchain_runner(func)
+
+            else:
+                raise ValueError(f"Unknown framework: {agent_config.framework}")
+
         return func

{evaldeck-0.1.1 → evaldeck-0.1.3}/src/evaldeck/integrations/__init__.py

@@ -3,7 +3,7 @@
 This module provides the OpenTelemetry/OpenInference adapter for capturing traces
 from any instrumented AI framework (LangChain, CrewAI, LiteLLM, OpenAI, Anthropic, etc.)
 
-Usage:
+Basic usage (manual setup):
     from evaldeck.integrations import EvaldeckSpanProcessor, setup_otel_tracing
     from openinference.instrumentation.langchain import LangChainInstrumentor
 
@@ -14,6 +14,19 @@ Usage:
 
     trace = processor.get_latest_trace()
     result = evaluator.evaluate(trace, test_case)
+
+With framework integration (automatic setup via evaldeck.yaml):
+    # evaldeck.yaml
+    agent:
+      module: my_agent
+      function: create_agent
+      framework: langchain
+
+    # my_agent.py
+    def create_agent():
+        return create_react_agent(llm, tools)
+
+    # Run: evaldeck run
 """
 
 from evaldeck.integrations.opentelemetry import (

evaldeck-0.1.3/src/evaldeck/integrations/langchain.py

@@ -0,0 +1,125 @@
+"""LangChain integration for evaldeck.
+
+Provides automatic instrumentation and trace capture for LangChain/LangGraph agents.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Callable
+
+if TYPE_CHECKING:
+    from evaldeck.trace import Trace
+
+
+class LangChainIntegration:
+    """LangChain/LangGraph integration.
+
+    Automatically sets up OpenTelemetry tracing and provides a wrapper
+    that invokes the agent and returns a Trace.
+    """
+
+    def __init__(self) -> None:
+        self._processor: Any = None
+        self._agent: Any = None
+        self._initialized = False
+
+    def setup(self, agent_factory: Callable[[], Any]) -> None:
+        """Set up instrumentation and create the agent.
+
+        Args:
+            agent_factory: Function that returns the agent instance.
+        """
+        if self._initialized:
+            return
+
+        # Import here to make langchain an optional dependency
+        try:
+            from openinference.instrumentation.langchain import LangChainInstrumentor
+        except ImportError as e:
+            raise ImportError(
+                "LangChain integration requires openinference-instrumentation-langchain. "
+                "Install with: pip install evaldeck[langchain]"
+            ) from e
+
+        from evaldeck.integrations import setup_otel_tracing
+
+        # Set up OTel tracing
+        self._processor = setup_otel_tracing()
+
+        # Instrument LangChain
+        LangChainInstrumentor().instrument()
+
+        # Create the agent
+        self._agent = agent_factory()
+        self._initialized = True
+
+    def run(self, input: str) -> Trace:
+        """Run the agent and return a trace.
+
+        Args:
+            input: The input string to send to the agent.
+
+        Returns:
+            Trace captured from the agent execution.
+        """
+        if not self._initialized:
+            raise RuntimeError("Integration not initialized. Call setup() first.")
+
+        # Reset processor for fresh trace
+        self._processor.reset()
+
+        # Invoke the agent - auto-detect format
+        self._invoke_agent(input)
+
+        # Get and return trace
+        trace = self._processor.get_latest_trace()
+        if trace is None:
+            raise RuntimeError("No trace captured from agent execution")
+
+        return trace
+
+    def _invoke_agent(self, input: str) -> Any:
+        """Invoke the agent with the appropriate format.
+
+        Auto-detects LangGraph vs legacy LangChain format.
+        """
+        # LangGraph style (current)
+        if hasattr(self._agent, "invoke"):
+            # Try LangGraph message format first
+            try:
+                return self._agent.invoke({"messages": [("human", input)]})
+            except (TypeError, KeyError):
+                # Fall back to simple input
+                try:
+                    return self._agent.invoke({"input": input})
+                except (TypeError, KeyError):
+                    return self._agent.invoke(input)
+
+        # Legacy LangChain style
+        if hasattr(self._agent, "run"):
+            return self._agent.run(input)
+
+        # Callable
+        if callable(self._agent):
+            return self._agent(input)
+
+        raise RuntimeError(
+            f"Don't know how to invoke agent of type {type(self._agent)}. "
+            "Agent must have invoke(), run(), or be callable."
+        )
+
+
+def create_langchain_runner(agent_factory: Callable[[], Any]) -> Callable[[str], Trace]:
+    """Create a runner function for LangChain agents.
+
+    This is the main entry point used by evaldeck's EvaluationRunner.
+
+    Args:
+        agent_factory: Function that returns the agent instance.
+
+    Returns:
+        A function that takes input and returns a Trace.
+    """
+    integration = LangChainIntegration()
+    integration.setup(agent_factory)
+    return integration.run

evaldeck-0.1.1/.claude/settings.local.json

@@ -1,34 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(python -m pytest:*)",
-      "Bash(grep:*)",
-      "Bash(python:*)",
-      "Bash(python3:*)",
-      "Bash(PYTHONPATH=src python3:*)",
-      "Bash(PYTHONPATH=src /opt/homebrew/bin/python3.11:*)",
-      "Bash(wc:*)",
-      "WebFetch(domain:docs.langchain.com)",
-      "WebSearch",
-      "WebFetch(domain:arize-ai.github.io)",
-      "WebFetch(domain:docs.arize.com)",
-      "WebFetch(domain:arize.com)",
-      "WebFetch(domain:github.com)",
-      "Bash(ls:*)",
-      "Bash(find:*)",
-      "Bash(pip install:*)",
-      "Bash(pip3 install:*)",
-      "Bash(mkdocs serve --help:*)",
-      "Bash(git init:*)",
-      "Bash(git add:*)",
-      "Bash(git commit:*)",
-      "Bash(git branch:*)",
-      "Bash(git remote add:*)",
-      "Bash(git push:*)",
-      "Bash(ruff check:*)",
-      "Bash(source:*)",
-      "Bash(./venv/bin/python -m pytest:*)",
-      "Bash(uv run pytest:*)"
-    ]
-  }
-}

evaldeck-0.1.1/.devcontainer/Dockerfile

@@ -1,30 +0,0 @@
-FROM mcr.microsoft.com/devcontainers/python:1-3.11-bullseye
-
-# Remove yarn repo (has expired GPG key) and install system dependencies
-RUN rm -f /etc/apt/sources.list.d/yarn.list && \
-    apt-get update && apt-get install -y --no-install-recommends \
-    build-essential \
-    curl \
-    && rm -rf /var/lib/apt/lists/*
-
-# Set working directory
-WORKDIR /workspaces/evaldeck
-
-# Upgrade pip
-RUN pip install --upgrade pip
-
-# Install Python development tools
-RUN pip install --no-cache-dir \
-    ruff \
-    mypy \
-    pytest \
-    pytest-asyncio \
-    pre-commit \
-    build \
-    twine
-
-# Create directory for evaldeck output
-RUN mkdir -p /workspaces/evaldeck/.evaldeck
-
-# Set Python path
-ENV PYTHONPATH="/workspaces/evaldeck/src:${PYTHONPATH}"

evaldeck-0.1.1/.devcontainer/devcontainer.json

@@ -1,60 +0,0 @@
-{
-  "name": "Evaldeck Development",
-  "build": {
-    "dockerfile": "Dockerfile",
-    "context": ".."
-  },
-  "features": {
-    "ghcr.io/devcontainers/features/git:1": {},
-    "ghcr.io/devcontainers/features/github-cli:1": {}
-  },
-  "customizations": {
-    "vscode": {
-      "extensions": [
-        "ms-python.python",
-        "ms-python.vscode-pylance",
-        "charliermarsh.ruff",
-        "tamasfe.even-better-toml",
-        "redhat.vscode-yaml",
-        "github.copilot"
-      ],
-      "settings": {
-        "python.defaultInterpreterPath": "/usr/local/bin/python",
-        "python.testing.pytestEnabled": true,
-        "python.testing.pytestArgs": [
-          "tests"
-        ],
-        "editor.formatOnSave": true,
-        "editor.codeActionsOnSave": {
-          "source.organizeImports": "explicit",
-          "source.fixAll": "explicit"
-        },
-        "[python]": {
-          "editor.defaultFormatter": "charliermarsh.ruff"
-        },
-        "files.exclude": {
-          "**/__pycache__": true,
-          "**/*.pyc": true,
-          "**/.pytest_cache": true,
-          "**/.mypy_cache": true,
-          "**/.ruff_cache": true,
-          "**/*.egg-info": true
-        }
-      }
-    }
-  },
-  "postCreateCommand": "pip install -e '.[dev,all,docs]' && (git rev-parse --git-dir > /dev/null 2>&1 && pre-commit install || echo 'Skipping pre-commit install (not a git repo)')",
-  "runArgs": [
-    "--env-file",
-    "${localWorkspaceFolder}/.env"
-  ],
-  "forwardPorts": [],
-  "remoteUser": "vscode",
-  "mounts": [
-    "source=${localWorkspaceFolder}/.evaldeck,target=/workspaces/evaldeck/.evaldeck,type=bind,consistency=cached"
-  ],
-  "remoteEnv": {
-    "PYTHONDONTWRITEBYTECODE": "1",
-    "PYTHONUNBUFFERED": "1"
-  }
-}

evaldeck-0.1.1/.github/workflows/ci.yaml

@@ -1,65 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.10", "3.11", "3.12"]
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e ".[dev]"
-
-      - name: Lint with ruff
-        run: |
-          ruff check src/ tests/
-
-      - name: Type check with mypy
-        run: |
-          mypy src/
-
-      - name: Run tests
-        run: |
-          pytest tests/ -v --tb=short
-
-  build:
-    runs-on: ubuntu-latest
-    needs: test
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-
-      - name: Install build dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install build
-
-      - name: Build package
-        run: |
-          python -m build
-
-      - name: Check package
-        run: |
-          pip install twine
-          twine check dist/*

evaldeck-0.1.1/.github/workflows/docs.yaml

@@ -1,50 +0,0 @@
-name: Deploy Docs
-
-on:
-  push:
-    branches:
-      - main
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-concurrency:
-  group: "pages"
-  cancel-in-progress: false
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-
-      - name: Install dependencies
-        run: |
-          pip install -e ".[docs]"
-
-      - name: Build docs
-        run: mkdocs build
-
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
-        with:
-          path: site/
-
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4

evaldeck-0.1.1/.github/workflows/publish.yaml

@@ -1,28 +0,0 @@
-name: Publish to PyPI
-
-on:
-  release:
-    types: [published]
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    environment: pypi
-    permissions:
-      id-token: write  # Required for trusted publishing
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-
-      - name: Install build tools
-        run: pip install build
-
-      - name: Build package
-        run: python -m build
-
-      - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1

evaldeck-0.1.1/docs/api/config.md

@@ -1,7 +0,0 @@
-# Configuration
-
-::: evaldeck.config.EvaldeckConfig
-    options:
-      show_root_heading: true
-      show_source: true
-      members_order: source

evaldeck-0.1.1/docs/api/evalcase.md

@@ -1,29 +0,0 @@
-# Test Case Models
-
-::: evaldeck.test_case.EvalCase
-    options:
-      show_root_heading: true
-      show_source: true
-      members_order: source
-
----
-
-::: evaldeck.test_case.ExpectedBehavior
-    options:
-      show_root_heading: true
-      show_source: true
-
----
-
-::: evaldeck.test_case.EvalSuite
-    options:
-      show_root_heading: true
-      show_source: true
-      members_order: source
-
----
-
-::: evaldeck.test_case.GraderConfig
-    options:
-      show_root_heading: true
-      show_source: true

evaldeck-0.1.1/docs/api/evaluation-result.md

@@ -1,23 +0,0 @@
-# Evaluation Results
-
-::: evaldeck.results.EvaluationResult
-    options:
-      show_root_heading: true
-      show_source: true
-      members_order: source
-
----
-
-::: evaldeck.results.SuiteResult
-    options:
-      show_root_heading: true
-      show_source: true
-      members_order: source
-
----
-
-::: evaldeck.results.RunResult
-    options:
-      show_root_heading: true
-      show_source: true
-      members_order: source

evaldeck-0.1.1/docs/api/evaluator.md

@@ -1,15 +0,0 @@
-# Evaluator
-
-::: evaldeck.evaluator.Evaluator
-    options:
-      show_root_heading: true
-      show_source: true
-      members_order: source
-
----
-
-::: evaldeck.evaluator.EvaluationRunner
-    options:
-      show_root_heading: true
-      show_source: true
-      members_order: source

evaldeck-0.1.1/docs/api/grade-result.md

@@ -1,20 +0,0 @@
-# Results
-
-::: evaldeck.results.GradeResult
-    options:
-      show_root_heading: true
-      show_source: true
-      members_order: source
-
----
-
-::: evaldeck.results.GradeStatus
-    options:
-      show_root_heading: true
-
----
-
-::: evaldeck.results.MetricResult
-    options:
-      show_root_heading: true
-      show_source: true

evaldeck-0.1.1/docs/api/graders/base.md

@@ -1,15 +0,0 @@
-# Base Grader
-
-::: evaldeck.graders.BaseGrader
-    options:
-      show_root_heading: true
-      show_source: true
-      members_order: source
-
----
-
-::: evaldeck.graders.CompositeGrader
-    options:
-      show_root_heading: true
-      show_source: true
-      members_order: source