evaldeck 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

evaldeck/config.py

@@ -15,6 +15,7 @@ class AgentConfig(BaseModel):
     module: str | None = None
     function: str | None = None
     class_name: str | None = None
+    framework: str | None = None  # "langchain", "crewai", etc.
 
 
 class GraderDefaults(BaseModel):

evaldeck/evaluator.py

@@ -571,4 +571,17 @@ class EvaluationRunner:
 
         module = importlib.import_module(agent_config.module)
         func = getattr(module, agent_config.function)
+
+        # Handle framework-specific integration
+        if agent_config.framework:
+            framework = agent_config.framework.lower()
+
+            if framework == "langchain":
+                from evaldeck.integrations.langchain import create_langchain_runner
+
+                return create_langchain_runner(func)
+
+            else:
+                raise ValueError(f"Unknown framework: {agent_config.framework}")
+
         return func

evaldeck/integrations/__init__.py

@@ -3,7 +3,7 @@
 This module provides the OpenTelemetry/OpenInference adapter for capturing traces
 from any instrumented AI framework (LangChain, CrewAI, LiteLLM, OpenAI, Anthropic, etc.)
 
-Usage:
+Basic usage (manual setup):
     from evaldeck.integrations import EvaldeckSpanProcessor, setup_otel_tracing
     from openinference.instrumentation.langchain import LangChainInstrumentor
 
@@ -14,6 +14,19 @@ Usage:
 
     trace = processor.get_latest_trace()
     result = evaluator.evaluate(trace, test_case)
+
+With framework integration (automatic setup via evaldeck.yaml):
+    # evaldeck.yaml
+    agent:
+      module: my_agent
+      function: create_agent
+      framework: langchain
+
+    # my_agent.py
+    def create_agent():
+        return create_react_agent(llm, tools)
+
+    # Run: evaldeck run
 """
 
 from evaldeck.integrations.opentelemetry import (

evaldeck/integrations/langchain.py

@@ -0,0 +1,125 @@
+"""LangChain integration for evaldeck.
+
+Provides automatic instrumentation and trace capture for LangChain/LangGraph agents.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Callable
+
+if TYPE_CHECKING:
+    from evaldeck.trace import Trace
+
+
+class LangChainIntegration:
+    """LangChain/LangGraph integration.
+
+    Automatically sets up OpenTelemetry tracing and provides a wrapper
+    that invokes the agent and returns a Trace.
+    """
+
+    def __init__(self) -> None:
+        self._processor: Any = None
+        self._agent: Any = None
+        self._initialized = False
+
+    def setup(self, agent_factory: Callable[[], Any]) -> None:
+        """Set up instrumentation and create the agent.
+
+        Args:
+            agent_factory: Function that returns the agent instance.
+        """
+        if self._initialized:
+            return
+
+        # Import here to make langchain an optional dependency
+        try:
+            from openinference.instrumentation.langchain import LangChainInstrumentor
+        except ImportError as e:
+            raise ImportError(
+                "LangChain integration requires openinference-instrumentation-langchain. "
+                "Install with: pip install evaldeck[langchain]"
+            ) from e
+
+        from evaldeck.integrations import setup_otel_tracing
+
+        # Set up OTel tracing
+        self._processor = setup_otel_tracing()
+
+        # Instrument LangChain
+        LangChainInstrumentor().instrument()
+
+        # Create the agent
+        self._agent = agent_factory()
+        self._initialized = True
+
+    def run(self, input: str) -> Trace:
+        """Run the agent and return a trace.
+
+        Args:
+            input: The input string to send to the agent.
+
+        Returns:
+            Trace captured from the agent execution.
+        """
+        if not self._initialized:
+            raise RuntimeError("Integration not initialized. Call setup() first.")
+
+        # Reset processor for fresh trace
+        self._processor.reset()
+
+        # Invoke the agent - auto-detect format
+        self._invoke_agent(input)
+
+        # Get and return trace
+        trace = self._processor.get_latest_trace()
+        if trace is None:
+            raise RuntimeError("No trace captured from agent execution")
+
+        return trace
+
+    def _invoke_agent(self, input: str) -> Any:
+        """Invoke the agent with the appropriate format.
+
+        Auto-detects LangGraph vs legacy LangChain format.
+        """
+        # LangGraph style (current)
+        if hasattr(self._agent, "invoke"):
+            # Try LangGraph message format first
+            try:
+                return self._agent.invoke({"messages": [("human", input)]})
+            except (TypeError, KeyError):
+                # Fall back to simple input
+                try:
+                    return self._agent.invoke({"input": input})
+                except (TypeError, KeyError):
+                    return self._agent.invoke(input)
+
+        # Legacy LangChain style
+        if hasattr(self._agent, "run"):
+            return self._agent.run(input)
+
+        # Callable
+        if callable(self._agent):
+            return self._agent(input)
+
+        raise RuntimeError(
+            f"Don't know how to invoke agent of type {type(self._agent)}. "
+            "Agent must have invoke(), run(), or be callable."
+        )
+
+
+def create_langchain_runner(agent_factory: Callable[[], Any]) -> Callable[[str], Trace]:
+    """Create a runner function for LangChain agents.
+
+    This is the main entry point used by evaldeck's EvaluationRunner.
+
+    Args:
+        agent_factory: Function that returns the agent instance.
+
+    Returns:
+        A function that takes input and returns a Trace.
+    """
+    integration = LangChainIntegration()
+    integration.setup(agent_factory)
+    return integration.run

{evaldeck-0.1.1.dist-info → evaldeck-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evaldeck
-Version: 0.1.1
+Version: 0.1.3
 Summary: The evaluation framework for AI agents. Pytest for agents.
 Project-URL: Homepage, https://github.com/tantra-run/evaldeck-py
 Project-URL: Documentation, https://tantra-run.github.io/evaldeck-py/
@@ -29,6 +29,7 @@ Requires-Dist: rich>=13.0
 Provides-Extra: all
 Requires-Dist: anthropic>=0.18; extra == 'all'
 Requires-Dist: openai>=1.0; extra == 'all'
+Requires-Dist: openinference-instrumentation-langchain>=0.1; extra == 'all'
 Provides-Extra: anthropic
 Requires-Dist: anthropic>=0.18; extra == 'anthropic'
 Provides-Extra: dev
@@ -45,6 +46,8 @@ Requires-Dist: mkdocs-autorefs>=0.5; extra == 'docs'
 Requires-Dist: mkdocs-material>=9.5; extra == 'docs'
 Requires-Dist: mkdocs>=1.5; extra == 'docs'
 Requires-Dist: mkdocstrings[python]>=0.24; extra == 'docs'
+Provides-Extra: langchain
+Requires-Dist: openinference-instrumentation-langchain>=0.1; extra == 'langchain'
 Provides-Extra: openai
 Requires-Dist: openai>=1.0; extra == 'openai'
 Description-Content-Type: text/markdown

{evaldeck-0.1.1.dist-info → evaldeck-0.1.3.dist-info}/RECORD

@@ -1,7 +1,7 @@
 evaldeck/__init__.py,sha256=SF9kMDGuf3UHMHrMeT8vBPzdLUtEqTjTAlRk6Fry9b0,1877
 evaldeck/cli.py,sha256=Khrl2CRkrYP18b1mG7sot82t-Glm4YAuNJxNkbRjuGU,10655
-evaldeck/config.py,sha256=V1o5q7nkek2GH1q-bCszxO5DqmrsrvuKDGvb2eZikvs,5847
-evaldeck/evaluator.py,sha256=fRT9Keav5tm1dkHgca8Z0hnVrapCyS6l8Irm1SqjiTE,19695
+evaldeck/config.py,sha256=0Ge9ZWxV_xZ68vzFkDy_IZTyPB_TtDDJtd_gN6tRyoY,5911
+evaldeck/evaluator.py,sha256=bOIL7vaafYieUD7oZWAs6cZCK9ILCp0Wh3OfLKNhXFQ,20115
 evaldeck/results.py,sha256=gygFnuh2cZdZv5ygxDB-Lksv_9N5sAj2HFkEXRgTnqQ,6039
 evaldeck/test_case.py,sha256=cy3Qfcuh4h1BlLPAncthzx3ILabtnnqN76MNhoA_9j8,5084
 evaldeck/trace.py,sha256=erVrdJyfUilutM1z6NioIp8FVbeCh5XP6VhGtbwAClU,5787
@@ -9,13 +9,14 @@ evaldeck/graders/__init__.py,sha256=M418zN3y06Dn449oUtBZLjwSw_tiUzCx3xvRriFhSd8,
 evaldeck/graders/base.py,sha256=CvLq_AQQfQzdrb4Hs1q6gcKB05e0qfWn31fxXir8T-k,4821
 evaldeck/graders/code.py,sha256=t2rfaB-U9LZnwtyCQ8NIW3Qxrb9aGVlgzgTU8oOHJuM,18012
 evaldeck/graders/llm.py,sha256=nWMPacy-wTLKcE-PnIBdWyD1OHpXKNaTOyF1eicbdK0,11725
-evaldeck/integrations/__init__.py,sha256=IFyhW7gmnm3rRo27RRfL6Q2sGZ6lye7Abt9XqUql9eI,821
+evaldeck/integrations/__init__.py,sha256=PwvBNsNCRBsFUVC5hd2vGk7r4DntuPZIDGMYJOojJfg,1131
+evaldeck/integrations/langchain.py,sha256=TYbtHgFOjUpGgLihW4Tnqyyq-AiC_9dy8gc7a0_7kIM,3839
 evaldeck/integrations/opentelemetry.py,sha256=j518FXsD0pqMNF4TvO97elX9oDiK_VaKxXd243q8dLE,15164
 evaldeck/metrics/__init__.py,sha256=jXTIx5k9f1CjwS-9jc25YLeodhencoUOLfbP8qvcbbw,551
 evaldeck/metrics/base.py,sha256=ibUQNfbkQEXTX1x8SqmFWelWAF1DQ785LXP1KYIZWUk,1790
 evaldeck/metrics/builtin.py,sha256=ghdqeZRN51PhLeG8bGnPW2NNoPUAaeD05HtYlWw5yQM,5399
-evaldeck-0.1.1.dist-info/METADATA,sha256=jTSRn_6z8fAWBhmO87g1cosAIOEfr55kWa6hpp4Ngtg,8596
-evaldeck-0.1.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-evaldeck-0.1.1.dist-info/entry_points.txt,sha256=wtyPiDMSTKf41ShIbQC5X8USDn68OybGecpTaMNaGts,47
-evaldeck-0.1.1.dist-info/licenses/LICENSE,sha256=sEp2tzjeTY9bP_jb1TWAGV4yvxNhVngHpJNglJkT9YA,10770
-evaldeck-0.1.1.dist-info/RECORD,,
+evaldeck-0.1.3.dist-info/METADATA,sha256=9vJ9PR4y-KaUfGjKSNn24T9Hx5hRdiMH9HrBN_nM6oE,8780
+evaldeck-0.1.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+evaldeck-0.1.3.dist-info/entry_points.txt,sha256=wtyPiDMSTKf41ShIbQC5X8USDn68OybGecpTaMNaGts,47
+evaldeck-0.1.3.dist-info/licenses/LICENSE,sha256=sEp2tzjeTY9bP_jb1TWAGV4yvxNhVngHpJNglJkT9YA,10770
+evaldeck-0.1.3.dist-info/RECORD,,