eval-protocol 0.2.22__tar.gz → 0.2.23__tar.gz

{eval_protocol-0.2.22/eval_protocol.egg-info → eval_protocol-0.2.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.22
+Version: 0.2.23
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -15,7 +15,7 @@ Requires-Dist: pydantic>=2.0.0
 Requires-Dist: dataclasses-json>=0.5.7
 Requires-Dist: uvicorn>=0.15.0
 Requires-Dist: python-dotenv>=0.19.0
-Requires-Dist: openai==1.78.1
+Requires-Dist: openai>=1.78.1
 Requires-Dist: aiosqlite
 Requires-Dist: aiohttp
 Requires-Dist: mcp>=1.9.2
@@ -56,7 +56,7 @@ Requires-Dist: types-requests; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
 Requires-Dist: types-docker; extra == "dev"
 Requires-Dist: versioneer>=0.20; extra == "dev"
-Requires-Dist: openai==1.78.1; extra == "dev"
+Requires-Dist: openai>=1.78.1; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
 Requires-Dist: e2b; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
@@ -100,6 +100,8 @@ Provides-Extra: supabase
 Requires-Dist: supabase>=2.18.1; extra == "supabase"
 Provides-Extra: chinook
 Requires-Dist: psycopg2-binary>=2.9.10; extra == "chinook"
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.3.0; extra == "langchain"
 Dynamic: license-file
 
 # Eval Protocol (EP)

{eval_protocol-0.2.22 → eval_protocol-0.2.23}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-08-29T13:54:58-0700",
+ "date": "2025-09-08T14:20:50-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "cb42aed7e588948fe7c5fa35e5751355dd206195",
- "version": "0.2.22"
+ "full-revisionid": "c6e0a5a4482876eb9e41c68d93ea6b7baac1781f",
+ "version": "0.2.23"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.22 → eval_protocol-0.2.23}/eval_protocol/adapters/bigquery.py

@@ -7,34 +7,36 @@ to EvaluationRow format for use in evaluation pipelines.
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, Union, cast, TypeAlias
 
 from eval_protocol.models import CompletionParams, EvaluationRow, InputMetadata, Message
 
 logger = logging.getLogger(__name__)
 
 try:
+    # Import at runtime if available
     from google.auth.exceptions import DefaultCredentialsError
-    from google.cloud import bigquery
+    from google.cloud import bigquery as _bigquery_runtime  # type: ignore
     from google.cloud.exceptions import Forbidden, NotFound
     from google.oauth2 import service_account
 
     BIGQUERY_AVAILABLE = True
 except ImportError:
+    # Provide fallbacks for type checking/runtime when package is missing
+    DefaultCredentialsError = Exception  # type: ignore[assignment]
+    Forbidden = Exception  # type: ignore[assignment]
+    NotFound = Exception  # type: ignore[assignment]
+    service_account: Any
+    service_account = None
+    _bigquery_runtime = None  # type: ignore[assignment]
     BIGQUERY_AVAILABLE = False
     # Optional dependency: avoid noisy warnings during import
     logger.debug("Google Cloud BigQuery not installed. Optional feature disabled.")
 
-# Avoid importing BigQuery types at runtime for annotations when not installed
-if TYPE_CHECKING:
-    from google.cloud import bigquery as _bigquery_type
-
-    QueryParameterType = Union[
-        _bigquery_type.ScalarQueryParameter,
-        _bigquery_type.ArrayQueryParameter,
-    ]
-else:
-    QueryParameterType = Any
+# Simple type aliases to avoid importing optional google types under pyright
+QueryParameterType: TypeAlias = Any
+BigQueryClient: TypeAlias = Any
+QueryJobConfig: TypeAlias = Any
 
 # Type alias for transformation function
 TransformFunction = Callable[[Dict[str, Any]], Dict[str, Any]]
@@ -98,7 +100,13 @@ class BigQueryAdapter:
                 client_args["location"] = location
 
             client_args.update(client_kwargs)
-            self.client = bigquery.Client(**client_args)
+            # Use runtime alias to avoid basedpyright import symbol error when lib is missing
+            if _bigquery_runtime is None:
+                raise ImportError(
+                    "google-cloud-bigquery is not installed. Install with: pip install 'eval-protocol[bigquery]'"
+                )
+            # Avoid strict typing on optional dependency
+            self.client = _bigquery_runtime.Client(**client_args)  # type: ignore[no-untyped-call, assignment]
 
         except DefaultCredentialsError as e:
             logger.error("Failed to authenticate with BigQuery: %s", e)
@@ -139,7 +147,9 @@ class BigQueryAdapter:
         """
         try:
             # Configure query job
-            job_config = bigquery.QueryJobConfig()
+            if _bigquery_runtime is None:
+                raise RuntimeError("BigQuery runtime not available")
+            job_config = _bigquery_runtime.QueryJobConfig()  # type: ignore[no-untyped-call]
             if query_params:
                 job_config.query_parameters = query_params
             if self.location:

eval_protocol-0.2.23/eval_protocol/adapters/langchain.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+import os
+from typing import Any, Dict, List, Optional
+
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage
+
+from eval_protocol.models import Message
+
+
+def _dbg_enabled() -> bool:
+    return os.getenv("EP_DEBUG_SERIALIZATION", "0").strip() == "1"
+
+
+def _dbg_print(*args):
+    if _dbg_enabled():
+        try:
+            print(*args)
+        except Exception:
+            pass
+
+
+def serialize_lc_message_to_ep(msg: BaseMessage) -> Message:
+    _dbg_print(
+        "[EP-Ser] Input LC msg:",
+        type(msg).__name__,
+        {
+            "has_additional_kwargs": isinstance(getattr(msg, "additional_kwargs", None), dict),
+            "content_type": type(getattr(msg, "content", None)).__name__,
+        },
+    )
+
+    if isinstance(msg, HumanMessage):
+        ep_msg = Message(role="user", content=str(msg.content))
+        _dbg_print("[EP-Ser] -> EP Message:", {"role": ep_msg.role, "len": len(ep_msg.content or "")})
+        return ep_msg
+
+    if isinstance(msg, AIMessage):
+        content = ""
+        if isinstance(msg.content, str):
+            content = msg.content
+        elif isinstance(msg.content, list):
+            parts: List[str] = []
+            for item in msg.content:
+                if isinstance(item, dict):
+                    if item.get("type") == "text":
+                        parts.append(str(item.get("text", "")))
+                elif isinstance(item, str):
+                    parts.append(item)
+            content = "\n".join(parts)
+
+        tool_calls_payload: Optional[List[Dict[str, Any]]] = None
+
+        def _normalize_tool_calls(tc_list: List[Any]) -> List[Dict[str, Any]]:
+            mapped: List[Dict[str, Any]] = []
+            for call in tc_list:
+                if not isinstance(call, dict):
+                    continue
+                try:
+                    call_id = call.get("id") or "toolcall_0"
+                    if isinstance(call.get("function"), dict):
+                        fn = call["function"]
+                        fn_name = fn.get("name") or call.get("name") or "tool"
+                        fn_args = fn.get("arguments")
+                    else:
+                        fn_name = call.get("name") or "tool"
+                        fn_args = call.get("arguments") if call.get("arguments") is not None else call.get("args")
+                    if not isinstance(fn_args, str):
+                        import json as _json
+
+                        fn_args = _json.dumps(fn_args or {}, ensure_ascii=False)
+                    mapped.append(
+                        {
+                            "id": call_id,
+                            "type": "function",
+                            "function": {"name": fn_name, "arguments": fn_args},
+                        }
+                    )
+                except Exception:
+                    continue
+            return mapped
+
+        ak = getattr(msg, "additional_kwargs", None)
+        if isinstance(ak, dict):
+            tc = ak.get("tool_calls")
+            if isinstance(tc, list) and tc:
+                mapped = _normalize_tool_calls(tc)
+                if mapped:
+                    tool_calls_payload = mapped
+
+        if tool_calls_payload is None:
+            raw_attr_tc = getattr(msg, "tool_calls", None)
+            if isinstance(raw_attr_tc, list) and raw_attr_tc:
+                mapped = _normalize_tool_calls(raw_attr_tc)
+                if mapped:
+                    tool_calls_payload = mapped
+
+        # Extract reasoning/thinking parts into reasoning_content
+        reasoning_content = None
+        if isinstance(msg.content, list):
+            collected = [
+                it.get("thinking", "") for it in msg.content if isinstance(it, dict) and it.get("type") == "thinking"
+            ]
+            if collected:
+                reasoning_content = "\n\n".join([s for s in collected if s]) or None
+
+        # Message.tool_calls expects List[ChatCompletionMessageToolCall] | None.
+        # We pass through Dicts at runtime but avoid type error by casting.
+        ep_msg = Message(
+            role="assistant",
+            content=content,
+            tool_calls=tool_calls_payload,  # type: ignore[arg-type]
+            reasoning_content=reasoning_content,
+        )
+        _dbg_print(
+            "[EP-Ser] -> EP Message:",
+            {
+                "role": ep_msg.role,
+                "content_len": len(ep_msg.content or ""),
+                "tool_calls": len(ep_msg.tool_calls or []) if isinstance(ep_msg.tool_calls, list) else 0,
+            },
+        )
+        return ep_msg
+
+    if isinstance(msg, ToolMessage):
+        tool_name = msg.name or "tool"
+        status = msg.status or "success"
+        content = str(msg.content)
+        tool_call_id = getattr(msg, "tool_call_id", None)
+        ep_msg = Message(
+            role="tool",
+            name=tool_name,
+            tool_call_id=tool_call_id,
+            content=f'<{tool_name} status="{status}">\n{content}\n</{tool_name}>',
+        )
+        _dbg_print(
+            "[EP-Ser] -> EP Message:", {"role": ep_msg.role, "name": ep_msg.name, "has_id": bool(ep_msg.tool_call_id)}
+        )
+        return ep_msg
+
+    ep_msg = Message(role=getattr(msg, "type", "assistant"), content=str(getattr(msg, "content", "")))
+    _dbg_print("[EP-Ser] -> EP Message (fallback):", {"role": ep_msg.role, "len": len(ep_msg.content or "")})
+    return ep_msg

{eval_protocol-0.2.22 → eval_protocol-0.2.23}/eval_protocol/adapters/langfuse.py

@@ -6,14 +6,14 @@ to EvaluationRow format for use in evaluation pipelines.
 
 import logging
 from datetime import datetime
-from typing import Any, Dict, Iterator, List, Optional
+from typing import Any, Dict, Iterator, List, Optional, cast
 
 from eval_protocol.models import EvaluationRow, InputMetadata, Message
 
 logger = logging.getLogger(__name__)
 
 try:
-    from langfuse import Langfuse
+    from langfuse import Langfuse  # pyright: ignore[reportPrivateImportUsage]
 
     LANGFUSE_AVAILABLE = True
 except ImportError:
@@ -63,7 +63,7 @@ class LangfuseAdapter:
         if not LANGFUSE_AVAILABLE:
             raise ImportError("Langfuse not installed. Install with: pip install 'eval-protocol[langfuse]'")
 
-        self.client = Langfuse(public_key=public_key, secret_key=secret_key, host=host)
+        self.client = cast(Any, Langfuse)(public_key=public_key, secret_key=secret_key, host=host)
         self.project_id = project_id
 
     def get_evaluation_rows(
@@ -75,7 +75,7 @@ class LangfuseAdapter:
         from_timestamp: Optional[datetime] = None,
         to_timestamp: Optional[datetime] = None,
         include_tool_calls: bool = True,
-    ) -> Iterator[EvaluationRow]:
+    ) -> List[EvaluationRow]:
         """Pull traces from Langfuse and convert to EvaluationRow format.
 
         Args:
@@ -90,8 +90,9 @@ class LangfuseAdapter:
         Yields:
             EvaluationRow: Converted evaluation rows
         """
-        # Get traces from Langfuse
-        traces = self.client.get_traces(
+        # Get traces from Langfuse using new API
+        eval_rows = []
+        traces = self.client.api.trace.list(
             limit=limit,
             tags=tags,
             user_id=user_id,
@@ -104,16 +105,17 @@ class LangfuseAdapter:
             try:
                 eval_row = self._convert_trace_to_evaluation_row(trace, include_tool_calls)
                 if eval_row:
-                    yield eval_row
+                    eval_rows.append(eval_row)
             except (AttributeError, ValueError, KeyError) as e:
                 logger.warning("Failed to convert trace %s: %s", trace.id, e)
                 continue
+        return eval_rows
 
     def get_evaluation_rows_by_ids(
         self,
         trace_ids: List[str],
         include_tool_calls: bool = True,
-    ) -> Iterator[EvaluationRow]:
+    ) -> List[EvaluationRow]:
         """Get specific traces by their IDs and convert to EvaluationRow format.
 
         Args:
@@ -123,15 +125,17 @@ class LangfuseAdapter:
         Yields:
             EvaluationRow: Converted evaluation rows
         """
+        eval_rows = []
         for trace_id in trace_ids:
             try:
-                trace = self.client.get_trace(trace_id)
+                trace = self.client.api.trace.get(trace_id)
                 eval_row = self._convert_trace_to_evaluation_row(trace, include_tool_calls)
                 if eval_row:
-                    yield eval_row
+                    eval_rows.append(eval_row)
             except (AttributeError, ValueError, KeyError) as e:
                 logger.warning("Failed to fetch/convert trace %s: %s", trace_id, e)
                 continue
+        return eval_rows
 
     def _convert_trace_to_evaluation_row(self, trace: Any, include_tool_calls: bool = True) -> Optional[EvaluationRow]:
         """Convert a Langfuse trace to EvaluationRow format.
@@ -145,10 +149,29 @@ class LangfuseAdapter:
         """
         try:
             # Get observations (generations, spans) from the trace
-            observations = self.client.get_observations(trace_id=trace.id).data
+            observations_response = self.client.api.observations.get_many(trace_id=trace.id, limit=100)
+            observations = (
+                observations_response.data if hasattr(observations_response, "data") else list(observations_response)
+            )
 
-            # Convert observations to messages
-            messages = self._extract_messages_from_observations(observations, include_tool_calls)
+            # Look for conversation history in trace output or observations
+            messages = []
+            conversation_found = False
+
+            # Look for complete conversation in observations
+            if not conversation_found:
+                for obs in observations:
+                    # Check each observation's output for complete conversation array
+                    if hasattr(obs, "output") and obs.output:
+                        conversation = self._extract_conversation_from_output(obs.output)
+                        if conversation:
+                            messages = conversation
+                            conversation_found = True
+                            break
+
+            # Fallback: try extracting from observations using old method
+            if not conversation_found:
+                messages = self._extract_messages_from_observations(observations, include_tool_calls)
 
             if not messages:
                 return None
@@ -266,6 +289,86 @@ class LangfuseAdapter:
             function_call=function_call,
         )
 
+    def _extract_conversation_from_output(self, output: Any) -> Optional[List[Message]]:
+        """Extract conversation history from PydanticAI agent run output.
+
+        This looks for the conversation format like:
+        [
+            {"role": "user", "content": "..."},
+            {"role": "assistant", "content": "...", "tool_calls": [...]},
+            {"role": "tool", "content": "...", "name": "execute_sql"},
+            ...
+        ]
+
+        Args:
+            output: The output object to search for conversation history
+
+        Returns:
+            List of Message objects or None if no conversation found
+        """
+        messages = []
+
+        try:
+            # Handle different output formats
+            conversation_data = None
+
+            if isinstance(output, list):
+                # Direct list of messages
+                conversation_data = output
+            elif isinstance(output, dict):
+                # Look for conversation in various nested formats
+                if "messages" in output:
+                    conversation_data = output["messages"]
+                elif "conversation" in output:
+                    conversation_data = output["conversation"]
+                elif "history" in output:
+                    conversation_data = output["history"]
+                elif "agent_run" in output:  # Handle nested conversation data PydanticAI style
+                    agent_run = output["agent_run"]
+                    if isinstance(agent_run, dict) and "messages" in agent_run:
+                        conversation_data = agent_run["messages"]
+                elif len(output.keys()) == 1:
+                    # Single key, check if its value is a list
+                    single_key = list(output.keys())[0]
+                    if isinstance(output[single_key], list):
+                        conversation_data = output[single_key]
+            elif isinstance(output, str):
+                # Try to parse JSON string
+                import json
+
+                try:
+                    parsed = json.loads(output)
+                    return self._extract_conversation_from_output(parsed)
+                except (json.JSONDecodeError, ValueError):
+                    pass
+
+            # Parse conversation data into messages
+            if conversation_data and isinstance(conversation_data, list):
+                for msg_data in conversation_data:
+                    if isinstance(msg_data, dict) and "role" in msg_data:
+                        role = msg_data.get("role")
+                        if role is None:
+                            continue
+                        content = msg_data.get("content", "")
+
+                        # Handle tool calls in assistant messages
+                        tool_calls = None
+                        if role == "assistant" and "tool_calls" in msg_data:
+                            tool_calls = msg_data["tool_calls"]
+
+                        # Handle tool responses
+                        name = None
+                        if role == "tool":
+                            name = msg_data.get("name")
+
+                        messages.append(Message(role=role, content=content, name=name, tool_calls=tool_calls))
+
+            return messages if messages else None
+
+        except Exception as e:
+            logger.debug("Error extracting conversation from output: %s", e)
+            return None
+
     def _create_input_metadata(self, trace: Any, observations: List[Any]) -> InputMetadata:
         """Create InputMetadata from trace and observations.
 
@@ -276,22 +379,32 @@ class LangfuseAdapter:
         Returns:
             InputMetadata object
         """
-        # Extract completion parameters from observations
+        # Extract completion parameters from trace input first, then observations
         completion_params = {}
 
-        # Look for model parameters in observations
-        for obs in observations:
-            if hasattr(obs, "model") and obs.model:
-                completion_params["model"] = obs.model
-            if hasattr(obs, "model_parameters") and obs.model_parameters:
-                params = obs.model_parameters
-                if "temperature" in params:
-                    completion_params["temperature"] = params["temperature"]
-                if "max_tokens" in params:
-                    completion_params["max_tokens"] = params["max_tokens"]
-                if "top_p" in params:
-                    completion_params["top_p"] = params["top_p"]
-                break
+        # First check trace input for evaluation test completion_params
+        if hasattr(trace, "input") and trace.input:
+            if isinstance(trace.input, dict):
+                kwargs = trace.input.get("kwargs", {})
+                if "completion_params" in kwargs:
+                    trace_completion_params = kwargs["completion_params"]
+                    if trace_completion_params and isinstance(trace_completion_params, dict):
+                        completion_params.update(trace_completion_params)
+
+        # Fallback: Look for model parameters in observations if not found in trace input
+        if not completion_params:
+            for obs in observations:
+                if hasattr(obs, "model") and obs.model:
+                    completion_params["model"] = obs.model
+                if hasattr(obs, "model_parameters") and obs.model_parameters:
+                    params = obs.model_parameters
+                    if "temperature" in params:
+                        completion_params["temperature"] = params["temperature"]
+                    if "max_tokens" in params:
+                        completion_params["max_tokens"] = params["max_tokens"]
+                    if "top_p" in params:
+                        completion_params["top_p"] = params["top_p"]
+                    break
 
         # Create dataset info from trace metadata
         dataset_info = {
@@ -331,6 +444,20 @@ class LangfuseAdapter:
         Returns:
             Ground truth string or None
         """
+        # First check trace input for evaluation test data structure
+        if hasattr(trace, "input") and trace.input:
+            if isinstance(trace.input, dict):
+                # Handle EP test format: kwargs.input_rows[0].ground_truth
+                kwargs = trace.input.get("kwargs", {})
+                if "input_rows" in kwargs:
+                    input_rows = kwargs["input_rows"]
+                    if input_rows and len(input_rows) > 0:
+                        first_row = input_rows[0]
+                        if isinstance(first_row, dict) and "ground_truth" in first_row:
+                            ground_truth = first_row["ground_truth"]
+                            if ground_truth:  # Only return if not None/empty
+                                return str(ground_truth)
+
         # Check trace metadata for ground truth
         if hasattr(trace, "metadata") and trace.metadata:
             if isinstance(trace.metadata, dict):