judgeval 0.0.42__tar.gz → 0.0.43__tar.gz

{judgeval-0.0.42 → judgeval-0.0.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.0.42
+Version: 0.0.43
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues

{judgeval-0.0.42 → judgeval-0.0.43}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.0.42"
+version = "0.0.43"
 authors = [
     { name="Andrew Li", email="andrew@judgmentlabs.ai" },
     { name="Alex Shan", email="alex@judgmentlabs.ai" },

{judgeval-0.0.42 → judgeval-0.0.43}/src/judgeval/common/tracer.py

@@ -2237,7 +2237,7 @@ def wrap(client: Any, trace_across_async_contexts: bool = Tracer.trace_across_as
     Supports OpenAI, Together, Anthropic, and Google GenAI clients.
     Patches both '.create' and Anthropic's '.stream' methods using a wrapper class.
     """
-    span_name, original_create, original_responses_create, original_stream = _get_client_config(client)
+    span_name, original_create, original_responses_create, original_stream, original_beta_parse = _get_client_config(client)
 
     def _get_current_trace():
         if trace_across_async_contexts:
@@ -2307,6 +2307,22 @@ def wrap(client: Any, trace_across_async_contexts: bool = Tracer.trace_across_as
                 _capture_exception_for_trace(span, sys.exc_info())
                 raise e
     
+    async def traced_beta_parse_async(*args, **kwargs):
+        current_trace = _get_current_trace()
+        if not current_trace:
+            return await original_beta_parse(*args, **kwargs)
+        
+        with current_trace.span(span_name, span_type="llm") as span:
+            is_streaming = _record_input_and_check_streaming(span, kwargs)
+            
+            try:
+                response_or_iterator = await original_beta_parse(*args, **kwargs)
+                return _format_and_record_output(span, response_or_iterator, is_streaming, True, False)
+            except Exception as e:
+                _capture_exception_for_trace(span, sys.exc_info())
+                raise e
+                
+    
     # Async responses for OpenAI clients
     async def traced_response_create_async(*args, **kwargs):
         current_trace = _get_current_trace()
@@ -2354,6 +2370,21 @@ def wrap(client: Any, trace_across_async_contexts: bool = Tracer.trace_across_as
             except Exception as e:
                 _capture_exception_for_trace(span, sys.exc_info())
                 raise e
+            
+    def traced_beta_parse_sync(*args, **kwargs):
+        current_trace = _get_current_trace()
+        if not current_trace:
+            return original_beta_parse(*args, **kwargs)
+        
+        with current_trace.span(span_name, span_type="llm") as span:
+            is_streaming = _record_input_and_check_streaming(span, kwargs)
+            
+            try:
+                response_or_iterator = original_beta_parse(*args, **kwargs)
+                return _format_and_record_output(span, response_or_iterator, is_streaming, False, False)
+            except Exception as e:
+                _capture_exception_for_trace(span, sys.exc_info())
+                raise e
     
     def traced_response_create_sync(*args, **kwargs):
         current_trace = _get_current_trace()
@@ -2392,7 +2423,7 @@ def wrap(client: Any, trace_across_async_contexts: bool = Tracer.trace_across_as
         if hasattr(client, "responses") and hasattr(client.responses, "create"):
             client.responses.create = traced_response_create_async
         if hasattr(client, "beta") and hasattr(client.beta, "chat") and hasattr(client.beta.chat, "completions") and hasattr(client.beta.chat.completions, "parse"):
-            client.beta.chat.completions.parse = traced_create_async
+            client.beta.chat.completions.parse = traced_beta_parse_async
     elif isinstance(client, AsyncAnthropic):
         client.messages.create = traced_create_async
         if original_stream:
@@ -2404,7 +2435,7 @@ def wrap(client: Any, trace_across_async_contexts: bool = Tracer.trace_across_as
         if hasattr(client, "responses") and hasattr(client.responses, "create"):
             client.responses.create = traced_response_create_sync
         if hasattr(client, "beta") and hasattr(client.beta, "chat") and hasattr(client.beta.chat, "completions") and hasattr(client.beta.chat.completions, "parse"):
-            client.beta.chat.completions.parse = traced_create_sync
+            client.beta.chat.completions.parse = traced_beta_parse_sync
     elif isinstance(client, Anthropic):
         client.messages.create = traced_create_sync
         if original_stream:
@@ -2423,23 +2454,24 @@ def _get_client_config(client: ApiClient) -> tuple[str, callable, Optional[calla
         client: An instance of OpenAI, Together, or Anthropic client
         
     Returns:
-        tuple: (span_name, create_method, stream_method)
+        tuple: (span_name, create_method, responses_method, stream_method, beta_parse_method)
             - span_name: String identifier for tracing
             - create_method: Reference to the client's creation method
             - responses_method: Reference to the client's responses method (if applicable)
             - stream_method: Reference to the client's stream method (if applicable)
+            - beta_parse_method: Reference to the client's beta parse method (if applicable)
             
     Raises:
         ValueError: If client type is not supported
     """
     if isinstance(client, (OpenAI, AsyncOpenAI)):
-        return "OPENAI_API_CALL", client.chat.completions.create, client.responses.create, None
+        return "OPENAI_API_CALL", client.chat.completions.create, client.responses.create, None, client.beta.chat.completions.parse
     elif isinstance(client, (Together, AsyncTogether)):
-        return "TOGETHER_API_CALL", client.chat.completions.create, None, None
+        return "TOGETHER_API_CALL", client.chat.completions.create, None, None, None
     elif isinstance(client, (Anthropic, AsyncAnthropic)):
-        return "ANTHROPIC_API_CALL", client.messages.create, None, client.messages.stream
+        return "ANTHROPIC_API_CALL", client.messages.create, None, client.messages.stream, None
     elif isinstance(client, (genai.Client, genai.client.AsyncClient)):
-        return "GOOGLE_API_CALL", client.models.generate_content, None, None
+        return "GOOGLE_API_CALL", client.models.generate_content, None, None, None
     raise ValueError(f"Unsupported client type: {type(client)}")
 
 def _format_input_data(client: ApiClient, **kwargs) -> dict:
@@ -2449,10 +2481,13 @@ def _format_input_data(client: ApiClient, **kwargs) -> dict:
     to ensure consistent tracing across different APIs.
     """
     if isinstance(client, (OpenAI, Together, AsyncOpenAI, AsyncTogether)):
-        return {
+        input_data = {
             "model": kwargs.get("model"),
             "messages": kwargs.get("messages"),
         }
+        if kwargs.get("response_format"):
+            input_data["response_format"] = kwargs.get("response_format")
+        return input_data
     elif isinstance(client, (genai.Client, genai.client.AsyncClient)):
         return {
             "model": kwargs.get("model"),
@@ -2522,7 +2557,10 @@ def _format_output_data(client: ApiClient, response: Any) -> dict:
         model_name = response.model
         prompt_tokens = response.usage.prompt_tokens
         completion_tokens = response.usage.completion_tokens
-        message_content = response.choices[0].message.content
+        if hasattr(response.choices[0].message, "parsed") and response.choices[0].message.parsed:
+            message_content = response.choices[0].message.parsed
+        else:
+            message_content = response.choices[0].message.content
     elif isinstance(client, (genai.Client, genai.client.AsyncClient)):
         model_name = response.model_version
         prompt_tokens = response.usage_metadata.prompt_token_count