judgeval 0.0.42__tar.gz → 0.0.44__tar.gz

{judgeval-0.0.42 → judgeval-0.0.44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.0.42
+Version: 0.0.44
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues

{judgeval-0.0.42 → judgeval-0.0.44}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.0.42"
+version = "0.0.44"
 authors = [
     { name="Andrew Li", email="andrew@judgmentlabs.ai" },
     { name="Alex Shan", email="alex@judgmentlabs.ai" },

{judgeval-0.0.42 → judgeval-0.0.44}/src/judgeval/common/tracer.py

@@ -1557,7 +1557,6 @@ class _DeepTracer:
 #         current_trace.record_output({"log": message})
     
 class Tracer:
-    _instance = None
 
     # Tracer.current_trace class variable is currently used in wrap()
     # TODO: Keep track of cross-context state for current trace and current span ID solely through class variables instead of instance variables?
@@ -1567,11 +1566,6 @@ class Tracer:
 
     trace_across_async_contexts: bool = False # BY default, we don't trace across async contexts
 
-    def __new__(cls, *args, **kwargs):
-        if cls._instance is None:
-            cls._instance = super(Tracer, cls).__new__(cls)
-        return cls._instance
-
     def __init__(
         self, 
         api_key: str = os.getenv("JUDGMENT_API_KEY"), 
@@ -1595,66 +1589,56 @@ class Tracer:
         span_flush_interval: float = 1.0,  # Time in seconds between automatic flushes
         span_num_workers: int = 10  # Number of worker threads for span processing
         ):
-        if not hasattr(self, 'initialized'):
-            if not api_key:
-                raise ValueError("Tracer must be configured with a Judgment API key")
-            
-            result, response = validate_api_key(api_key)
-            if not result:
-                raise JudgmentAPIError(f"Issue with passed in Judgment API key: {response}")
-            
-            if not organization_id:
-                raise ValueError("Tracer must be configured with an Organization ID")
-            if use_s3 and not s3_bucket_name:
-                raise ValueError("S3 bucket name must be provided when use_s3 is True")
-            
-            self.api_key: str = api_key
-            self.project_name: str = project_name or str(uuid.uuid4())
-            self.organization_id: str = organization_id
-            self.rules: List[Rule] = rules or []  # Store rules at tracer level
-            self.traces: List[Trace] = []
-            self.initialized: bool = True
-            self.enable_monitoring: bool = enable_monitoring
-            self.enable_evaluations: bool = enable_evaluations
-            self.class_identifiers: Dict[str, str] = {}  # Dictionary to store class identifiers
-            self.span_id_to_previous_span_id: Dict[str, str] = {}
-            self.trace_id_to_previous_trace: Dict[str, TraceClient] = {}
-            self.current_span_id: Optional[str] = None
-            self.current_trace: Optional[TraceClient] = None
-            self.trace_across_async_contexts: bool = trace_across_async_contexts
-            Tracer.trace_across_async_contexts = trace_across_async_contexts
-
-            # Initialize S3 storage if enabled
-            self.use_s3 = use_s3
-            if use_s3:
-                from judgeval.common.s3_storage import S3Storage
-                self.s3_storage = S3Storage(
-                    bucket_name=s3_bucket_name,
-                    aws_access_key_id=s3_aws_access_key_id,
-                    aws_secret_access_key=s3_aws_secret_access_key,
-                    region_name=s3_region_name
-                )
-            self.offline_mode: bool = offline_mode
-            self.deep_tracing: bool = deep_tracing  # NEW: Store deep tracing setting
-            
-            # Initialize background span service
-            self.enable_background_spans: bool = enable_background_spans
-            self.background_span_service: Optional[BackgroundSpanService] = None
-            if enable_background_spans and not offline_mode:
-                self.background_span_service = BackgroundSpanService(
-                    judgment_api_key=api_key,
-                    organization_id=organization_id,
-                    batch_size=span_batch_size,
-                    flush_interval=span_flush_interval,
-                    num_workers=span_num_workers
-                )
-
-        elif hasattr(self, 'project_name') and self.project_name != project_name:
-            warnings.warn(
-                f"Attempting to initialize Tracer with project_name='{project_name}' but it was already initialized with "
-                f"project_name='{self.project_name}'. Due to the singleton pattern, the original project_name will be used. "
-                "To use a different project name, ensure the first Tracer initialization uses the desired project name.",
-                RuntimeWarning
+        if not api_key:
+            raise ValueError("Tracer must be configured with a Judgment API key")
+        
+        result, response = validate_api_key(api_key)
+        if not result:
+            raise JudgmentAPIError(f"Issue with passed in Judgment API key: {response}")
+        
+        if not organization_id:
+            raise ValueError("Tracer must be configured with an Organization ID")
+        if use_s3 and not s3_bucket_name:
+            raise ValueError("S3 bucket name must be provided when use_s3 is True")
+        
+        self.api_key: str = api_key
+        self.project_name: str = project_name or str(uuid.uuid4())
+        self.organization_id: str = organization_id
+        self.rules: List[Rule] = rules or []  # Store rules at tracer level
+        self.traces: List[Trace] = []
+        self.enable_monitoring: bool = enable_monitoring
+        self.enable_evaluations: bool = enable_evaluations
+        self.class_identifiers: Dict[str, str] = {}  # Dictionary to store class identifiers
+        self.span_id_to_previous_span_id: Dict[str, str] = {}
+        self.trace_id_to_previous_trace: Dict[str, TraceClient] = {}
+        self.current_span_id: Optional[str] = None
+        self.current_trace: Optional[TraceClient] = None
+        self.trace_across_async_contexts: bool = trace_across_async_contexts
+        Tracer.trace_across_async_contexts = trace_across_async_contexts
+
+        # Initialize S3 storage if enabled
+        self.use_s3 = use_s3
+        if use_s3:
+            from judgeval.common.s3_storage import S3Storage
+            self.s3_storage = S3Storage(
+                bucket_name=s3_bucket_name,
+                aws_access_key_id=s3_aws_access_key_id,
+                aws_secret_access_key=s3_aws_secret_access_key,
+                region_name=s3_region_name
+            )
+        self.offline_mode: bool = offline_mode
+        self.deep_tracing: bool = deep_tracing  # NEW: Store deep tracing setting
+        
+        # Initialize background span service
+        self.enable_background_spans: bool = enable_background_spans
+        self.background_span_service: Optional[BackgroundSpanService] = None
+        if enable_background_spans and not offline_mode:
+            self.background_span_service = BackgroundSpanService(
+                judgment_api_key=api_key,
+                organization_id=organization_id,
+                batch_size=span_batch_size,
+                flush_interval=span_flush_interval,
+                num_workers=span_num_workers
             )
 
     def set_current_span(self, span_id: str):
@@ -2237,7 +2221,7 @@ def wrap(client: Any, trace_across_async_contexts: bool = Tracer.trace_across_as
     Supports OpenAI, Together, Anthropic, and Google GenAI clients.
     Patches both '.create' and Anthropic's '.stream' methods using a wrapper class.
     """
-    span_name, original_create, original_responses_create, original_stream = _get_client_config(client)
+    span_name, original_create, original_responses_create, original_stream, original_beta_parse = _get_client_config(client)
 
     def _get_current_trace():
         if trace_across_async_contexts:
@@ -2307,6 +2291,22 @@ def wrap(client: Any, trace_across_async_contexts: bool = Tracer.trace_across_as
                 _capture_exception_for_trace(span, sys.exc_info())
                 raise e
     
+    async def traced_beta_parse_async(*args, **kwargs):
+        current_trace = _get_current_trace()
+        if not current_trace:
+            return await original_beta_parse(*args, **kwargs)
+        
+        with current_trace.span(span_name, span_type="llm") as span:
+            is_streaming = _record_input_and_check_streaming(span, kwargs)
+            
+            try:
+                response_or_iterator = await original_beta_parse(*args, **kwargs)
+                return _format_and_record_output(span, response_or_iterator, is_streaming, True, False)
+            except Exception as e:
+                _capture_exception_for_trace(span, sys.exc_info())
+                raise e
+                
+    
     # Async responses for OpenAI clients
     async def traced_response_create_async(*args, **kwargs):
         current_trace = _get_current_trace()
@@ -2354,6 +2354,21 @@ def wrap(client: Any, trace_across_async_contexts: bool = Tracer.trace_across_as
             except Exception as e:
                 _capture_exception_for_trace(span, sys.exc_info())
                 raise e
+            
+    def traced_beta_parse_sync(*args, **kwargs):
+        current_trace = _get_current_trace()
+        if not current_trace:
+            return original_beta_parse(*args, **kwargs)
+        
+        with current_trace.span(span_name, span_type="llm") as span:
+            is_streaming = _record_input_and_check_streaming(span, kwargs)
+            
+            try:
+                response_or_iterator = original_beta_parse(*args, **kwargs)
+                return _format_and_record_output(span, response_or_iterator, is_streaming, False, False)
+            except Exception as e:
+                _capture_exception_for_trace(span, sys.exc_info())
+                raise e
     
     def traced_response_create_sync(*args, **kwargs):
         current_trace = _get_current_trace()
@@ -2392,7 +2407,7 @@ def wrap(client: Any, trace_across_async_contexts: bool = Tracer.trace_across_as
         if hasattr(client, "responses") and hasattr(client.responses, "create"):
             client.responses.create = traced_response_create_async
         if hasattr(client, "beta") and hasattr(client.beta, "chat") and hasattr(client.beta.chat, "completions") and hasattr(client.beta.chat.completions, "parse"):
-            client.beta.chat.completions.parse = traced_create_async
+            client.beta.chat.completions.parse = traced_beta_parse_async
     elif isinstance(client, AsyncAnthropic):
         client.messages.create = traced_create_async
         if original_stream:
@@ -2404,7 +2419,7 @@ def wrap(client: Any, trace_across_async_contexts: bool = Tracer.trace_across_as
         if hasattr(client, "responses") and hasattr(client.responses, "create"):
             client.responses.create = traced_response_create_sync
         if hasattr(client, "beta") and hasattr(client.beta, "chat") and hasattr(client.beta.chat, "completions") and hasattr(client.beta.chat.completions, "parse"):
-            client.beta.chat.completions.parse = traced_create_sync
+            client.beta.chat.completions.parse = traced_beta_parse_sync
     elif isinstance(client, Anthropic):
         client.messages.create = traced_create_sync
         if original_stream:
@@ -2423,23 +2438,24 @@ def _get_client_config(client: ApiClient) -> tuple[str, callable, Optional[calla
         client: An instance of OpenAI, Together, or Anthropic client
         
     Returns:
-        tuple: (span_name, create_method, stream_method)
+        tuple: (span_name, create_method, responses_method, stream_method, beta_parse_method)
             - span_name: String identifier for tracing
             - create_method: Reference to the client's creation method
             - responses_method: Reference to the client's responses method (if applicable)
             - stream_method: Reference to the client's stream method (if applicable)
+            - beta_parse_method: Reference to the client's beta parse method (if applicable)
             
     Raises:
         ValueError: If client type is not supported
     """
     if isinstance(client, (OpenAI, AsyncOpenAI)):
-        return "OPENAI_API_CALL", client.chat.completions.create, client.responses.create, None
+        return "OPENAI_API_CALL", client.chat.completions.create, client.responses.create, None, client.beta.chat.completions.parse
     elif isinstance(client, (Together, AsyncTogether)):
-        return "TOGETHER_API_CALL", client.chat.completions.create, None, None
+        return "TOGETHER_API_CALL", client.chat.completions.create, None, None, None
     elif isinstance(client, (Anthropic, AsyncAnthropic)):
-        return "ANTHROPIC_API_CALL", client.messages.create, None, client.messages.stream
+        return "ANTHROPIC_API_CALL", client.messages.create, None, client.messages.stream, None
     elif isinstance(client, (genai.Client, genai.client.AsyncClient)):
-        return "GOOGLE_API_CALL", client.models.generate_content, None, None
+        return "GOOGLE_API_CALL", client.models.generate_content, None, None, None
     raise ValueError(f"Unsupported client type: {type(client)}")
 
 def _format_input_data(client: ApiClient, **kwargs) -> dict:
@@ -2449,10 +2465,13 @@ def _format_input_data(client: ApiClient, **kwargs) -> dict:
     to ensure consistent tracing across different APIs.
     """
     if isinstance(client, (OpenAI, Together, AsyncOpenAI, AsyncTogether)):
-        return {
+        input_data = {
             "model": kwargs.get("model"),
             "messages": kwargs.get("messages"),
         }
+        if kwargs.get("response_format"):
+            input_data["response_format"] = kwargs.get("response_format")
+        return input_data
     elif isinstance(client, (genai.Client, genai.client.AsyncClient)):
         return {
             "model": kwargs.get("model"),
@@ -2522,7 +2541,10 @@ def _format_output_data(client: ApiClient, response: Any) -> dict:
         model_name = response.model
         prompt_tokens = response.usage.prompt_tokens
         completion_tokens = response.usage.completion_tokens
-        message_content = response.choices[0].message.content
+        if hasattr(response.choices[0].message, "parsed") and response.choices[0].message.parsed:
+            message_content = response.choices[0].message.parsed
+        else:
+            message_content = response.choices[0].message.content
     elif isinstance(client, (genai.Client, genai.client.AsyncClient)):
         model_name = response.model_version
         prompt_tokens = response.usage_metadata.prompt_token_count