rapidata 2.35.0__py3-none-any.whl → 2.35.2__py3-none-any.whl

rapidata/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.35.0"
+__version__ = "2.35.2"
 
 from .rapidata_client import (
     RapidataClient,

rapidata/api_client/api/leaderboard_api.py

@@ -3117,7 +3117,7 @@ class LeaderboardApi:
         _headers: Optional[Dict[StrictStr, Any]] = None,
         _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
     ) -> None:
-        """Updates the name of a leaderboard.
+        """Updates the response config of a leaderboard.
 
 
         :param leaderboard_id:  (required)
@@ -3187,7 +3187,7 @@ class LeaderboardApi:
         _headers: Optional[Dict[StrictStr, Any]] = None,
         _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
     ) -> ApiResponse[None]:
-        """Updates the name of a leaderboard.
+        """Updates the response config of a leaderboard.
 
 
         :param leaderboard_id:  (required)
@@ -3257,7 +3257,7 @@ class LeaderboardApi:
         _headers: Optional[Dict[StrictStr, Any]] = None,
         _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
     ) -> RESTResponseType:
-        """Updates the name of a leaderboard.
+        """Updates the response config of a leaderboard.
 
 
         :param leaderboard_id:  (required)

rapidata/api_client_README.md

@@ -135,7 +135,7 @@ Class | Method | HTTP request | Description
 *LeaderboardApi* | [**leaderboard_leaderboard_id_participants_post**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_leaderboard_id_participants_post) | **POST** /leaderboard/{leaderboardId}/participants | Creates a participant in a leaderboard.
 *LeaderboardApi* | [**leaderboard_leaderboard_id_prompts_get**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_leaderboard_id_prompts_get) | **GET** /leaderboard/{leaderboardId}/prompts | returns the paged prompts of a leaderboard by its ID.
 *LeaderboardApi* | [**leaderboard_leaderboard_id_prompts_post**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_leaderboard_id_prompts_post) | **POST** /leaderboard/{leaderboardId}/prompts | adds a new prompt to a leaderboard.
-*LeaderboardApi* | [**leaderboard_leaderboard_id_response_config_put**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_leaderboard_id_response_config_put) | **PUT** /leaderboard/{leaderboardId}/response-config | Updates the name of a leaderboard.
+*LeaderboardApi* | [**leaderboard_leaderboard_id_response_config_put**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_leaderboard_id_response_config_put) | **PUT** /leaderboard/{leaderboardId}/response-config | Updates the response config of a leaderboard.
 *LeaderboardApi* | [**leaderboard_leaderboard_id_runs_get**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_leaderboard_id_runs_get) | **GET** /leaderboard/{leaderboardId}/runs | Gets the runs related to a leaderboard
 *LeaderboardApi* | [**leaderboard_leaderboard_id_standings_get**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_leaderboard_id_standings_get) | **GET** /leaderboard/{leaderboardId}/standings | queries all the participants connected to leaderboard by its ID.
 *LeaderboardApi* | [**leaderboard_post**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_post) | **POST** /leaderboard | Creates a new leaderboard with the specified name and criteria.

rapidata/rapidata_client/api/rapidata_exception.py

@@ -2,106 +2,135 @@ from typing import Optional, Any
 from rapidata.api_client.api_client import ApiClient, rest, ApiResponse, ApiResponseT
 from rapidata.api_client.exceptions import ApiException
 import json
+import threading
+from contextlib import contextmanager
 from rapidata.rapidata_client.logging import logger
 
+# Thread-local storage for controlling error logging
+_thread_local = threading.local()
+
+
+@contextmanager
+def suppress_rapidata_error_logging():
+    """Context manager to suppress error logging for RapidataApiClient calls."""
+    old_value = getattr(_thread_local, "suppress_error_logging", False)
+    _thread_local.suppress_error_logging = True
+    try:
+        yield
+    finally:
+        _thread_local.suppress_error_logging = old_value
+
+
+def _should_suppress_error_logging() -> bool:
+    """Check if error logging should be suppressed for the current thread."""
+    return getattr(_thread_local, "suppress_error_logging", False)
+
+
 class RapidataError(Exception):
     """Custom error class for Rapidata API errors."""
-    
+
     def __init__(
-        self, 
-        status_code: Optional[int] = None, 
-        message: str | None = None, 
+        self,
+        status_code: Optional[int] = None,
+        message: str | None = None,
         original_exception: Exception | None = None,
-        details: Any = None
+        details: Any = None,
     ):
         self.status_code = status_code
         self.message = message
         self.original_exception = original_exception
         self.details = details
-        
+
         # Create a nice error message
         error_msg = "Rapidata API Error"
         if status_code:
             error_msg += f" ({status_code})"
         if message:
             error_msg += f": {message}"
-            
+
         super().__init__(error_msg)
 
     def __str__(self):
-        """Return a string representation of the error."""        
+        """Return a string representation of the error."""
         # Extract information from message if available
         title = None
         errors = None
         trace_id = None
-        
+
         # Try to extract from details if available and is a dict
         if self.details and isinstance(self.details, dict):
-            title = self.details.get('title')
-            errors = self.details.get('errors')
-            trace_id = self.details.get('traceId')
-        
+            title = self.details.get("title")
+            errors = self.details.get("errors")
+            trace_id = self.details.get("traceId")
+
         # Build the error string
         error_parts = []
-        
+
         # Main error line
         if title:
             error_parts.append(f"{title}")
         else:
             error_parts.append(f"{self.message or 'Unknown error'}")
-        
+
         # Reasons
         if errors:
             if isinstance(errors, dict):
                 error_parts.append(f"Reasons: {json.dumps({'errors': errors})}")
             else:
                 error_parts.append(f"Reasons: {errors}")
-        
+
         # Trace ID
         if trace_id:
             error_parts.append(f"Trace Id: {trace_id}")
         else:
             error_parts.append("Trace Id: N/A")
-        
+
         return "\n".join(error_parts)
 
+
 class RapidataApiClient(ApiClient):
     """Custom API client that wraps errors in RapidataError."""
 
     def response_deserialize(
         self,
         response_data: rest.RESTResponse,
-        response_types_map: Optional[dict[str, ApiResponseT]] = None
+        response_types_map: Optional[dict[str, ApiResponseT]] = None,
     ) -> ApiResponse[ApiResponseT]:
         """Override the response_deserialize method to catch and convert exceptions."""
         try:
             return super().response_deserialize(response_data, response_types_map)
         except ApiException as e:
-            status_code = getattr(e, 'status', None)
+            status_code = getattr(e, "status", None)
             message = str(e)
             details = None
-            
+
             # Extract more detailed error message from response body if available
-            if hasattr(e, 'body') and e.body:
+            if hasattr(e, "body") and e.body:
                 try:
                     body_json = json.loads(e.body)
                     if isinstance(body_json, dict):
-                        if 'message' in body_json:
-                            message = body_json['message']
-                        elif 'error' in body_json:
-                            message = body_json['error']
-                        
+                        if "message" in body_json:
+                            message = body_json["message"]
+                        elif "error" in body_json:
+                            message = body_json["error"]
+
                         # Store the full error details for debugging
                         details = body_json
                 except (json.JSONDecodeError, AttributeError):
                     # If we can't parse the body as JSON, use the original message
                     pass
-            
-            error_formatted =  RapidataError(
-                status_code=status_code, 
-                message=message, 
+
+            error_formatted = RapidataError(
+                status_code=status_code,
+                message=message,
                 original_exception=e,
-                details=details
+                details=details,
             )
-            logger.error(f"Error: {error_formatted}")
+
+            # Only log error if not suppressed
+            if not _should_suppress_error_logging():
+                logger.error("Error: %s", error_formatted)
+            else:
+                logger.debug("Suppressed Error: %s", error_formatted)
+
             raise error_formatted from None

rapidata/rapidata_client/benchmark/_detail_mapper.py

@@ -2,8 +2,6 @@ from typing import Literal
 
 
 class DetailMapper:
-    MIN_RESPONSES = 3
-
     @staticmethod
     def get_budget(
         level_of_detail: Literal["low", "medium", "high", "very high"]

rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py

@@ -31,6 +31,7 @@ class RapidataLeaderboard:
         show_prompt_asset: bool,
         inverse_ranking: bool,
         response_budget: int,
+        min_responses_per_matchup: int,
         id: str,
         openapi_service: OpenAPIService,
     ):
@@ -41,6 +42,7 @@ class RapidataLeaderboard:
         self.__show_prompt_asset = show_prompt_asset
         self.__inverse_ranking = inverse_ranking
         self.__response_budget = response_budget
+        self.__min_responses_per_matchup = min_responses_per_matchup
         self.id = id
 
     @property
@@ -62,11 +64,41 @@ class RapidataLeaderboard:
             leaderboard_id=self.id,
             update_leaderboard_response_config_model=UpdateLeaderboardResponseConfigModel(
                 responseBudget=DetailMapper.get_budget(level_of_detail),
-                minResponses=DetailMapper.MIN_RESPONSES,
+                minResponses=self.__min_responses_per_matchup,
             ),
         )
         self.__response_budget = DetailMapper.get_budget(level_of_detail)
 
+    @property
+    def min_responses_per_matchup(self) -> int:
+        """
+        Returns the minimum number of responses required to be considered for the leaderboard.
+        """
+        return self.__min_responses_per_matchup
+
+    @min_responses_per_matchup.setter
+    def min_responses_per_matchup(self, min_responses: int):
+        """
+        Sets the minimum number of responses required to be considered for the leaderboard.
+        """
+        if not isinstance(min_responses, int):
+            raise ValueError("Min responses per matchup must be an integer")
+
+        if min_responses < 3:
+            raise ValueError("Min responses per matchup must be at least 3")
+
+        logger.debug(
+            f"Setting min responses per matchup to {min_responses} for leaderboard {self.name}"
+        )
+        self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_response_config_put(
+            leaderboard_id=self.id,
+            update_leaderboard_response_config_model=UpdateLeaderboardResponseConfigModel(
+                responseBudget=self.__response_budget,
+                minResponses=min_responses,
+            ),
+        )
+        self.__min_responses_per_matchup = min_responses
+
     @property
     def show_prompt_asset(self) -> bool:
         """

rapidata/rapidata_client/benchmark/rapidata_benchmark.py

@@ -172,6 +172,7 @@ class RapidataBenchmark:
                             leaderboard.show_prompt_asset,
                             leaderboard.is_inversed,
                             leaderboard.response_budget,
+                            leaderboard.min_responses,
                             leaderboard.id,
                             self.__openapi_service,
                         )
@@ -258,6 +259,7 @@ class RapidataBenchmark:
         show_prompt_asset: bool = False,
         inverse_ranking: bool = False,
         level_of_detail: Literal["low", "medium", "high", "very high"] = "low",
+        min_responses_per_matchup: int = 3,
     ) -> RapidataLeaderboard:
         """
         Creates a new leaderboard for the benchmark.
@@ -269,7 +271,14 @@ class RapidataBenchmark:
             show_prompt_asset: Whether to show the prompt asset to the users. (only works if the prompt asset is a URL) (default: False)
             inverse_ranking: Whether to inverse the ranking of the leaderboard. (if the question is inversed, e.g. "Which video is worse?")
             level_of_detail: The level of detail of the leaderboard. This will effect how many comparisons are done per model evaluation. (default: "low")
+            min_responses_per_matchup: The minimum number of responses required to be considered for the leaderboard. (default: 3)
         """
+        if not isinstance(min_responses_per_matchup, int):
+            raise ValueError("Min responses per matchup must be an integer")
+
+        if min_responses_per_matchup < 3:
+            raise ValueError("Min responses per matchup must be at least 3")
+
         leaderboard_result = self.__openapi_service.leaderboard_api.leaderboard_post(
             create_leaderboard_model=CreateLeaderboardModel(
                 benchmarkId=self.id,
@@ -278,7 +287,7 @@ class RapidataBenchmark:
                 showPrompt=show_prompt,
                 showPromptAsset=show_prompt_asset,
                 isInversed=inverse_ranking,
-                minResponses=DetailMapper.MIN_RESPONSES,
+                minResponses=min_responses_per_matchup,
                 responseBudget=DetailMapper.get_budget(level_of_detail),
             )
         )
@@ -294,6 +303,7 @@ class RapidataBenchmark:
             show_prompt_asset,
             inverse_ranking,
             leaderboard_result.response_budget,
+            min_responses_per_matchup,
             leaderboard_result.id,
             self.__openapi_service,
         )

rapidata/rapidata_client/datapoints/assets/_multi_asset.py

@@ -5,7 +5,7 @@ Defines the MultiAsset class for handling multiple BaseAsset instances.
 
 from rapidata.rapidata_client.datapoints.assets._base_asset import BaseAsset
 from rapidata.rapidata_client.datapoints.assets import MediaAsset, TextAsset
-from typing import Iterator, Sequence, cast
+from typing import Iterator, Sequence
 
 
 class MultiAsset(BaseAsset):
@@ -26,16 +26,16 @@ class MultiAsset(BaseAsset):
         """
         if len(assets) != 2:
             raise ValueError("Assets must come in pairs for comparison tasks.")
-        
+
         for asset in assets:
             if not isinstance(asset, (TextAsset, MediaAsset)):
-                raise TypeError("All assets must be a TextAsset or MediaAsset.")    
-        
+                raise TypeError("All assets must be a TextAsset or MediaAsset.")
+
         if not all(isinstance(asset, type(assets[0])) for asset in assets):
             raise ValueError("All assets must be of the same type.")
-        
+
         self.assets = assets
-        
+
     def __len__(self) -> int:
         """
         Get the number of assets in the MultiAsset.
@@ -56,6 +56,6 @@ class MultiAsset(BaseAsset):
 
     def __str__(self) -> str:
         return f"MultiAsset(assets={self.assets})"
-    
+
     def __repr__(self) -> str:
         return f"MultiAsset(assets={self.assets})"