rapidata 2.35.0__py3-none-any.whl → 2.35.1__py3-none-any.whl

rapidata/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.35.0"
+__version__ = "2.35.1"
 
 from .rapidata_client import (
     RapidataClient,

rapidata/rapidata_client/benchmark/_detail_mapper.py

@@ -2,8 +2,6 @@ from typing import Literal
 
 
 class DetailMapper:
-    MIN_RESPONSES = 3
-
     @staticmethod
     def get_budget(
         level_of_detail: Literal["low", "medium", "high", "very high"]

rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py

@@ -31,6 +31,7 @@ class RapidataLeaderboard:
         show_prompt_asset: bool,
         inverse_ranking: bool,
         response_budget: int,
+        min_responses_per_matchup: int,
         id: str,
         openapi_service: OpenAPIService,
     ):
@@ -41,6 +42,7 @@ class RapidataLeaderboard:
         self.__show_prompt_asset = show_prompt_asset
         self.__inverse_ranking = inverse_ranking
         self.__response_budget = response_budget
+        self.__min_responses_per_matchup = min_responses_per_matchup
         self.id = id
 
     @property
@@ -62,11 +64,41 @@ class RapidataLeaderboard:
             leaderboard_id=self.id,
             update_leaderboard_response_config_model=UpdateLeaderboardResponseConfigModel(
                 responseBudget=DetailMapper.get_budget(level_of_detail),
-                minResponses=DetailMapper.MIN_RESPONSES,
+                minResponses=self.__min_responses_per_matchup,
             ),
         )
         self.__response_budget = DetailMapper.get_budget(level_of_detail)
 
+    @property
+    def min_responses_per_matchup(self) -> int:
+        """
+        Returns the minimum number of responses required to be considered for the leaderboard.
+        """
+        return self.__min_responses_per_matchup
+
+    @min_responses_per_matchup.setter
+    def min_responses_per_matchup(self, min_responses: int):
+        """
+        Sets the minimum number of responses required to be considered for the leaderboard.
+        """
+        if not isinstance(min_responses, int):
+            raise ValueError("Min responses per matchup must be an integer")
+
+        if min_responses < 3:
+            raise ValueError("Min responses per matchup must be at least 3")
+
+        logger.debug(
+            f"Setting min responses per matchup to {min_responses} for leaderboard {self.name}"
+        )
+        self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_response_config_put(
+            leaderboard_id=self.id,
+            update_leaderboard_response_config_model=UpdateLeaderboardResponseConfigModel(
+                responseBudget=self.__response_budget,
+                minResponses=min_responses,
+            ),
+        )
+        self.__min_responses_per_matchup = min_responses
+
     @property
     def show_prompt_asset(self) -> bool:
         """

rapidata/rapidata_client/benchmark/rapidata_benchmark.py

@@ -172,6 +172,7 @@ class RapidataBenchmark:
                             leaderboard.show_prompt_asset,
                             leaderboard.is_inversed,
                             leaderboard.response_budget,
+                            leaderboard.min_responses,
                             leaderboard.id,
                             self.__openapi_service,
                         )
@@ -258,6 +259,7 @@ class RapidataBenchmark:
         show_prompt_asset: bool = False,
         inverse_ranking: bool = False,
         level_of_detail: Literal["low", "medium", "high", "very high"] = "low",
+        min_responses_per_matchup: int = 3,
     ) -> RapidataLeaderboard:
         """
         Creates a new leaderboard for the benchmark.
@@ -269,7 +271,14 @@ class RapidataBenchmark:
             show_prompt_asset: Whether to show the prompt asset to the users. (only works if the prompt asset is a URL) (default: False)
             inverse_ranking: Whether to inverse the ranking of the leaderboard. (if the question is inversed, e.g. "Which video is worse?")
             level_of_detail: The level of detail of the leaderboard. This will effect how many comparisons are done per model evaluation. (default: "low")
+            min_responses_per_matchup: The minimum number of responses required to be considered for the leaderboard. (default: 3)
         """
+        if not isinstance(min_responses_per_matchup, int):
+            raise ValueError("Min responses per matchup must be an integer")
+
+        if min_responses_per_matchup < 3:
+            raise ValueError("Min responses per matchup must be at least 3")
+
         leaderboard_result = self.__openapi_service.leaderboard_api.leaderboard_post(
             create_leaderboard_model=CreateLeaderboardModel(
                 benchmarkId=self.id,
@@ -278,7 +287,7 @@ class RapidataBenchmark:
                 showPrompt=show_prompt,
                 showPromptAsset=show_prompt_asset,
                 isInversed=inverse_ranking,
-                minResponses=DetailMapper.MIN_RESPONSES,
+                minResponses=min_responses_per_matchup,
                 responseBudget=DetailMapper.get_budget(level_of_detail),
             )
         )
@@ -294,6 +303,7 @@ class RapidataBenchmark:
             show_prompt_asset,
             inverse_ranking,
             leaderboard_result.response_budget,
+            min_responses_per_matchup,
             leaderboard_result.id,
             self.__openapi_service,
         )

{rapidata-2.35.0.dist-info → rapidata-2.35.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rapidata
-Version: 2.35.0
+Version: 2.35.1
 Summary: Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way.
 License: Apache-2.0
 Author: Rapidata AG

{rapidata-2.35.0.dist-info → rapidata-2.35.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rapidata/__init__.py,sha256=LHrPkAuvFTYjxU8zeMmh15W2xmXsvyVcIK_bv8qScRg,897
+rapidata/__init__.py,sha256=7ZoBaYqvNh-HjfP3kpiQCpwmvJYhFGdAVtpwwzyhLsQ,897
 rapidata/api_client/__init__.py,sha256=fb2lqv3sj48wAgarp3g6hvtTgd7bfI01DJJLuBZQnFI,34850
 rapidata/api_client/api/__init__.py,sha256=dGnSE9oPO_ahGh-E1jtw4_VuM_vQueQFuv0IVMQo6uo,1546
 rapidata/api_client/api/benchmark_api.py,sha256=bC8hAPgHIDU5u1e0loWPWnZX33BW6gsAR8oc5199q2k,129777
@@ -542,12 +542,12 @@ rapidata/rapidata_client/__init__.py,sha256=CfkQxCdURXzJsVP6sxKmufze2u-IE_snG_G8
 rapidata/rapidata_client/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 rapidata/rapidata_client/api/rapidata_exception.py,sha256=BIdmHRrJUGW-Mqhp1H_suemZaR6w9TgjWq-ZW5iUPdQ,3878
 rapidata/rapidata_client/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-rapidata/rapidata_client/benchmark/_detail_mapper.py,sha256=IcCW0d3_vwAdyG7V9P5UMPASoblk6JX5UEhrfHcaAIs,967
+rapidata/rapidata_client/benchmark/_detail_mapper.py,sha256=HmzJwR2dojs0c2PaEJ5lwSPtjaqeoPXI7RQk5kwSB9A,944
 rapidata/rapidata_client/benchmark/leaderboard/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py,sha256=ucbE6wSNaT8Li_XTxoze1iEYppaHw8OTcqlP-PW5KMw,4767
+rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py,sha256=xRNXw__K4-4wb4UI-EgLyJRwGO5OkFdoN3A1S3MkdF4,6092
 rapidata/rapidata_client/benchmark/participant/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 rapidata/rapidata_client/benchmark/participant/_participant.py,sha256=yN82EWrZXYszsM8Ns0HRMXCTivltkyxcpGRK-cdT01Y,3683
-rapidata/rapidata_client/benchmark/rapidata_benchmark.py,sha256=wZgN79jV4WF6HtNYl8gqTHArCEi8WHy3zN1xnxzTS6A,14268
+rapidata/rapidata_client/benchmark/rapidata_benchmark.py,sha256=Z0jT9hiWyS3km0BwtGXbdmtopdnecf1z-ucDdwg06Y0,14793
 rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py,sha256=NnXCsU2_8SlGxdI51MeIUWK7Ku0pEylDSPhnzOPzTV0,5291
 rapidata/rapidata_client/country_codes/__init__.py,sha256=FB9Dcks44J6C6YBSYmTmNZ71tE130x6NO_3aLJ8fKzQ,40
 rapidata/rapidata_client/country_codes/country_codes.py,sha256=ePHqeb7y9DWQZAnddBzPx1puYBcrgUjdR2sbFijuFD8,283
@@ -651,7 +651,7 @@ rapidata/service/__init__.py,sha256=s9bS1AJZaWIhLtJX_ZA40_CK39rAAkwdAmymTMbeWl4,
 rapidata/service/credential_manager.py,sha256=pUEEtp6VrFWYhfUUtyqmS0AlRqe2Y0kFkY6o22IT4KM,8682
 rapidata/service/local_file_service.py,sha256=pgorvlWcx52Uh3cEG6VrdMK_t__7dacQ_5AnfY14BW8,877
 rapidata/service/openapi_service.py,sha256=v2fhPbHmD0J11ZRZY6f80PdIdGwpRFlbfMH9t8Ypg5A,5403
-rapidata-2.35.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-rapidata-2.35.0.dist-info/METADATA,sha256=lGStAMXDWn_ztSIkIrN6bpJJ2HiZE7ivo1Z0LQ_eRAo,1264
-rapidata-2.35.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-rapidata-2.35.0.dist-info/RECORD,,
+rapidata-2.35.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+rapidata-2.35.1.dist-info/METADATA,sha256=mQGouVcYnBiJg-noTAW2JpQPxqSId68ZqIznzJDyJWE,1264
+rapidata-2.35.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+rapidata-2.35.1.dist-info/RECORD,,