rapidata 2.27.6__py3-none-any.whl → 2.28.0__py3-none-any.whl

rapidata/__init__.py

@@ -1,9 +1,10 @@
-__version__ = "2.27.6"
+__version__ = "2.28.0"
 
 from .rapidata_client import (
     RapidataClient,
     DemographicSelection,
     LabelingSelection,
+    EffortEstimationSelection,
     RetrievalMode,
     ValidationSelection,
     ConditionalValidationSelection,
@@ -17,6 +18,7 @@ from .rapidata_client import (
     NoShuffle,
     PlayVideoUntilTheEnd,
     CustomSetting,
+    AllowNeitherBoth,
     CountryFilter,
     LanguageFilter,
     UserScoreFilter,

rapidata/rapidata_client/__init__.py

@@ -7,6 +7,7 @@ from .selection import (
     CappedSelection,
     ShufflingSelection,
     RetrievalMode,
+    EffortEstimationSelection,
 )
 from .metadata import (
     PrivateTextMetadata,
@@ -23,6 +24,7 @@ from .settings import (
     NoShuffle,
     PlayVideoUntilTheEnd,
     CustomSetting,
+    AllowNeitherBoth,
     )
 from .country_codes import CountryCodes
 from .assets import (

rapidata/rapidata_client/leaderboard/rapidata_leaderboard.py

@@ -0,0 +1,127 @@
+from rapidata.api_client.models.create_leaderboard_participant_model import CreateLeaderboardParticipantModel
+from rapidata.api_client.models.page_info import PageInfo
+from rapidata.api_client.models.query_model import QueryModel
+
+from rapidata.rapidata_client.order._rapidata_dataset import RapidataDataset
+from rapidata.rapidata_client.assets import MediaAsset
+from rapidata.rapidata_client.metadata import PromptMetadata
+from rapidata.service.openapi_service import OpenAPIService
+
+
+class RapidataLeaderboard:
+    """
+    An instance of a Rapidata leaderboard.
+
+    Used to interact with a specific leaderboard in the Rapidata system, such as retrieving prompts and evaluating models.
+
+    Args:
+        name: The name that will be used to identify the leaderboard on the overview.
+        instruction: The instruction that will determine what how the models will be evaluated.
+        show_prompt: Whether to show the prompt to the users.
+        id: The ID of the leaderboard.
+        openapi_service: The OpenAPIService instance for API interaction.
+    """
+    def __init__(self, name: str, instruction: str, show_prompt: bool, id: str, openapi_service: OpenAPIService):
+        self.__openapi_service = openapi_service
+        self.name = name
+        self.instruction = instruction
+        self.show_prompt = show_prompt
+        self._prompts: list[str] = []
+        self.id = id
+
+    @property
+    def prompts(self) -> list[str]:
+        """
+        Returns the prompts that are registered for the leaderboard.
+        """
+        if not self._prompts:
+            current_page = 1
+            total_pages = None
+            
+            while True:
+                prompts_result = self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_prompts_get(
+                    leaderboard_id=self.id,
+                    request=QueryModel(
+                        page=PageInfo(
+                            index=current_page,
+                            size=100
+                        )
+                    )
+                )
+                
+                if prompts_result.total_pages is None:
+                    raise ValueError("An error occurred while fetching prompts: total_pages is None")
+                
+                total_pages = prompts_result.total_pages
+                
+                self._prompts.extend([prompt.prompt for prompt in prompts_result.items])
+                
+                if current_page >= total_pages:
+                    break
+                    
+                current_page += 1
+                
+        return self._prompts
+    
+    def _register_prompts(self, prompts: list[str]):
+        """
+        Registers the prompts for the leaderboard.
+        """
+        for prompt in prompts:
+            self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_prompts_post(
+                leaderboard_id=self.id,
+                body=prompt
+            )
+        self._prompts = prompts
+
+    def evaluate_model(self, name: str, media: list[str], prompts: list[str]) -> None:
+        """
+        Evaluates a model on the leaderboard.
+
+        Args:
+            name: The name of the model.
+            media: The generated images/videos that will be used to evaluate the model.
+            prompts: The prompts that correspond to the media. The order of the prompts must match the order of the media.
+                The prompts that are used must be registered for the leaderboard. To see the registered prompts, use the prompts property.
+        """
+        if not media:
+            raise ValueError("Media must be a non-empty list of strings")
+        
+        if len(media) != len(prompts):
+            raise ValueError("Media and prompts must have the same length")
+        
+        if not all(prompt in self.prompts for prompt in prompts):
+            raise ValueError("All prompts must be in the registered prompts list. To see the registered prompts, use the prompts property.")
+        
+        # happens before the creation of the participant to ensure all media paths are valid
+        assets = []
+        prompts_metadata: list[list[PromptMetadata]] = []
+        for media_path, prompt in zip(media, prompts):
+            assets.append(MediaAsset(media_path))
+            prompts_metadata.append([PromptMetadata(prompt)])
+
+        participant_result = self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_participants_post(
+            leaderboard_id=self.id,
+            create_leaderboard_participant_model=CreateLeaderboardParticipantModel(
+                name=name,
+            )
+        )
+        dataset = RapidataDataset(participant_result.dataset_id, self.__openapi_service)
+
+        dataset._add_datapoints(assets, prompts_metadata)
+
+        self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_participants_participant_id_submit_post(
+            leaderboard_id=self.id,
+            participant_id=participant_result.participant_id
+        )
+
+    def __str__(self) -> str:
+        return f"RapidataLeaderboard(name={self.name}, instruction={self.instruction}, show_prompt={self.show_prompt}, leaderboard_id={self.id})"
+    
+    def __repr__(self) -> str:
+        return self.__str__()
+
+
+        
+
+

rapidata/rapidata_client/leaderboard/rapidata_leaderboard_manager.py

@@ -0,0 +1,80 @@
+from rapidata.service.openapi_service import OpenAPIService
+from rapidata.rapidata_client.leaderboard.rapidata_leaderboard import RapidataLeaderboard
+from rapidata.api_client.models.query_model import QueryModel
+from rapidata.api_client.models.page_info import PageInfo
+from rapidata.api_client.models.root_filter import RootFilter
+from rapidata.api_client.models.filter import Filter
+from rapidata.api_client.models.sort_criterion import SortCriterion
+from rapidata.api_client.models.create_leaderboard_model import CreateLeaderboardModel
+
+class RapidataLeaderboardManager:
+    """
+    A manager for leaderboards.
+
+    Used to create and retrieve leaderboards.
+
+    Args:
+        openapi_service: The OpenAPIService instance for API interaction.
+    """
+    def __init__(self, openapi_service: OpenAPIService):
+        self.__openapi_service = openapi_service
+
+    def create_new_leaderboard(self, name: str, instruction: str, prompts: list[str], show_prompt: bool = False) -> RapidataLeaderboard:
+        """
+        Creates a new leaderboard with the given name, instruction, and prompts.
+
+        Args:
+            leaderboard_name: The name of the leaderboard. Will be used to identify the leaderboard on the overview.
+            instruction: The instruction for the leaderboard. Will determine how the models will be evaluated.
+            prompts: The prompts for the leaderboard. Will be registered for the leaderboard and able to be retrieved again later.
+            show_prompt: Whether to show the prompt to the users when they are evaluating the models.
+        """
+        leaderboard_id = self.__register_new_leaderboard(name, instruction, show_prompt)
+        leaderboard = RapidataLeaderboard(name, instruction, show_prompt, leaderboard_id, self.__openapi_service)
+        leaderboard._register_prompts(prompts)
+        return leaderboard
+    
+    def get_leaderboard_by_id(self, leaderboard_id: str) -> RapidataLeaderboard:
+        """
+        Retrieves a leaderboard by its ID.
+
+        Args:
+            leaderboard_id: The ID of the leaderboard.
+        """
+        leaderboard_result = self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_get(
+            leaderboard_id=leaderboard_id
+        )
+        return RapidataLeaderboard(leaderboard_result.name, leaderboard_result.instruction, leaderboard_result.show_prompt, leaderboard_id, self.__openapi_service)
+    
+    def find_leaderboards(self, name: str = "", amount: int = 10) -> list[RapidataLeaderboard]:
+        """
+        Find your recent leaderboards given criteria. If nothing is provided, it will return the most recent leaderboard.
+
+        Args:
+            name (str, optional): The name of the leaderboard - matching leaderboard will contain the name. Defaults to "" for any leaderboard.
+            amount (int, optional): The amount of leaderboards to return. Defaults to 10.
+        """
+        leaderboard_result = self.__openapi_service.leaderboard_api.leaderboards_get(
+            request=QueryModel(
+                page=PageInfo(
+                    index=1,
+                    size=amount
+                ),
+                filter=RootFilter(filters=[Filter(field="Name", operator="Contains", value=name)]),
+                sortCriteria=[SortCriterion(direction="Desc", propertyName="CreatedAt")]
+            )
+        )
+        leaderboards = []
+        for leaderboard in leaderboard_result.items:
+            leaderboards.append(RapidataLeaderboard(leaderboard.name, leaderboard.instruction, leaderboard.show_prompt, leaderboard.id, self.__openapi_service))
+        return leaderboards
+
+    def __register_new_leaderboard(self, name: str, instruction: str, show_prompt: bool) -> str:
+        leaderboard_id = self.__openapi_service.leaderboard_api.leaderboard_post(
+            CreateLeaderboardModel(
+                name=name,
+                instruction=instruction,
+                showPrompt=show_prompt
+            )
+        ).id
+        return leaderboard_id

rapidata/rapidata_client/order/rapidata_order.py

@@ -169,7 +169,6 @@ class RapidataOrder:
         Args:
             preliminary_results: If True, returns the preliminary results of the order. Defaults to False. 
                 Note that preliminary results are not final and may not contain all the datapoints & responses. Only the onese that are already available.
-                This will throw an exception if there are no responses available yet.
         """
         logger.info(f"Getting results for order '{self}'...")
         if preliminary_results and self.get_status() not in [OrderState.COMPLETED]:

rapidata/rapidata_client/order/rapidata_order_manager.py

@@ -49,7 +49,7 @@ class RapidataOrderManager:
         selections (RapidataSelections): The RapidataSelections instance."""
 
     def __init__(self, openapi_service: OpenAPIService):
-        self._openapi_service = openapi_service
+        self.__openapi_service = openapi_service
         self.filters = RapidataFilters
         self.settings = RapidataSettings
         self.selections = RapidataSelections
@@ -104,7 +104,7 @@ class RapidataOrderManager:
                 max_vote_count=responses_per_datapoint,
             )
 
-        order_builder = RapidataOrderBuilder(name=name, openapi_service=self._openapi_service)
+        order_builder = RapidataOrderBuilder(name=name, openapi_service=self.__openapi_service)
 
         if selections and validation_set_id:
             logger.warning("Warning: Both selections and validation_set_id provided. Ignoring validation_set_id.")
@@ -637,12 +637,12 @@ class RapidataOrderManager:
             RapidataOrder: The Order instance.
         """
         
-        order = self._openapi_service.order_api.order_order_id_get(order_id)
+        order = self.__openapi_service.order_api.order_order_id_get(order_id)
 
         return RapidataOrder(
             order_id=order_id, 
             name=order.order_name,
-            openapi_service=self._openapi_service)
+            openapi_service=self.__openapi_service)
 
     def find_orders(self, name: str = "", amount: int = 10) -> list[RapidataOrder]:
         """Find your recent orders given criteria. If nothing is provided, it will return the most recent order.
@@ -654,7 +654,7 @@ class RapidataOrderManager:
         Returns:
             list[RapidataOrder]: A list of RapidataOrder instances.
         """
-        order_page_result = self._openapi_service.order_api.orders_get(QueryModel(
+        order_page_result = self.__openapi_service.order_api.orders_get(QueryModel(
             page=PageInfo(index=1, size=amount),
             filter=RootFilter(filters=[Filter(field="OrderName", operator="Contains", value=name)]),
             sortCriteria=[SortCriterion(direction="Desc", propertyName="OrderDate")]

rapidata/rapidata_client/order/rapidata_results.py

@@ -180,31 +180,39 @@ class RapidataResults(dict):
 
         rows = []
         for result in self["results"]:
-            # Get the image names from the first metric we find
+            # Get all asset names from the first metric we find
+            assets = []
             for key in result:
-                if isinstance(result[key], dict) and len(result[key]) == 2:
+                if isinstance(result[key], dict) and len(result[key]) >= 2:
                     assets = list(result[key].keys())
                     break
             else:
                 continue
 
-            asset_a, asset_b = assets[0], assets[1]
+            assets = [asset for asset in assets if asset not in ["Both", "Neither"]]
             
             # Initialize row with non-comparative fields
             row = {
                 key: value for key, value in result.items() 
                 if not isinstance(value, dict)
             }
-
-            row["assetA"] = asset_a
-            row["assetB"] = asset_b
+            row["assetA"] = assets[0]
+            row["assetB"] = assets[1]
 
             # Handle comparative metrics
             for key, values in result.items():
-                if isinstance(values, dict) and len(values) == 2:
-                    row[f'A_{key}'] = values[asset_a]
-                    row[f'B_{key}'] = values[asset_b]
+                if isinstance(values, dict) and len(values) >= 2:
+                    # Add main asset columns
+                    for i, asset in enumerate(assets[:2]):  # Limit to first 2 main assets
+                        column_prefix = "A_" if i == 0 else "B_"
+                        row[f'{column_prefix}{key}'] = values.get(asset, 0)
                     
+                    # Add special option columns if they exist
+                    if "Both" in values:
+                        row[f'Both_{key}'] = values.get("Both", 0)
+                    if "Neither" in values:
+                        row[f'Neither_{key}'] = values.get("Neither", 0)
+                        
             rows.append(row)
             
         return pd.DataFrame(rows)

rapidata/rapidata_client/rapidata_client.py

@@ -5,6 +5,7 @@ from rapidata import __version__
 from rapidata.service.openapi_service import OpenAPIService
 
 from rapidata.rapidata_client.order.rapidata_order_manager import RapidataOrderManager
+from rapidata.rapidata_client.leaderboard.rapidata_leaderboard_manager import RapidataLeaderboardManager
 
 from rapidata.rapidata_client.validation.validation_set_manager import (
     ValidationSetManager,
@@ -65,6 +66,9 @@ class RapidataClient:
 
         logger.debug("Initializing DemographicManager")
         self._demographic = DemographicManager(openapi_service=self._openapi_service)
+
+        logger.debug("Initializing RapidataLeaderboardManager")
+        self.mri = RapidataLeaderboardManager(openapi_service=self._openapi_service)
         
     def reset_credentials(self):
         """Reset the credentials saved in the configuration file for the current environment."""

rapidata/rapidata_client/selection/__init__.py

@@ -8,3 +8,4 @@ from .shuffling_selection import ShufflingSelection
 from .ab_test_selection import AbTestSelection
 from .static_selection import StaticSelection
 from .retrieval_modes import RetrievalMode
+from .effort_selection import EffortEstimationSelection

rapidata/rapidata_client/selection/effort_selection.py

@@ -0,0 +1,19 @@
+from rapidata.rapidata_client.selection._base_selection import RapidataSelection
+from rapidata.api_client.models.effort_capped_selection import EffortCappedSelection as EffortCappedSelectionModel
+from rapidata.rapidata_client.selection.retrieval_modes import RetrievalMode
+
+
+class EffortEstimationSelection(RapidataSelection):
+    
+
+    def __init__(self, effort_budget: int, retrieval_mode: RetrievalMode = RetrievalMode.Shuffled, max_iterations: int | None = None):
+        self.effort_budget = effort_budget
+        self.retrieval_mode = retrieval_mode
+        self.max_iterations = max_iterations
+
+    def _to_model(self):
+        return EffortCappedSelectionModel(
+            _t="EffortCappedSelection", 
+            effortBudget=self.effort_budget, 
+            retrievalMode=self.retrieval_mode.value, 
+            maxIterations=self.max_iterations)

rapidata/rapidata_client/settings/__init__.py

@@ -7,3 +7,4 @@ from .play_video_until_the_end import PlayVideoUntilTheEnd
 from .custom_setting import CustomSetting
 from .rapidata_settings import RapidataSettings
 from ._rapidata_setting import RapidataSetting
+from .allow_neither_both import AllowNeitherBoth

rapidata/rapidata_client/settings/allow_neither_both.py

@@ -0,0 +1,15 @@
+from rapidata.rapidata_client.settings._rapidata_setting import RapidataSetting
+from rapidata.rapidata_client.logging import managed_print, logger
+
+class AllowNeitherBoth(RapidataSetting):
+    """
+    Set whether to allow neither or both options.
+    This setting only works for compare orders.
+
+    Args:
+        value (bool): Whether to allow neither or both options. Defaults to True.
+            If this setting is not added to an order, the users won't be able to select neither or both.
+    """
+
+    def __init__(self, value: bool = True):
+        super().__init__(key="compare_unsure", value=value)

rapidata/rapidata_client/settings/rapidata_settings.py

@@ -4,6 +4,7 @@ from rapidata.rapidata_client.settings import (
     FreeTextMinimumCharacters,
     NoShuffle,
     PlayVideoUntilTheEnd,
+    AllowNeitherBoth,
     )
 
 class RapidataSettings:
@@ -18,6 +19,7 @@ class RapidataSettings:
         free_text_minimum_characters (FreeTextMinimumCharacters): Only for free text tasks. Set the minimum number of characters a user has to type.
         no_shuffle (NoShuffle): Only for classification and compare tasks. If true, the order of the categories / images will not be shuffled and presented in the same order as specified.
         play_video_until_the_end (PlayVideoUntilTheEnd): Allows users to only answer once the video has finished playing.
+        allow_neither_both (AllowNeitherBoth): Only for compare tasks. If true, the users will be able to select neither or both instead of exclusively one of the options.
 
     Example:
         ```python
@@ -33,4 +35,4 @@ class RapidataSettings:
     free_text_minimum_characters = FreeTextMinimumCharacters
     no_shuffle = NoShuffle
     play_video_until_the_end = PlayVideoUntilTheEnd
-
+    allow_neither_both = AllowNeitherBoth

rapidata/service/openapi_service.py

@@ -6,6 +6,7 @@ from rapidata.api_client.api.dataset_api import DatasetApi
 from rapidata.api_client.api.order_api import OrderApi
 from rapidata.api_client.api.pipeline_api import PipelineApi
 from rapidata.api_client.api.rapid_api import RapidApi
+from rapidata.api_client.api.leaderboard_api import LeaderboardApi
 from rapidata.api_client.api.validation_set_api import ValidationSetApi
 from rapidata.api_client.api.workflow_api import WorkflowApi
 from rapidata.api_client.configuration import Configuration
@@ -107,6 +108,10 @@ class OpenAPIService:
     @property
     def workflow_api(self) -> WorkflowApi:
         return WorkflowApi(self.api_client)
+    
+    @property
+    def leaderboard_api(self) -> LeaderboardApi:
+        return LeaderboardApi(self.api_client)
 
     def _get_rapidata_package_version(self):
         """

{rapidata-2.27.6.dist-info → rapidata-2.28.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rapidata
-Version: 2.27.6
+Version: 2.28.0
 Summary: Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way.
 License: Apache-2.0
 Author: Rapidata AG

{rapidata-2.27.6.dist-info → rapidata-2.28.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rapidata/__init__.py,sha256=h-fS9jURWePpNXWIPSVFdQz2xJ-oaHFEIy0Y_2epD1w,812
+rapidata/__init__.py,sha256=U0Jnzk7Duyre8xS9I1pgxHtLblb_2zvnYOh7I_sWO3A,865
 rapidata/api_client/__init__.py,sha256=hJ8b6i_S5KZCpmLupQJRh839MRweXxX85ti0Px7vI78,30530
 rapidata/api_client/api/__init__.py,sha256=K67VMR3zWtv824onw87pEm-K2xrDphRIY04TtKg_k2U,1359
 rapidata/api_client/api/campaign_api.py,sha256=ZEYXEp8_mzsElbklLXBLGnKEfPB1mx8-G5CXfSnibq0,80791
@@ -466,7 +466,7 @@ rapidata/api_client/models/workflow_split_model_filter_configs_inner.py,sha256=1
 rapidata/api_client/models/workflow_state.py,sha256=5LAK1se76RCoozeVB6oxMPb8p_5bhLZJqn7q5fFQWis,850
 rapidata/api_client/rest.py,sha256=rtIMcgINZOUaDFaJIinJkXRSddNJmXvMRMfgO2Ezk2o,10835
 rapidata/api_client_README.md,sha256=81eV2jPGLgFdtvjLD_SSn2jixfyHOO5MPvj3hvF65qk,60685
-rapidata/rapidata_client/__init__.py,sha256=xiMeESpBd-q11SnzRNSupqGqncLIjrQJgxdoilC1muA,1080
+rapidata/rapidata_client/__init__.py,sha256=MLl41ZPDYezE9ookAjHS75wFqfCTOKq-U01GJbHFjrA,1133
 rapidata/rapidata_client/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 rapidata/rapidata_client/api/rapidata_exception.py,sha256=BIdmHRrJUGW-Mqhp1H_suemZaR6w9TgjWq-ZW5iUPdQ,3878
 rapidata/rapidata_client/assets/__init__.py,sha256=eQkqUrYFME1FCxPY2Xh2bbonKVPnsFElJ6aPFcsWGxI,361
@@ -497,6 +497,9 @@ rapidata/rapidata_client/filter/or_filter.py,sha256=u6vkXMTG_j15SbY3bkbnXbxJMDgE
 rapidata/rapidata_client/filter/rapidata_filters.py,sha256=aHl8bjvL0wJLhzm6BCHy7mPGYYYKTLZhY0WZVeHx0ZA,2014
 rapidata/rapidata_client/filter/response_count_filter.py,sha256=sDv9Dvy0FbnIQRSAxFGrUf9SIMISTNxnlAQcrFKBjXE,1989
 rapidata/rapidata_client/filter/user_score_filter.py,sha256=2C78zkWm5TnfkxGbV1ER2xB7s9ynpacaibzyRZKG8Cc,1566
+rapidata/rapidata_client/leaderboard/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rapidata/rapidata_client/leaderboard/rapidata_leaderboard.py,sha256=sBYRlWxT2TeZnvwU3uuhVJl-o_iv5aM7f80dKOPc_Bk,5222
+rapidata/rapidata_client/leaderboard/rapidata_leaderboard_manager.py,sha256=OjTpHucHh02uMWvKxSu37Hv8re41x1DbmgXLSN_I62w,3982
 rapidata/rapidata_client/logging/__init__.py,sha256=4gLxePW8TvgYDZmPWMcf6fA8bEyu35vMKOmlPj5oXNE,110
 rapidata/rapidata_client/logging/logger.py,sha256=9vULXUizGObQeqMY-CryiAQsq8xDZw0ChLhvV8oa99s,3907
 rapidata/rapidata_client/logging/output_manager.py,sha256=AmSVZ2emVW5UWgOiNqkXNVRItsvd5Ox0hsIoZQhYYYo,653
@@ -510,36 +513,38 @@ rapidata/rapidata_client/metadata/_select_words_metadata.py,sha256=-MK5yQDi_G3BK
 rapidata/rapidata_client/order/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 rapidata/rapidata_client/order/_rapidata_dataset.py,sha256=v5b86EDF0ITIOV4k4QU8gQ4eFPz2ow-4HV_mmC9tb4c,21264
 rapidata/rapidata_client/order/_rapidata_order_builder.py,sha256=ioNGmWQF4KMdzvm-GIfAeflK8AgKaczZ1FfKkrZ1xXY,12649
-rapidata/rapidata_client/order/rapidata_order.py,sha256=uk2p6Hx2KTN4Oq2S35esdip7yLR44y-kkamS-5TBPFE,12752
-rapidata/rapidata_client/order/rapidata_order_manager.py,sha256=K9Nc66UmBfFQjSe78SMdZTOa4z3j5TRyvBnEX0Cs0u4,38306
-rapidata/rapidata_client/order/rapidata_results.py,sha256=UllYpuqpm2inKdRNhClaUwApuxsMLrvrGDsrHA5KqbY,8111
-rapidata/rapidata_client/rapidata_client.py,sha256=iAycSS8Dyt0_xnILpU5N9DoeGlB3v2UtlinmxcFh-eo,4029
+rapidata/rapidata_client/order/rapidata_order.py,sha256=CmZr0wAtogjhunhXACA9DvWCYf0Q5dXNthDCAUqjT00,12666
+rapidata/rapidata_client/order/rapidata_order_manager.py,sha256=trEbTLe6-SsuWn5Q_DISxURxxlVjVSdBgjCjTVLZ3Mw,38311
+rapidata/rapidata_client/order/rapidata_results.py,sha256=ZY0JyHMBZlR6-t6SqKt2OLEO6keR_KvKg9Wk6_I29x4,8653
+rapidata/rapidata_client/rapidata_client.py,sha256=EMli04-y8WJ8BQpuh7VLDKHYudvoKz5rv5L-QKIYcsg,4284
 rapidata/rapidata_client/referee/__init__.py,sha256=q0Hv9nmfEpyChejtyMLT8hWKL0vTTf_UgUXPYNJ-H6M,153
 rapidata/rapidata_client/referee/_base_referee.py,sha256=MdFOhdxt3sRnWXLDKLJZKFdVpjBGn9jypPnWWQ6msQA,496
 rapidata/rapidata_client/referee/_early_stopping_referee.py,sha256=ULbokQZ91wc9D_20qHUhe55D28D9eTY1J1cMp_-oIDc,2088
 rapidata/rapidata_client/referee/_naive_referee.py,sha256=PVR8uy8hfRjr2DBzdOFyvou6S3swNc-4UvgjhO-09TU,1209
-rapidata/rapidata_client/selection/__init__.py,sha256=vrE6pbkz_iDCy6ID4sdnHjtsaPrkJk2ZXVmgDiD8lK0,518
+rapidata/rapidata_client/selection/__init__.py,sha256=vC2XbykShj_VW1uz5IZfQQXjgeIzzdYqC3n0K2c8cIs,574
 rapidata/rapidata_client/selection/_base_selection.py,sha256=tInbWOgxT_4CHkr5QHoG55ZcUi1ZmfcEGIwLKKCnN20,147
 rapidata/rapidata_client/selection/ab_test_selection.py,sha256=fymubkVMawqJmYp9FKzWXTki9tgBgoj3cOP8rG9oOd0,1284
 rapidata/rapidata_client/selection/capped_selection.py,sha256=iWhbM1LcayhgFm7oKADXCaKHGdiQIupI0jbYuuEVM2A,1184
 rapidata/rapidata_client/selection/conditional_validation_selection.py,sha256=OcNYSBi19vIcy2bLDmj9cv-gg5LFSvdjc3tooV0Z7Oc,2842
 rapidata/rapidata_client/selection/demographic_selection.py,sha256=l4vnNbzlf9ED6BKqN4k5cZXShkXu9L1C5DtO78Vwr5M,1454
+rapidata/rapidata_client/selection/effort_selection.py,sha256=uS8ctK2o-40Blu02jB5w7i8WtRSw21LhXszkkq30pM8,858
 rapidata/rapidata_client/selection/labeling_selection.py,sha256=0X8DJHgwvgwekEbzVxWPyzZ1QAPcULZNDjfLQYUlcLM,1348
 rapidata/rapidata_client/selection/rapidata_selections.py,sha256=lgwRivdzSnCri3K-Z-ngqR5RXwTl7iYuKTRpuyl5UMY,1853
 rapidata/rapidata_client/selection/retrieval_modes.py,sha256=J2jzPEJ4wdllm_RnU_FYPh3eO3xeZS7QUk-NXgTB2u4,668
 rapidata/rapidata_client/selection/shuffling_selection.py,sha256=FzOp7mnBLxNzM5at_-935wd77IHyWnFR1f8uqokiMOg,1201
 rapidata/rapidata_client/selection/static_selection.py,sha256=-RWD3ChPe7-J31Shmah9JBNCgc5a16C5NOUl1f8tZCM,680
 rapidata/rapidata_client/selection/validation_selection.py,sha256=sedeIa8lpXVXKtFJA9IDeRvo9A1Ne4ZGcepaWDUGhCU,851
-rapidata/rapidata_client/settings/__init__.py,sha256=DTEAT_YykwodZJXqKYOtWRwimLCA-Jxo0F0d-H6A3vM,458
+rapidata/rapidata_client/settings/__init__.py,sha256=V5KW-_ffp4oZhwAMweRV0f2jSpXOb7oYYFiAwpsrkg4,507
 rapidata/rapidata_client/settings/_rapidata_setting.py,sha256=MD5JhhogSLLrjFKjvL3JhMszOMCygyqLF-st0EwMSkw,352
 rapidata/rapidata_client/settings/alert_on_fast_response.py,sha256=qW9Kv3mfl_V5NkPvKzg1Wz6QCRU-CmcAn1yaWCYI16U,989
+rapidata/rapidata_client/settings/allow_neither_both.py,sha256=PMt1_n_kfERMoeENarx0uZqyGD2OmGONWognTmdvAgw,606
 rapidata/rapidata_client/settings/custom_setting.py,sha256=TIbbedz98bfoIxQyiSMRYnQAaRFI1J7dYoTY89uNIoA,579
 rapidata/rapidata_client/settings/free_text_minimum_characters.py,sha256=-kKO36Z7SsK9hGNEaVeHd1MLMqY6lkAuVAm3ST0y_gs,729
 rapidata/rapidata_client/settings/models/__init__.py,sha256=IW7OuWg7xWIwFYrMAOX5N0HGGcqE6fFpgYin3vWRkOU,71
 rapidata/rapidata_client/settings/models/translation_behaviour_options.py,sha256=7eR_Tk_WouW2g466dXdwuDuYdSxSAeaVP4KR3mW9q7A,479
 rapidata/rapidata_client/settings/no_shuffle.py,sha256=9LNx4LptHZsQxamNYeY6lz9uakefyzVWM09tr76yx18,684
 rapidata/rapidata_client/settings/play_video_until_the_end.py,sha256=LLHx2_72k5ZqqG4opdlOS0HpiVmqbq0Inuxb5iVvrW8,747
-rapidata/rapidata_client/settings/rapidata_settings.py,sha256=wXsF5VIJmD2gq7j86rGXp2ck1ucz9jh7UGJXAyMZ7AM,1594
+rapidata/rapidata_client/settings/rapidata_settings.py,sha256=qxwktsrrmGHqd6rKPMEZj0CK9k1SUtNjM2kedPaHTVs,1829
 rapidata/rapidata_client/settings/translation_behaviour.py,sha256=i9n_H0eKJyKW6m3MKH_Cm1XEKWVEWsAV_79xGmGIC-4,742
 rapidata/rapidata_client/validation/__init__.py,sha256=s5wHVtcJkncXSFuL9I0zNwccNOKpWAqxqUjkeohzi2E,24
 rapidata/rapidata_client/validation/rapidata_validation_set.py,sha256=PgsutRTpzJ_e_H-xJxibSKHTskcM43aNwnm9dFhpMpw,1812
@@ -562,8 +567,8 @@ rapidata/rapidata_client/workflow/_timestamp_workflow.py,sha256=tPi2zu1-SlE_ppbG
 rapidata/service/__init__.py,sha256=s9bS1AJZaWIhLtJX_ZA40_CK39rAAkwdAmymTMbeWl4,68
 rapidata/service/credential_manager.py,sha256=pUEEtp6VrFWYhfUUtyqmS0AlRqe2Y0kFkY6o22IT4KM,8682
 rapidata/service/local_file_service.py,sha256=pgorvlWcx52Uh3cEG6VrdMK_t__7dacQ_5AnfY14BW8,877
-rapidata/service/openapi_service.py,sha256=J07TB4P3cz9KCU7k_fwuMQwGXlq_nJx_m1_xHbZoCg0,4867
-rapidata-2.27.6.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-rapidata-2.27.6.dist-info/METADATA,sha256=v6CiMxdWsWEeWVkPHVlGXs1JsDZqXZpbeTNQAAg-Jm0,1264
-rapidata-2.27.6.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-rapidata-2.27.6.dist-info/RECORD,,
+rapidata/service/openapi_service.py,sha256=yW8F6UcGs5VCzB4k3-FSBToojjpgJ4Y_LiEgZZ7uprw,5049
+rapidata-2.28.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+rapidata-2.28.0.dist-info/METADATA,sha256=w2LFXQjfXsycagrDvCzvpFuezxfIKlOCO7-MK1oBZgk,1264
+rapidata-2.28.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+rapidata-2.28.0.dist-info/RECORD,,