rapidata 2.33.1__py3-none-any.whl → 2.34.0__py3-none-any.whl

rapidata/api_client/models/tags_by_benchmark_result.py

@@ -0,0 +1,87 @@
+# coding: utf-8
+
+"""
+    Rapidata.Dataset
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: v1
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+from pydantic import BaseModel, ConfigDict, StrictStr
+from typing import Any, ClassVar, Dict, List
+from typing import Optional, Set
+from typing_extensions import Self
+
+class TagsByBenchmarkResult(BaseModel):
+    """
+    TagsByBenchmarkResult
+    """ # noqa: E501
+    tags: List[StrictStr]
+    __properties: ClassVar[List[str]] = ["tags"]
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        validate_assignment=True,
+        protected_namespaces=(),
+    )
+
+
+    def to_str(self) -> str:
+        """Returns the string representation of the model using alias"""
+        return pprint.pformat(self.model_dump(by_alias=True))
+
+    def to_json(self) -> str:
+        """Returns the JSON representation of the model using alias"""
+        # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_json(cls, json_str: str) -> Optional[Self]:
+        """Create an instance of TagsByBenchmarkResult from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return the dictionary representation of the model using alias.
+
+        This has the following differences from calling pydantic's
+        `self.model_dump(by_alias=True)`:
+
+        * `None` is only added to the output dict for nullable fields that
+          were set at model initialization. Other fields with value `None`
+          are ignored.
+        """
+        excluded_fields: Set[str] = set([
+        ])
+
+        _dict = self.model_dump(
+            by_alias=True,
+            exclude=excluded_fields,
+            exclude_none=True,
+        )
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
+        """Create an instance of TagsByBenchmarkResult from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return cls.model_validate(obj)
+
+        _obj = cls.model_validate({
+            "tags": obj.get("tags")
+        })
+        return _obj
+
+

rapidata/api_client/models/update_prompt_tags_model.py

@@ -0,0 +1,87 @@
+# coding: utf-8
+
+"""
+    Rapidata.Dataset
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: v1
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+from pydantic import BaseModel, ConfigDict, Field, StrictStr
+from typing import Any, ClassVar, Dict, List
+from typing import Optional, Set
+from typing_extensions import Self
+
+class UpdatePromptTagsModel(BaseModel):
+    """
+    The model for updating prompt tags.
+    """ # noqa: E501
+    tags: List[StrictStr] = Field(description="The list of tags to be associated with the prompt.")
+    __properties: ClassVar[List[str]] = ["tags"]
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        validate_assignment=True,
+        protected_namespaces=(),
+    )
+
+
+    def to_str(self) -> str:
+        """Returns the string representation of the model using alias"""
+        return pprint.pformat(self.model_dump(by_alias=True))
+
+    def to_json(self) -> str:
+        """Returns the JSON representation of the model using alias"""
+        # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_json(cls, json_str: str) -> Optional[Self]:
+        """Create an instance of UpdatePromptTagsModel from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return the dictionary representation of the model using alias.
+
+        This has the following differences from calling pydantic's
+        `self.model_dump(by_alias=True)`:
+
+        * `None` is only added to the output dict for nullable fields that
+          were set at model initialization. Other fields with value `None`
+          are ignored.
+        """
+        excluded_fields: Set[str] = set([
+        ])
+
+        _dict = self.model_dump(
+            by_alias=True,
+            exclude=excluded_fields,
+            exclude_none=True,
+        )
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
+        """Create an instance of UpdatePromptTagsModel from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return cls.model_validate(obj)
+
+        _obj = cls.model_validate({
+            "tags": obj.get("tags")
+        })
+        return _obj
+
+

rapidata/api_client_README.md

@@ -135,7 +135,6 @@ Class | Method | HTTP request | Description
 *LeaderboardApi* | [**leaderboard_leaderboard_id_participants_post**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_leaderboard_id_participants_post) | **POST** /leaderboard/{leaderboardId}/participants | Creates a participant in a leaderboard.
 *LeaderboardApi* | [**leaderboard_leaderboard_id_prompts_get**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_leaderboard_id_prompts_get) | **GET** /leaderboard/{leaderboardId}/prompts | returns the paged prompts of a leaderboard by its ID.
 *LeaderboardApi* | [**leaderboard_leaderboard_id_prompts_post**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_leaderboard_id_prompts_post) | **POST** /leaderboard/{leaderboardId}/prompts | adds a new prompt to a leaderboard.
-*LeaderboardApi* | [**leaderboard_leaderboard_id_refresh_post**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_leaderboard_id_refresh_post) | **POST** /leaderboard/{leaderboardId}/refresh | This will force an update to all standings of a leaderboard. this could happen if the recorded matches and scores are out of sync
 *LeaderboardApi* | [**leaderboard_leaderboard_id_runs_get**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_leaderboard_id_runs_get) | **GET** /leaderboard/{leaderboardId}/runs | Gets the runs related to a leaderboard
 *LeaderboardApi* | [**leaderboard_leaderboard_id_standings_get**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_leaderboard_id_standings_get) | **GET** /leaderboard/{leaderboardId}/standings | queries all the participants connected to leaderboard by its ID.
 *LeaderboardApi* | [**leaderboard_post**](rapidata/api_client/docs/LeaderboardApi.md#leaderboard_post) | **POST** /leaderboard | Creates a new leaderboard with the specified name and criteria.
@@ -178,6 +177,7 @@ Class | Method | HTTP request | Description
 *PipelineApi* | [**pipeline_pipeline_id_get**](rapidata/api_client/docs/PipelineApi.md#pipeline_pipeline_id_get) | **GET** /pipeline/{pipelineId} | Gets a pipeline by its id.
 *PipelineApi* | [**pipeline_pipeline_id_preliminary_download_post**](rapidata/api_client/docs/PipelineApi.md#pipeline_pipeline_id_preliminary_download_post) | **POST** /pipeline/{pipelineId}/preliminary-download | Initiates a preliminary download of the pipeline.
 *PipelineApi* | [**pipeline_preliminary_download_preliminary_download_id_get**](rapidata/api_client/docs/PipelineApi.md#pipeline_preliminary_download_preliminary_download_id_get) | **GET** /pipeline/preliminary-download/{preliminaryDownloadId} | Gets the preliminary download.
+*PromptApi* | [**benchmark_prompt_prompt_id_tags_put**](rapidata/api_client/docs/PromptApi.md#benchmark_prompt_prompt_id_tags_put) | **PUT** /benchmark-prompt/{promptId}/tags | Updates the tags associated with a prompt.
 *RapidataIdentityAPIApi* | [**root_get**](rapidata/api_client/docs/RapidataIdentityAPIApi.md#root_get) | **GET** / | 
 *SimpleWorkflowApi* | [**workflow_simple_workflow_id_results_get**](rapidata/api_client/docs/SimpleWorkflowApi.md#workflow_simple_workflow_id_results_get) | **GET** /workflow/simple/{workflowId}/results | Get the result overview for a simple workflow.
 *UserInfoApi* | [**connect_userinfo_get**](rapidata/api_client/docs/UserInfoApi.md#connect_userinfo_get) | **GET** /connect/userinfo | Retrieves information about the authenticated user.
@@ -197,7 +197,7 @@ Class | Method | HTTP request | Description
 *ValidationSetApi* | [**validation_set_validation_set_id_rapid_texts_post**](rapidata/api_client/docs/ValidationSetApi.md#validation_set_validation_set_id_rapid_texts_post) | **POST** /validation-set/{validationSetId}/rapid/texts | Adds a new validation rapid to the specified validation set using text sources to create the assets.
 *ValidationSetApi* | [**validation_set_validation_set_id_rapids_get**](rapidata/api_client/docs/ValidationSetApi.md#validation_set_validation_set_id_rapids_get) | **GET** /validation-set/{validationSetId}/rapids | Queries the validation rapids for a specific validation set.
 *ValidationSetApi* | [**validation_set_validation_set_id_shouldalert_patch**](rapidata/api_client/docs/ValidationSetApi.md#validation_set_validation_set_id_shouldalert_patch) | **PATCH** /validation-set/{validationSetId}/shouldalert | Updates the dimensions of all rapids within a validation set.
-*ValidationSetApi* | [**validation_set_validation_set_id_shouldalert_put**](rapidata/api_client/docs/ValidationSetApi.md#validation_set_validation_set_id_shouldalert_put) | **PUT** /validation-set/{validationSetId}/shouldalert | Updates the dimensions of all rapids within a validation set.
+*ValidationSetApi* | [**validation_set_validation_set_id_shouldalert_put**](rapidata/api_client/docs/ValidationSetApi.md#validation_set_validation_set_id_shouldalert_put) | **PUT** /validation-set/{validationSetId}/shouldalert | Updates the  of all rapshouldAlert property of all rapids within a validation set.
 *ValidationSetApi* | [**validation_set_zip_compare_post**](rapidata/api_client/docs/ValidationSetApi.md#validation_set_zip_compare_post) | **POST** /validation-set/zip/compare | Imports a compare validation set from a zip file.
 *ValidationSetApi* | [**validation_set_zip_post**](rapidata/api_client/docs/ValidationSetApi.md#validation_set_zip_post) | **POST** /validation-set/zip | Imports a validation set from a zip file.
 *ValidationSetApi* | [**validation_sets_available_get**](rapidata/api_client/docs/ValidationSetApi.md#validation_sets_available_get) | **GET** /validation-sets/available | Gets the available validation sets for the current user.
@@ -529,6 +529,7 @@ Class | Method | HTTP request | Description
  - [StandingByLeaderboard](rapidata/api_client/docs/StandingByLeaderboard.md)
  - [StandingByLeaderboardPagedResult](rapidata/api_client/docs/StandingByLeaderboardPagedResult.md)
  - [StandingStatus](rapidata/api_client/docs/StandingStatus.md)
+ - [StandingsByLeaderboardResult](rapidata/api_client/docs/StandingsByLeaderboardResult.md)
  - [StaticSelection](rapidata/api_client/docs/StaticSelection.md)
  - [StickyState](rapidata/api_client/docs/StickyState.md)
  - [StreamFileWrapper](rapidata/api_client/docs/StreamFileWrapper.md)
@@ -540,6 +541,7 @@ Class | Method | HTTP request | Description
  - [SubmitParticipantResult](rapidata/api_client/docs/SubmitParticipantResult.md)
  - [SubmitPromptModel](rapidata/api_client/docs/SubmitPromptModel.md)
  - [SubmitPromptModelPromptAsset](rapidata/api_client/docs/SubmitPromptModelPromptAsset.md)
+ - [TagsByBenchmarkResult](rapidata/api_client/docs/TagsByBenchmarkResult.md)
  - [TextAsset](rapidata/api_client/docs/TextAsset.md)
  - [TextAssetInput](rapidata/api_client/docs/TextAssetInput.md)
  - [TextAssetModel](rapidata/api_client/docs/TextAssetModel.md)
@@ -563,6 +565,7 @@ Class | Method | HTTP request | Description
  - [UpdateLeaderboardNameModel](rapidata/api_client/docs/UpdateLeaderboardNameModel.md)
  - [UpdateOrderNameModel](rapidata/api_client/docs/UpdateOrderNameModel.md)
  - [UpdateParticipantNameModel](rapidata/api_client/docs/UpdateParticipantNameModel.md)
+ - [UpdatePromptTagsModel](rapidata/api_client/docs/UpdatePromptTagsModel.md)
  - [UpdateShouldAlertModel](rapidata/api_client/docs/UpdateShouldAlertModel.md)
  - [UpdateValidationRapidModel](rapidata/api_client/docs/UpdateValidationRapidModel.md)
  - [UpdateValidationRapidModelTruth](rapidata/api_client/docs/UpdateValidationRapidModelTruth.md)

rapidata/rapidata_client/__init__.py

@@ -7,7 +7,7 @@ from .selection import (
     CappedSelection,
     ShufflingSelection,
     RetrievalMode,
-    EffortEstimationSelection,
+    EffortSelection,
 )
 from .datapoints import Datapoint
 from .datapoints.metadata import (

rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py

@@ -1,8 +1,5 @@
 import pandas as pd
-
-from rapidata.api_client.models.query_model import QueryModel
-from rapidata.api_client.models.page_info import PageInfo
-from rapidata.api_client.models.sort_criterion import SortCriterion
+from typing import Optional
 
 from rapidata.service.openapi_service import OpenAPIService
 
@@ -89,16 +86,22 @@ class RapidataLeaderboard:
         """
         return self.__name
 
-    def get_standings(self) -> pd.DataFrame:
+    def get_standings(self, tags: Optional[list[str]] = None) -> pd.DataFrame:
         """
         Returns the standings of the leaderboard.
+
+        Args:
+            tags: The matchups with these tags should be used to create the standings.
+                If tags are None, all matchups will be considered.
+                If tags are empty, no matchups will be considered.
+
+        Returns:
+            A pandas DataFrame containing the standings of the leaderboard.
         """
+
         participants = self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_standings_get(
             leaderboard_id=self.id,
-            request=QueryModel(
-                page=PageInfo(index=1, size=1000),
-                sortCriteria=[SortCriterion(direction="Desc", propertyName="Score")]
-            )
+            tags=tags
         )
 
         standings = []

rapidata/rapidata_client/benchmark/participant/_participant.py

@@ -0,0 +1,102 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
+
+from rapidata.rapidata_client.datapoints.assets import MediaAsset
+from rapidata.rapidata_client.logging import logger
+from rapidata.rapidata_client.logging.output_manager import RapidataOutputManager
+from rapidata.api_client.models.create_sample_model import CreateSampleModel
+from rapidata.service.openapi_service import OpenAPIService
+
+
+class BenchmarkParticipant:
+    def __init__(self, name: str, id: str, openapi_service: OpenAPIService):
+        self.name = name
+        self.id = id
+        self.__openapi_service = openapi_service
+
+    def _process_single_sample_upload(
+        self,
+        asset: MediaAsset,
+        identifier: str,
+    ) -> tuple[MediaAsset | None, MediaAsset | None]:
+        """
+        Process single sample upload with retry logic and error tracking.
+        
+        Args:
+            asset: MediaAsset to upload
+            identifier: Identifier for the sample
+            
+        Returns:
+            tuple[MediaAsset | None, MediaAsset | None]: (successful_asset, failed_asset)
+        """
+        if asset.is_local():
+            files = [asset.to_file()]
+            urls = []
+        else:
+            files = []
+            urls = [asset.path]
+
+        last_exception = None
+        try:
+            self.__openapi_service.participant_api.participant_participant_id_sample_post(
+                participant_id=self.id,
+                model=CreateSampleModel(
+                    identifier=identifier
+                ),
+                files=files,
+                urls=urls
+            )
+            
+            return asset, None
+            
+        except Exception as e:
+            last_exception = e
+
+        logger.error(f"Upload failed for {identifier}. Error: {str(last_exception)}")
+        return None, asset
+
+    def upload_media(
+        self,
+        assets: list[MediaAsset],
+        identifiers: list[str],
+        max_workers: int = 10,
+    ) -> tuple[list[MediaAsset], list[MediaAsset]]:
+        """
+        Upload samples concurrently with proper error handling and progress tracking.
+        
+        Args:
+            assets: List of MediaAsset objects to upload
+            identifiers: List of identifiers matching the assets
+            max_workers: Maximum number of concurrent upload workers
+            
+        Returns:
+            tuple[list[str], list[str]]: Lists of successful and failed identifiers
+        """
+        successful_uploads: list[MediaAsset] = []
+        failed_uploads: list[MediaAsset] = []
+        total_uploads = len(assets)
+        
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = [
+                executor.submit(
+                    self._process_single_sample_upload,
+                    asset,
+                    identifier,
+                )
+                for asset, identifier in zip(assets, identifiers)
+            ]
+            
+            with tqdm(total=total_uploads, desc="Uploading media", disable=RapidataOutputManager.silent_mode) as pbar:
+                for future in as_completed(futures):
+                    try:
+                        successful_id, failed_id = future.result()
+                        if successful_id:
+                            successful_uploads.append(successful_id)
+                        if failed_id:
+                            failed_uploads.append(failed_id)
+                    except Exception as e:
+                        logger.error(f"Future execution failed: {str(e)}")
+                        
+                    pbar.update(1)
+        
+        return successful_uploads, failed_uploads

rapidata/rapidata_client/benchmark/rapidata_benchmark.py

@@ -1,4 +1,5 @@
 import re
+from typing import Optional
 from rapidata.api_client.models.root_filter import RootFilter
 from rapidata.api_client.models.filter import Filter
 from rapidata.api_client.models.query_model import QueryModel
@@ -11,14 +12,13 @@ from rapidata.api_client.models.url_asset_input import UrlAssetInput
 from rapidata.api_client.models.file_asset_model import FileAssetModel
 from rapidata.api_client.models.source_url_metadata_model import SourceUrlMetadataModel
 
+
+from rapidata.rapidata_client.benchmark.participant._participant import BenchmarkParticipant
 from rapidata.rapidata_client.logging import logger
 from rapidata.service.openapi_service import OpenAPIService
 
 from rapidata.rapidata_client.benchmark.leaderboard.rapidata_leaderboard import RapidataLeaderboard
-from rapidata.rapidata_client.datapoints.metadata import PromptIdentifierMetadata
 from rapidata.rapidata_client.datapoints.assets import MediaAsset
-from rapidata.rapidata_client.order._rapidata_dataset import RapidataDataset
-from rapidata.rapidata_client.datapoints.datapoint import Datapoint
 
 class RapidataBenchmark:
     """
@@ -39,7 +39,8 @@ class RapidataBenchmark:
         self.__prompt_assets: list[str | None] = []
         self.__leaderboards: list[RapidataLeaderboard] = []
         self.__identifiers: list[str] = []
-
+        self.__tags: list[list[str]] = []
+    
     def __instantiate_prompts(self) -> None:
         current_page = 1
         total_pages = None
@@ -70,7 +71,8 @@ class RapidataBenchmark:
                     source_url = prompt.prompt_asset.actual_instance.metadata["sourceUrl"].actual_instance
                     assert isinstance(source_url, SourceUrlMetadataModel)
                     self.__prompt_assets.append(source_url.url)
-            
+
+                self.__tags.append(prompt.tags)
             if current_page >= total_pages:
                 break
                 
@@ -104,6 +106,15 @@ class RapidataBenchmark:
         return self.__prompt_assets
     
     @property
+    def tags(self) -> list[list[str]]:
+        """
+        Returns the tags that are registered for the benchmark.
+        """
+        if not self.__tags:
+            self.__instantiate_prompts()
+        
+        return self.__tags
+    
     def leaderboards(self) -> list[RapidataLeaderboard]:
         """
         Returns the leaderboards that are registered for the benchmark.
@@ -152,7 +163,7 @@ class RapidataBenchmark:
                 
         return self.__leaderboards
     
-    def add_prompt(self, identifier: str, prompt: str | None = None, asset: str | None = None):
+    def add_prompt(self, identifier: str, prompt: str | None = None, asset: str | None = None, tags: Optional[list[str]] = None):
         """
         Adds a prompt to the benchmark.
 
@@ -160,7 +171,11 @@ class RapidataBenchmark:
             identifier: The identifier of the prompt/asset that will be used to match up the media.
             prompt: The prompt that will be used to evaluate the model.
             asset: The asset that will be used to evaluate the model. Provided as a link to the asset.
+            tags: The tags can be used to filter the leaderboard results. They will NOT be shown to the users.
         """
+        if tags is None:
+            tags = []
+
         if not isinstance(identifier, str):
             raise ValueError("Identifier must be a string.")
         
@@ -179,8 +194,12 @@ class RapidataBenchmark:
         if asset is not None and not re.match(r'^https?://', asset):
             raise ValueError("Asset must be a link to the asset.")
         
+        if tags is not None and (not isinstance(tags, list) or not all(isinstance(tag, str) for tag in tags)):
+            raise ValueError("Tags must be a list of strings.")
+        
         self.__identifiers.append(identifier)
 
+        self.__tags.append(tags)
         self.__prompts.append(prompt)
         self.__prompt_assets.append(asset)
 
@@ -194,7 +213,8 @@ class RapidataBenchmark:
                         _t="UrlAssetInput",
                         url=asset
                     )
-                ) if asset is not None else None
+                ) if asset is not None else None,
+                tags=tags
             )
         )
 
@@ -250,7 +270,7 @@ class RapidataBenchmark:
             leaderboard_result.id,
             self.__openapi_service
         )
-    
+
     def evaluate_model(self, name: str, media: list[str], identifiers: list[str]) -> None:
         """
         Evaluates a model on the benchmark across all leaderboards.
@@ -272,11 +292,9 @@ class RapidataBenchmark:
 \nTo see the prompts that are associated with the identifiers, use the prompts property.")
         
         # happens before the creation of the participant to ensure all media paths are valid
-        assets = []
-        prompts_metadata: list[list[PromptIdentifierMetadata]] = []
-        for media_path, identifier in zip(media, identifiers):
+        assets: list[MediaAsset] = []
+        for media_path in media:
             assets.append(MediaAsset(media_path))
-            prompts_metadata.append([PromptIdentifierMetadata(identifier=identifier)])
 
         participant_result = self.__openapi_service.benchmark_api.benchmark_benchmark_id_participants_post(
             benchmark_id=self.id,
@@ -285,22 +303,27 @@ class RapidataBenchmark:
             )
         )
 
-        dataset = RapidataDataset(participant_result.dataset_id, self.__openapi_service)
-        
-        try:
-            dataset.add_datapoints([Datapoint(asset=asset, metadata=metadata) for asset, metadata in zip(assets, prompts_metadata)])
-        except Exception as e:
-            logger.warning(f"An error occurred while adding datapoints to the dataset: {e}")
-            upload_progress = self.__openapi_service.dataset_api.dataset_dataset_id_progress_get(
-                dataset_id=dataset.id
-            )
-            if upload_progress.ready == 0:
-                raise RuntimeError("None of the media was uploaded successfully. Please check the media paths and try again.")
-            
-            logger.warning(f"{upload_progress.failed} datapoints failed to upload. \n{upload_progress.ready} datapoints were uploaded successfully. \nEvaluation will continue with the uploaded datapoints.")
+        logger.info(f"Participant created: {participant_result.participant_id}")
 
-        self.__openapi_service.benchmark_api.benchmark_benchmark_id_participants_participant_id_submit_post(
-            benchmark_id=self.id,
+        participant = BenchmarkParticipant(name, participant_result.participant_id, self.__openapi_service)
+
+        successful_uploads, failed_uploads = participant.upload_media(
+            assets,
+            identifiers,
+        )
+
+        total_uploads = len(assets)
+        success_rate = (len(successful_uploads) / total_uploads * 100) if total_uploads > 0 else 0
+        logger.info(f"Upload complete: {len(successful_uploads)} successful, {len(failed_uploads)} failed ({success_rate:.1f}% success rate)")
+
+        if failed_uploads:
+            logger.error(f"Failed uploads for media: {[asset.path for asset in failed_uploads]}")
+            logger.warning("Some uploads failed. The model evaluation may be incomplete.")
+
+        if len(successful_uploads) == 0:
+            raise RuntimeError("No uploads were successful. The model evaluation will not be completed.")
+
+        self.__openapi_service.participant_api.participants_participant_id_submit_post(
             participant_id=participant_result.participant_id
         )
 

rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py

@@ -25,8 +25,9 @@ class RapidataBenchmarkManager:
     def create_new_benchmark(self, 
                              name: str,
                              identifiers: list[str],
-                             prompts: Optional[list[str]] = None,
-                             prompt_assets: Optional[list[str]] = None,
+                             prompts: Optional[list[str | None]] = None,
+                             prompt_assets: Optional[list[str | None]] = None,
+                             tags: Optional[list[list[str] | None]] = None,
                              ) -> RapidataBenchmark:
         """
         Creates a new benchmark with the given name, identifiers, prompts, and media assets.
@@ -37,15 +38,16 @@ class RapidataBenchmarkManager:
             name: The name of the benchmark.
             prompts: The prompts that will be registered for the benchmark.
             prompt_assets: The prompt assets that will be registered for the benchmark.
+            tags: The tags that will be associated with the prompts to use for filtering the leaderboard results. They will NOT be shown to the users.
         """
         if not isinstance(name, str):
             raise ValueError("Name must be a string.")
         
-        if prompts and (not isinstance(prompts, list) or not all(isinstance(prompt, str) for prompt in prompts)):
-            raise ValueError("Prompts must be a list of strings.")
+        if prompts and (not isinstance(prompts, list) or not all(isinstance(prompt, str) or prompt is None for prompt in prompts)):
+            raise ValueError("Prompts must be a list of strings or None.")
         
-        if prompt_assets and (not isinstance(prompt_assets, list) or not all(isinstance(asset, str) for asset in prompt_assets)):
-            raise ValueError("Media assets must be a list of strings.")
+        if prompt_assets and (not isinstance(prompt_assets, list) or not all(isinstance(asset, str) or asset is None for asset in prompt_assets)):
+            raise ValueError("Media assets must be a list of strings or None.")
         
         if not isinstance(identifiers, list) or not all(isinstance(identifier, str) for identifier in identifiers):
             raise ValueError("Identifiers must be a list of strings.")
@@ -61,6 +63,9 @@ class RapidataBenchmarkManager:
         
         if len(set(identifiers)) != len(identifiers):
             raise ValueError("Identifiers must be unique.")
+        
+        if tags and len(identifiers) != len(tags):
+            raise ValueError("Identifiers and tags must have the same length.")
 
         benchmark_result = self.__openapi_service.benchmark_api.benchmark_post(
             create_benchmark_model=CreateBenchmarkModel(
@@ -72,9 +77,10 @@ class RapidataBenchmarkManager:
 
         prompts_list = prompts if prompts is not None else [None] * len(identifiers)
         media_assets_list = prompt_assets if prompt_assets is not None else [None] * len(identifiers)
+        tags_list = tags if tags is not None else [None] * len(identifiers)
 
-        for identifier, prompt, asset in zip(identifiers, prompts_list, media_assets_list):
-            benchmark.add_prompt(identifier, prompt, asset)
+        for identifier, prompt, asset, tag in zip(identifiers, prompts_list, media_assets_list, tags_list):
+            benchmark.add_prompt(identifier, prompt, asset, tag)
 
         return benchmark
     

rapidata/rapidata_client/selection/__init__.py

@@ -8,4 +8,4 @@ from .shuffling_selection import ShufflingSelection
 from .ab_test_selection import AbTestSelection
 from .static_selection import StaticSelection
 from .retrieval_modes import RetrievalMode
-from .effort_selection import EffortEstimationSelection
+from .effort_selection import EffortSelection

rapidata/rapidata_client/selection/effort_selection.py

@@ -3,9 +3,16 @@ from rapidata.api_client.models.effort_capped_selection import EffortCappedSelec
 from rapidata.rapidata_client.selection.retrieval_modes import RetrievalMode
 
 
-class EffortEstimationSelection(RapidataSelection):
-    
+class EffortSelection(RapidataSelection):
+    """
+    With this selection you can define the effort budget you have for a task.
+    As an example, you have a task that takes 10 seconds to complete. The effort budget would be 10.
 
+    Args:
+        effort_budget (int): The effort budget for the task.
+        retrieval_mode (RetrievalMode): The retrieval mode for the task.
+        max_iterations (int | None): The maximum number of iterations for the task.
+    """
     def __init__(self, effort_budget: int, retrieval_mode: RetrievalMode = RetrievalMode.Shuffled, max_iterations: int | None = None):
         self.effort_budget = effort_budget
         self.retrieval_mode = retrieval_mode

rapidata/service/openapi_service.py

@@ -10,6 +10,7 @@ from rapidata.api_client.api.rapid_api import RapidApi
 from rapidata.api_client.api.leaderboard_api import LeaderboardApi
 from rapidata.api_client.api.validation_set_api import ValidationSetApi
 from rapidata.api_client.api.workflow_api import WorkflowApi
+from rapidata.api_client.api.participant_api import ParticipantApi
 from rapidata.api_client.configuration import Configuration
 from rapidata.service.credential_manager import CredentialManager
 from rapidata.rapidata_client.api.rapidata_exception import RapidataApiClient
@@ -117,6 +118,10 @@ class OpenAPIService:
     @property
     def benchmark_api(self) -> BenchmarkApi:
         return BenchmarkApi(self.api_client)
+    
+    @property
+    def participant_api(self) -> ParticipantApi:
+        return ParticipantApi(self.api_client)
 
     def _get_rapidata_package_version(self):
         """

{rapidata-2.33.1.dist-info → rapidata-2.34.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rapidata
-Version: 2.33.1
+Version: 2.34.0
 Summary: Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way.
 License: Apache-2.0
 Author: Rapidata AG