rapidata 2.33.1__tar.gz → 2.33.2__tar.gz

{rapidata-2.33.1 → rapidata-2.33.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rapidata
-Version: 2.33.1
+Version: 2.33.2
 Summary: Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way.
 License: Apache-2.0
 Author: Rapidata AG

{rapidata-2.33.1 → rapidata-2.33.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rapidata"
-version = "2.33.1"
+version = "2.33.2"
 description = "Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way."
 authors = ["Rapidata AG <info@rapidata.ai>"]
 license = "Apache-2.0"

{rapidata-2.33.1 → rapidata-2.33.2}/src/rapidata/__init__.py

@@ -1,10 +1,10 @@
-__version__ = "2.33.1"
+__version__ = "2.33.2"
 
 from .rapidata_client import (
     RapidataClient,
     DemographicSelection,
     LabelingSelection,
-    EffortEstimationSelection,
+    EffortSelection,
     RetrievalMode,
     ValidationSelection,
     ConditionalValidationSelection,

{rapidata-2.33.1 → rapidata-2.33.2}/src/rapidata/rapidata_client/__init__.py

@@ -7,7 +7,7 @@ from .selection import (
     CappedSelection,
     ShufflingSelection,
     RetrievalMode,
-    EffortEstimationSelection,
+    EffortSelection,
 )
 from .datapoints import Datapoint
 from .datapoints.metadata import (

rapidata-2.33.2/src/rapidata/rapidata_client/benchmark/participant/_participant.py

@@ -0,0 +1,102 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
+
+from rapidata.rapidata_client.datapoints.assets import MediaAsset
+from rapidata.rapidata_client.logging import logger
+from rapidata.rapidata_client.logging.output_manager import RapidataOutputManager
+from rapidata.api_client.models.create_sample_model import CreateSampleModel
+from rapidata.service.openapi_service import OpenAPIService
+
+
+class BenchmarkParticipant:
+    def __init__(self, name: str, id: str, openapi_service: OpenAPIService):
+        self.name = name
+        self.id = id
+        self.__openapi_service = openapi_service
+
+    def _process_single_sample_upload(
+        self,
+        asset: MediaAsset,
+        identifier: str,
+    ) -> tuple[MediaAsset | None, MediaAsset | None]:
+        """
+        Process single sample upload with retry logic and error tracking.
+        
+        Args:
+            asset: MediaAsset to upload
+            identifier: Identifier for the sample
+            
+        Returns:
+            tuple[MediaAsset | None, MediaAsset | None]: (successful_asset, failed_asset)
+        """
+        if asset.is_local():
+            files = [asset.to_file()]
+            urls = []
+        else:
+            files = []
+            urls = [asset.path]
+
+        last_exception = None
+        try:
+            self.__openapi_service.participant_api.participant_participant_id_sample_post(
+                participant_id=self.id,
+                model=CreateSampleModel(
+                    identifier=identifier
+                ),
+                files=files,
+                urls=urls
+            )
+            
+            return asset, None
+            
+        except Exception as e:
+            last_exception = e
+
+        logger.error(f"Upload failed for {identifier}. Error: {str(last_exception)}")
+        return None, asset
+
+    def upload_media(
+        self,
+        assets: list[MediaAsset],
+        identifiers: list[str],
+        max_workers: int = 10,
+    ) -> tuple[list[MediaAsset], list[MediaAsset]]:
+        """
+        Upload samples concurrently with proper error handling and progress tracking.
+        
+        Args:
+            assets: List of MediaAsset objects to upload
+            identifiers: List of identifiers matching the assets
+            max_workers: Maximum number of concurrent upload workers
+            
+        Returns:
+            tuple[list[str], list[str]]: Lists of successful and failed identifiers
+        """
+        successful_uploads: list[MediaAsset] = []
+        failed_uploads: list[MediaAsset] = []
+        total_uploads = len(assets)
+        
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = [
+                executor.submit(
+                    self._process_single_sample_upload,
+                    asset,
+                    identifier,
+                )
+                for asset, identifier in zip(assets, identifiers)
+            ]
+            
+            with tqdm(total=total_uploads, desc="Uploading media", disable=RapidataOutputManager.silent_mode) as pbar:
+                for future in as_completed(futures):
+                    try:
+                        successful_id, failed_id = future.result()
+                        if successful_id:
+                            successful_uploads.append(successful_id)
+                        if failed_id:
+                            failed_uploads.append(failed_id)
+                    except Exception as e:
+                        logger.error(f"Future execution failed: {str(e)}")
+                        
+                    pbar.update(1)
+        
+        return successful_uploads, failed_uploads

{rapidata-2.33.1 → rapidata-2.33.2}/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py

@@ -11,14 +11,13 @@ from rapidata.api_client.models.url_asset_input import UrlAssetInput
 from rapidata.api_client.models.file_asset_model import FileAssetModel
 from rapidata.api_client.models.source_url_metadata_model import SourceUrlMetadataModel
 
+
+from rapidata.rapidata_client.benchmark.participant._participant import BenchmarkParticipant
 from rapidata.rapidata_client.logging import logger
 from rapidata.service.openapi_service import OpenAPIService
 
 from rapidata.rapidata_client.benchmark.leaderboard.rapidata_leaderboard import RapidataLeaderboard
-from rapidata.rapidata_client.datapoints.metadata import PromptIdentifierMetadata
 from rapidata.rapidata_client.datapoints.assets import MediaAsset
-from rapidata.rapidata_client.order._rapidata_dataset import RapidataDataset
-from rapidata.rapidata_client.datapoints.datapoint import Datapoint
 
 class RapidataBenchmark:
     """
@@ -250,7 +249,7 @@ class RapidataBenchmark:
             leaderboard_result.id,
             self.__openapi_service
         )
-    
+
     def evaluate_model(self, name: str, media: list[str], identifiers: list[str]) -> None:
         """
         Evaluates a model on the benchmark across all leaderboards.
@@ -272,11 +271,9 @@ class RapidataBenchmark:
 \nTo see the prompts that are associated with the identifiers, use the prompts property.")
         
         # happens before the creation of the participant to ensure all media paths are valid
-        assets = []
-        prompts_metadata: list[list[PromptIdentifierMetadata]] = []
-        for media_path, identifier in zip(media, identifiers):
+        assets: list[MediaAsset] = []
+        for media_path in media:
             assets.append(MediaAsset(media_path))
-            prompts_metadata.append([PromptIdentifierMetadata(identifier=identifier)])
 
         participant_result = self.__openapi_service.benchmark_api.benchmark_benchmark_id_participants_post(
             benchmark_id=self.id,
@@ -285,22 +282,27 @@ class RapidataBenchmark:
             )
         )
 
-        dataset = RapidataDataset(participant_result.dataset_id, self.__openapi_service)
-        
-        try:
-            dataset.add_datapoints([Datapoint(asset=asset, metadata=metadata) for asset, metadata in zip(assets, prompts_metadata)])
-        except Exception as e:
-            logger.warning(f"An error occurred while adding datapoints to the dataset: {e}")
-            upload_progress = self.__openapi_service.dataset_api.dataset_dataset_id_progress_get(
-                dataset_id=dataset.id
-            )
-            if upload_progress.ready == 0:
-                raise RuntimeError("None of the media was uploaded successfully. Please check the media paths and try again.")
-            
-            logger.warning(f"{upload_progress.failed} datapoints failed to upload. \n{upload_progress.ready} datapoints were uploaded successfully. \nEvaluation will continue with the uploaded datapoints.")
+        logger.info(f"Participant created: {participant_result.participant_id}")
 
-        self.__openapi_service.benchmark_api.benchmark_benchmark_id_participants_participant_id_submit_post(
-            benchmark_id=self.id,
+        participant = BenchmarkParticipant(name, participant_result.participant_id, self.__openapi_service)
+
+        successful_uploads, failed_uploads = participant.upload_media(
+            assets,
+            identifiers,
+        )
+
+        total_uploads = len(assets)
+        success_rate = (len(successful_uploads) / total_uploads * 100) if total_uploads > 0 else 0
+        logger.info(f"Upload complete: {len(successful_uploads)} successful, {len(failed_uploads)} failed ({success_rate:.1f}% success rate)")
+
+        if failed_uploads:
+            logger.error(f"Failed uploads for media: {[asset.path for asset in failed_uploads]}")
+            logger.warning("Some uploads failed. The model evaluation may be incomplete.")
+
+        if len(successful_uploads) == 0:
+            raise RuntimeError("No uploads were successful. The model evaluation will not be completed.")
+
+        self.__openapi_service.participant_api.participants_participant_id_submit_post(
             participant_id=participant_result.participant_id
         )
 

{rapidata-2.33.1 → rapidata-2.33.2}/src/rapidata/rapidata_client/selection/__init__.py

@@ -8,4 +8,4 @@ from .shuffling_selection import ShufflingSelection
 from .ab_test_selection import AbTestSelection
 from .static_selection import StaticSelection
 from .retrieval_modes import RetrievalMode
-from .effort_selection import EffortEstimationSelection
+from .effort_selection import EffortSelection

{rapidata-2.33.1 → rapidata-2.33.2}/src/rapidata/rapidata_client/selection/effort_selection.py

@@ -3,9 +3,16 @@ from rapidata.api_client.models.effort_capped_selection import EffortCappedSelec
 from rapidata.rapidata_client.selection.retrieval_modes import RetrievalMode
 
 
-class EffortEstimationSelection(RapidataSelection):
-    
+class EffortSelection(RapidataSelection):
+    """
+    With this selection you can define the effort budget you have for a task.
+    As an example, you have a task that takes 10 seconds to complete. The effort budget would be 10.
 
+    Args:
+        effort_budget (int): The effort budget for the task.
+        retrieval_mode (RetrievalMode): The retrieval mode for the task.
+        max_iterations (int | None): The maximum number of iterations for the task.
+    """
     def __init__(self, effort_budget: int, retrieval_mode: RetrievalMode = RetrievalMode.Shuffled, max_iterations: int | None = None):
         self.effort_budget = effort_budget
         self.retrieval_mode = retrieval_mode

{rapidata-2.33.1 → rapidata-2.33.2}/src/rapidata/service/openapi_service.py

@@ -10,6 +10,7 @@ from rapidata.api_client.api.rapid_api import RapidApi
 from rapidata.api_client.api.leaderboard_api import LeaderboardApi
 from rapidata.api_client.api.validation_set_api import ValidationSetApi
 from rapidata.api_client.api.workflow_api import WorkflowApi
+from rapidata.api_client.api.participant_api import ParticipantApi
 from rapidata.api_client.configuration import Configuration
 from rapidata.service.credential_manager import CredentialManager
 from rapidata.rapidata_client.api.rapidata_exception import RapidataApiClient
@@ -117,6 +118,10 @@ class OpenAPIService:
     @property
     def benchmark_api(self) -> BenchmarkApi:
         return BenchmarkApi(self.api_client)
+    
+    @property
+    def participant_api(self) -> ParticipantApi:
+        return ParticipantApi(self.api_client)
 
     def _get_rapidata_package_version(self):
         """