rapidata 2.35.1__py3-none-any.whl → 3.0.0a8__py3-none-any.whl

rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py

@@ -1,12 +1,15 @@
-import pandas as pd
-from typing import Literal, Optional
+import urllib.parse
+import webbrowser
+from colorama import Fore
+from typing import Literal, Optional, TYPE_CHECKING
 
-from rapidata.rapidata_client.logging import logger
-from rapidata.rapidata_client.benchmark._detail_mapper import DetailMapper
+from rapidata.rapidata_client.config import logger, managed_print, tracer
+from rapidata.rapidata_client.benchmark._detail_mapper import DetailMapper, LevelOfDetail
 from rapidata.service.openapi_service import OpenAPIService
-from rapidata.api_client.models.update_leaderboard_response_config_model import (
-    UpdateLeaderboardResponseConfigModel,
-)
+from rapidata.api_client.models.update_leaderboard_model import UpdateLeaderboardModel
+
+if TYPE_CHECKING:
+    import pandas as pd
 
 
 class RapidataLeaderboard:
@@ -32,6 +35,7 @@ class RapidataLeaderboard:
         inverse_ranking: bool,
         response_budget: int,
         min_responses_per_matchup: int,
+        benchmark_id: str,
         id: str,
         openapi_service: OpenAPIService,
     ):
@@ -43,31 +47,26 @@ class RapidataLeaderboard:
         self.__inverse_ranking = inverse_ranking
         self.__response_budget = response_budget
         self.__min_responses_per_matchup = min_responses_per_matchup
+        self.__benchmark_id = benchmark_id
         self.id = id
+        self.__leaderboard_page = f"https://app.{self.__openapi_service.environment}/mri/benchmarks/{self.__benchmark_id}/leaderboard/{self.id}"
 
     @property
-    def level_of_detail(self) -> Literal["low", "medium", "high", "very high"]:
+    def level_of_detail(self) -> LevelOfDetail:
         """
         Returns the level of detail of the leaderboard.
         """
         return DetailMapper.get_level_of_detail(self.__response_budget)
 
     @level_of_detail.setter
-    def level_of_detail(
-        self, level_of_detail: Literal["low", "medium", "high", "very high"]
-    ):
+    def level_of_detail(self, level_of_detail: LevelOfDetail):
         """
         Sets the level of detail of the leaderboard.
         """
-        logger.debug(f"Setting level of detail to {level_of_detail}")
-        self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_response_config_put(
-            leaderboard_id=self.id,
-            update_leaderboard_response_config_model=UpdateLeaderboardResponseConfigModel(
-                responseBudget=DetailMapper.get_budget(level_of_detail),
-                minResponses=self.__min_responses_per_matchup,
-            ),
-        )
-        self.__response_budget = DetailMapper.get_budget(level_of_detail)
+        with tracer.start_as_current_span("RapidataLeaderboard.level_of_detail.setter"):
+            logger.debug(f"Setting level of detail to {level_of_detail}")
+            self.__response_budget = DetailMapper.get_budget(level_of_detail)
+            self._update_config()
 
     @property
     def min_responses_per_matchup(self) -> int:
@@ -81,23 +80,20 @@ class RapidataLeaderboard:
         """
         Sets the minimum number of responses required to be considered for the leaderboard.
         """
-        if not isinstance(min_responses, int):
-            raise ValueError("Min responses per matchup must be an integer")
+        with tracer.start_as_current_span(
+            "RapidataLeaderboard.min_responses_per_matchup.setter"
+        ):
+            if not isinstance(min_responses, int):
+                raise ValueError("Min responses per matchup must be an integer")
 
-        if min_responses < 3:
-            raise ValueError("Min responses per matchup must be at least 3")
+            if min_responses < 3:
+                raise ValueError("Min responses per matchup must be at least 3")
 
-        logger.debug(
-            f"Setting min responses per matchup to {min_responses} for leaderboard {self.name}"
-        )
-        self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_response_config_put(
-            leaderboard_id=self.id,
-            update_leaderboard_response_config_model=UpdateLeaderboardResponseConfigModel(
-                responseBudget=self.__response_budget,
-                minResponses=min_responses,
-            ),
-        )
-        self.__min_responses_per_matchup = min_responses
+            logger.debug(
+                f"Setting min responses per matchup to {min_responses} for leaderboard {self.name}"
+            )
+            self.__min_responses_per_matchup = min_responses
+            self._update_config()
 
     @property
     def show_prompt_asset(self) -> bool:
@@ -134,7 +130,21 @@ class RapidataLeaderboard:
         """
         return self.__name
 
-    def get_standings(self, tags: Optional[list[str]] = None) -> pd.DataFrame:
+    @name.setter
+    def name(self, name: str):
+        """
+        Sets the name of the leaderboard.
+        """
+        with tracer.start_as_current_span("RapidataLeaderboard.name.setter"):
+            if not isinstance(name, str):
+                raise ValueError("Name must be a string")
+            if len(name) < 1:
+                raise ValueError("Name must be at least 1 character long")
+
+            self.__name = name
+            self._update_config()
+
+    def get_standings(self, tags: Optional[list[str]] = None) -> "pd.DataFrame":
         """
         Returns the standings of the leaderboard.
 
@@ -146,27 +156,61 @@ class RapidataLeaderboard:
         Returns:
             A pandas DataFrame containing the standings of the leaderboard.
         """
+        with tracer.start_as_current_span("RapidataLeaderboard.get_standings"):
+            participants = self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_standings_get(
+                leaderboard_id=self.id, tags=tags
+            )
 
-        participants = self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_standings_get(
-            leaderboard_id=self.id, tags=tags
-        )
-
-        standings = []
-        for participant in participants.items:
-            standings.append(
-                {
-                    "name": participant.name,
-                    "wins": participant.wins,
-                    "total_matches": participant.total_matches,
-                    "score": (
-                        round(participant.score, 2)
-                        if participant.score is not None
-                        else None
-                    ),
-                }
+            import pandas as pd
+
+            standings = []
+            for participant in participants.items:
+                standings.append(
+                    {
+                        "name": participant.name,
+                        "wins": participant.wins,
+                        "total_matches": participant.total_matches,
+                        "score": (
+                            round(participant.score, 2)
+                            if participant.score is not None
+                            else None
+                        ),
+                    }
+                )
+
+            return pd.DataFrame(standings)
+
+    def view(self) -> None:
+        """
+        Views the leaderboard.
+        """
+        logger.info("Opening leaderboard page in browser...")
+        could_open_browser = webbrowser.open(self.__leaderboard_page)
+        if not could_open_browser:
+            encoded_url = urllib.parse.quote(
+                self.__leaderboard_page, safe="%/:=&?~#+!$,;'@()*[]"
+            )
+            managed_print(
+                Fore.RED
+                + f"Please open this URL in your browser: '{encoded_url}'"
+                + Fore.RESET
             )
 
-        return pd.DataFrame(standings)
+    def _custom_config(self, response_budget: int, min_responses_per_matchup: int):
+        self.__response_budget = response_budget
+        self.__min_responses_per_matchup = min_responses_per_matchup
+        self._update_config()
+
+    def _update_config(self):
+        with tracer.start_as_current_span("RapidataLeaderboard._update_config"):
+            self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_patch(
+                leaderboard_id=self.id,
+                update_leaderboard_model=UpdateLeaderboardModel(
+                    name=self.__name,
+                    responseBudget=self.__response_budget,
+                    minResponses=self.__min_responses_per_matchup,
+                ),
+            )
 
     def __str__(self) -> str:
         return f"RapidataLeaderboard(name={self.name}, instruction={self.instruction}, show_prompt={self.show_prompt}, leaderboard_id={self.id})"

rapidata/rapidata_client/benchmark/participant/_participant.py

@@ -1,10 +1,19 @@
+from __future__ import annotations
+
 from concurrent.futures import ThreadPoolExecutor, as_completed
+import time
 from tqdm import tqdm
 
-from rapidata.rapidata_client.datapoints.assets import MediaAsset
-from rapidata.rapidata_client.logging import logger
-from rapidata.rapidata_client.logging.output_manager import RapidataOutputManager
-from rapidata.api_client.models.create_sample_model import CreateSampleModel
+from rapidata.rapidata_client.config import logger
+from rapidata.rapidata_client.config.rapidata_config import rapidata_config
+from rapidata.rapidata_client.api.rapidata_api_client import (
+    suppress_rapidata_error_logging,
+)
+
+from opentelemetry import context as otel_context
+from rapidata.rapidata_client.datapoints._asset_uploader import AssetUploader
+
+
 from rapidata.service.openapi_service import OpenAPIService
 
 
@@ -12,91 +21,126 @@ class BenchmarkParticipant:
     def __init__(self, name: str, id: str, openapi_service: OpenAPIService):
         self.name = name
         self.id = id
-        self.__openapi_service = openapi_service
+        self._openapi_service = openapi_service
+        self._asset_uploader = AssetUploader(openapi_service)
 
     def _process_single_sample_upload(
         self,
-        asset: MediaAsset,
+        asset: str,
         identifier: str,
-    ) -> tuple[MediaAsset | None, MediaAsset | None]:
+    ) -> tuple[str | None, str | None]:
         """
         Process single sample upload with retry logic and error tracking.
-        
+
         Args:
             asset: MediaAsset to upload
             identifier: Identifier for the sample
-            
+
         Returns:
             tuple[MediaAsset | None, MediaAsset | None]: (successful_asset, failed_asset)
         """
-        if asset.is_local():
-            files = [asset.to_file()]
-            urls = []
-        else:
-            files = []
-            urls = [asset.path]
+        from rapidata.api_client.models.create_sample_model import CreateSampleModel
+        from rapidata.api_client.models.i_asset_input_existing_asset_input import (
+            IAssetInputExistingAssetInput,
+        )
+
+        from rapidata.api_client.models.i_asset_input import IAssetInput
 
         last_exception = None
-        try:
-            self.__openapi_service.participant_api.participant_participant_id_sample_post(
-                participant_id=self.id,
-                model=CreateSampleModel(
-                    identifier=identifier
-                ),
-                files=files,
-                urls=urls
-            )
-            
-            return asset, None
-            
-        except Exception as e:
-            last_exception = e
+        for attempt in range(rapidata_config.upload.maxRetries):
+            try:
+                with suppress_rapidata_error_logging():
+                    self._openapi_service.participant_api.participant_participant_id_sample_post(
+                        participant_id=self.id,
+                        create_sample_model=CreateSampleModel(
+                            identifier=identifier,
+                            asset=IAssetInput(
+                                actual_instance=IAssetInputExistingAssetInput(
+                                    _t="ExistingAssetInput",
+                                    name=self._asset_uploader.upload_asset(asset),
+                                ),
+                            ),
+                        ),
+                    )
+
+                return asset, None
+
+            except Exception as e:
+                last_exception = e
+                if attempt < rapidata_config.upload.maxRetries - 1:
+                    # Exponential backoff: wait 1s, then 2s, then 4s
+                    retry_delay = 2**attempt
+                    time.sleep(retry_delay)
+                    logger.debug("Error: %s", str(last_exception))
+                    logger.debug(
+                        "Retrying %s of %s...",
+                        attempt + 1,
+                        rapidata_config.upload.maxRetries,
+                    )
 
         logger.error(f"Upload failed for {identifier}. Error: {str(last_exception)}")
         return None, asset
 
     def upload_media(
         self,
-        assets: list[MediaAsset],
+        assets: list[str],
         identifiers: list[str],
-        max_workers: int = 10,
-    ) -> tuple[list[MediaAsset], list[MediaAsset]]:
+    ) -> tuple[list[str], list[str]]:
         """
         Upload samples concurrently with proper error handling and progress tracking.
-        
+
         Args:
-            assets: List of MediaAsset objects to upload
+            assets: List of strings to upload
             identifiers: List of identifiers matching the assets
-            max_workers: Maximum number of concurrent upload workers
-            
+
         Returns:
             tuple[list[str], list[str]]: Lists of successful and failed identifiers
         """
-        successful_uploads: list[MediaAsset] = []
-        failed_uploads: list[MediaAsset] = []
+
+        def upload_with_context(
+            context: otel_context.Context, asset: str, identifier: str
+        ) -> tuple[str | None, str | None]:
+            """Wrapper function that runs _process_single_sample_upload with the provided context."""
+            token = otel_context.attach(context)
+            try:
+                return self._process_single_sample_upload(asset, identifier)
+            finally:
+                otel_context.detach(token)
+
+        successful_uploads: list[str] = []
+        failed_uploads: list[str] = []
         total_uploads = len(assets)
-        
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+
+        # Capture the current OpenTelemetry context before creating threads
+        current_context = otel_context.get_current()
+
+        with ThreadPoolExecutor(
+            max_workers=rapidata_config.upload.maxWorkers
+        ) as executor:
             futures = [
                 executor.submit(
-                    self._process_single_sample_upload,
+                    upload_with_context,
+                    current_context,
                     asset,
                     identifier,
                 )
                 for asset, identifier in zip(assets, identifiers)
             ]
-            
-            with tqdm(total=total_uploads, desc="Uploading media", disable=RapidataOutputManager.silent_mode) as pbar:
+
+            with tqdm(
+                total=total_uploads,
+                desc="Uploading media",
+                disable=rapidata_config.logging.silent_mode,
+            ) as pbar:
                 for future in as_completed(futures):
                     try:
                         successful_id, failed_id = future.result()
                         if successful_id:
                             successful_uploads.append(successful_id)
+                            pbar.update(1)
                         if failed_id:
                             failed_uploads.append(failed_id)
                     except Exception as e:
                         logger.error(f"Future execution failed: {str(e)}")
-                        
-                    pbar.update(1)
-        
+
         return successful_uploads, failed_uploads