rapidata 2.23.2__py3-none-any.whl → 3.0.1__py3-none-any.whl

rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py

@@ -0,0 +1,257 @@
+from __future__ import annotations
+import urllib.parse
+import webbrowser
+from colorama import Fore
+from typing import Optional, TYPE_CHECKING
+
+from rapidata.rapidata_client.config import logger, managed_print, tracer
+from rapidata.rapidata_client.benchmark._detail_mapper import (
+    DetailMapper,
+    LevelOfDetail,
+)
+from rapidata.service.openapi_service import OpenAPIService
+from rapidata.api_client.models.update_leaderboard_model import UpdateLeaderboardModel
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+
+class RapidataLeaderboard:
+    """
+    An instance of a Rapidata leaderboard.
+
+    Used to interact with a specific leaderboard in the Rapidata system, such as retrieving prompts and evaluating models.
+
+    Args:
+        name: The name that will be used to identify the leaderboard on the overview.
+        instruction: The instruction that will determine what how the models will be evaluated.
+        show_prompt: Whether to show the prompt to the users.
+        id: The ID of the leaderboard.
+        openapi_service: The OpenAPIService instance for API interaction.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        instruction: str,
+        show_prompt: bool,
+        show_prompt_asset: bool,
+        inverse_ranking: bool,
+        response_budget: int,
+        min_responses_per_matchup: int,
+        benchmark_id: str,
+        id: str,
+        openapi_service: OpenAPIService,
+    ):
+        self.__openapi_service = openapi_service
+        self.__name = name
+        self.__instruction = instruction
+        self.__show_prompt = show_prompt
+        self.__show_prompt_asset = show_prompt_asset
+        self.__inverse_ranking = inverse_ranking
+        self.__response_budget = response_budget
+        self.__min_responses_per_matchup = min_responses_per_matchup
+        self.__benchmark_id = benchmark_id
+        self.id = id
+        self.__leaderboard_page = f"https://app.{self.__openapi_service.environment}/mri/benchmarks/{self.__benchmark_id}/leaderboard/{self.id}"
+
+    @property
+    def level_of_detail(self) -> LevelOfDetail:
+        """
+        Returns the level of detail of the leaderboard.
+        """
+        return DetailMapper.get_level_of_detail(self.__response_budget)
+
+    @level_of_detail.setter
+    def level_of_detail(self, level_of_detail: LevelOfDetail):
+        """
+        Sets the level of detail of the leaderboard.
+        """
+        with tracer.start_as_current_span("RapidataLeaderboard.level_of_detail.setter"):
+            logger.debug(f"Setting level of detail to {level_of_detail}")
+            self.__response_budget = DetailMapper.get_budget(level_of_detail)
+            self._update_config()
+
+    @property
+    def min_responses_per_matchup(self) -> int:
+        """
+        Returns the minimum number of responses required to be considered for the leaderboard.
+        """
+        return self.__min_responses_per_matchup
+
+    @min_responses_per_matchup.setter
+    def min_responses_per_matchup(self, min_responses: int):
+        """
+        Sets the minimum number of responses required to be considered for the leaderboard.
+        """
+        with tracer.start_as_current_span(
+            "RapidataLeaderboard.min_responses_per_matchup.setter"
+        ):
+            if not isinstance(min_responses, int):
+                raise ValueError("Min responses per matchup must be an integer")
+
+            if min_responses < 3:
+                raise ValueError("Min responses per matchup must be at least 3")
+
+            logger.debug(
+                f"Setting min responses per matchup to {min_responses} for leaderboard {self.name}"
+            )
+            self.__min_responses_per_matchup = min_responses
+            self._update_config()
+
+    @property
+    def show_prompt_asset(self) -> bool:
+        """
+        Returns whether the prompt asset is shown to the users.
+        """
+        return self.__show_prompt_asset
+
+    @property
+    def inverse_ranking(self) -> bool:
+        """
+        Returns whether the ranking is inverse.
+        """
+        return self.__inverse_ranking
+
+    @property
+    def show_prompt(self) -> bool:
+        """
+        Returns whether the prompt is shown to the users.
+        """
+        return self.__show_prompt
+
+    @property
+    def instruction(self) -> str:
+        """
+        Returns the instruction of the leaderboard.
+        """
+        return self.__instruction
+
+    @property
+    def name(self) -> str:
+        """
+        Returns the name of the leaderboard.
+        """
+        return self.__name
+
+    @name.setter
+    def name(self, name: str):
+        """
+        Sets the name of the leaderboard.
+        """
+        with tracer.start_as_current_span("RapidataLeaderboard.name.setter"):
+            if not isinstance(name, str):
+                raise ValueError("Name must be a string")
+            if len(name) < 1:
+                raise ValueError("Name must be at least 1 character long")
+
+            self.__name = name
+            self._update_config()
+
+    def get_standings(self, tags: Optional[list[str]] = None) -> "pd.DataFrame":
+        """
+        Returns the standings of the leaderboard.
+
+        Args:
+            tags: The matchups with these tags should be used to create the standings.
+                If tags are None, all matchups will be considered.
+                If tags are empty, no matchups will be considered.
+
+        Returns:
+            A pandas DataFrame containing the standings of the leaderboard.
+        """
+        with tracer.start_as_current_span("RapidataLeaderboard.get_standings"):
+            participants = self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_standings_get(
+                leaderboard_id=self.id, tags=tags
+            )
+
+            import pandas as pd
+
+            standings = []
+            for participant in participants.items:
+                standings.append(
+                    {
+                        "name": participant.name,
+                        "wins": participant.wins,
+                        "total_matches": participant.total_matches,
+                        "score": (
+                            round(participant.score, 2)
+                            if participant.score is not None
+                            else None
+                        ),
+                    }
+                )
+
+            return pd.DataFrame(standings)
+
+    def get_win_loss_matrix(
+        self,
+        tags: Optional[list[str]] = None,
+        use_weighted_scoring: Optional[bool] = None,
+    ) -> pd.DataFrame:
+        """
+        Returns the win/loss matrix for all participants in this leaderboard.
+
+        The matrix shows pairwise comparison results where each cell [i, j] represents
+        the number of wins participant i has against participant j.
+
+        Args:
+            tags: Filter matchups by these tags. If None, all matchups are considered.
+            use_weighted_scoring: Whether to use weighted scoring for the matrix calculation.
+
+        Returns:
+            A pandas DataFrame with participants as both index and columns,
+            containing the pairwise win counts.
+        """
+        with tracer.start_as_current_span("RapidataLeaderboard.get_win_loss_matrix"):
+            result = self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_matrix_get(
+                leaderboard_id=self.id,
+                tags=tags,
+                use_weighted_scoring=use_weighted_scoring,
+            )
+
+            import pandas as pd
+
+            return pd.DataFrame(
+                data=result.data,
+                index=pd.Index(result.index),
+                columns=pd.Index(result.columns),
+            )
+
+    def view(self) -> None:
+        """
+        Views the leaderboard.
+        """
+        logger.info("Opening leaderboard page in browser...")
+        could_open_browser = webbrowser.open(self.__leaderboard_page)
+        if not could_open_browser:
+            encoded_url = urllib.parse.quote(
+                self.__leaderboard_page, safe="%/:=&?~#+!$,;'@()*[]"
+            )
+            managed_print(
+                Fore.RED
+                + f"Please open this URL in your browser: '{encoded_url}'"
+                + Fore.RESET
+            )
+
+    def _custom_config(self, response_budget: int, min_responses_per_matchup: int):
+        self.__response_budget = response_budget
+        self.__min_responses_per_matchup = min_responses_per_matchup
+        self._update_config()
+
+    def _update_config(self):
+        with tracer.start_as_current_span("RapidataLeaderboard._update_config"):
+            self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_patch(
+                leaderboard_id=self.id,
+                update_leaderboard_model=UpdateLeaderboardModel(
+                    name=self.__name,
+                    responseBudget=self.__response_budget,
+                    minResponses=self.__min_responses_per_matchup,
+                ),
+            )
+
+    def __str__(self) -> str:
+        return f"RapidataLeaderboard(name={self.name}, instruction={self.instruction}, show_prompt={self.show_prompt}, leaderboard_id={self.id})"
+
+    def __repr__(self) -> str:
+        return self.__str__()

rapidata/rapidata_client/benchmark/participant/_participant.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import time
+from tqdm import tqdm
+
+from rapidata.rapidata_client.config import logger
+from rapidata.rapidata_client.config.rapidata_config import rapidata_config
+from rapidata.rapidata_client.api.rapidata_api_client import (
+    suppress_rapidata_error_logging,
+)
+
+from opentelemetry import context as otel_context
+from rapidata.rapidata_client.datapoints._asset_uploader import AssetUploader
+
+
+from rapidata.service.openapi_service import OpenAPIService
+
+
+class BenchmarkParticipant:
+    def __init__(self, name: str, id: str, openapi_service: OpenAPIService):
+        self.name = name
+        self.id = id
+        self._openapi_service = openapi_service
+        self._asset_uploader = AssetUploader(openapi_service)
+
+    def _process_single_sample_upload(
+        self,
+        asset: str,
+        identifier: str,
+    ) -> tuple[str | None, str | None]:
+        """
+        Process single sample upload with retry logic and error tracking.
+
+        Args:
+            asset: MediaAsset to upload
+            identifier: Identifier for the sample
+
+        Returns:
+            tuple[MediaAsset | None, MediaAsset | None]: (successful_asset, failed_asset)
+        """
+        from rapidata.api_client.models.create_sample_model import CreateSampleModel
+        from rapidata.api_client.models.i_asset_input_existing_asset_input import (
+            IAssetInputExistingAssetInput,
+        )
+
+        from rapidata.api_client.models.i_asset_input import IAssetInput
+
+        last_exception = None
+        for attempt in range(rapidata_config.upload.maxRetries):
+            try:
+                with suppress_rapidata_error_logging():
+                    self._openapi_service.participant_api.participant_participant_id_sample_post(
+                        participant_id=self.id,
+                        create_sample_model=CreateSampleModel(
+                            identifier=identifier,
+                            asset=IAssetInput(
+                                actual_instance=IAssetInputExistingAssetInput(
+                                    _t="ExistingAssetInput",
+                                    name=self._asset_uploader.upload_asset(asset),
+                                ),
+                            ),
+                        ),
+                    )
+
+                return asset, None
+
+            except Exception as e:
+                last_exception = e
+                if attempt < rapidata_config.upload.maxRetries - 1:
+                    # Exponential backoff: wait 1s, then 2s, then 4s
+                    retry_delay = 2**attempt
+                    time.sleep(retry_delay)
+                    logger.debug("Error: %s", str(last_exception))
+                    logger.debug(
+                        "Retrying %s of %s...",
+                        attempt + 1,
+                        rapidata_config.upload.maxRetries,
+                    )
+
+        logger.error(f"Upload failed for {identifier}. Error: {str(last_exception)}")
+        return None, asset
+
+    def upload_media(
+        self,
+        assets: list[str],
+        identifiers: list[str],
+    ) -> tuple[list[str], list[str]]:
+        """
+        Upload samples concurrently with proper error handling and progress tracking.
+
+        Args:
+            assets: List of strings to upload
+            identifiers: List of identifiers matching the assets
+
+        Returns:
+            tuple[list[str], list[str]]: Lists of successful and failed identifiers
+        """
+
+        def upload_with_context(
+            context: otel_context.Context, asset: str, identifier: str
+        ) -> tuple[str | None, str | None]:
+            """Wrapper function that runs _process_single_sample_upload with the provided context."""
+            token = otel_context.attach(context)
+            try:
+                return self._process_single_sample_upload(asset, identifier)
+            finally:
+                otel_context.detach(token)
+
+        successful_uploads: list[str] = []
+        failed_uploads: list[str] = []
+        total_uploads = len(assets)
+
+        # Capture the current OpenTelemetry context before creating threads
+        current_context = otel_context.get_current()
+
+        with ThreadPoolExecutor(
+            max_workers=rapidata_config.upload.maxWorkers
+        ) as executor:
+            futures = [
+                executor.submit(
+                    upload_with_context,
+                    current_context,
+                    asset,
+                    identifier,
+                )
+                for asset, identifier in zip(assets, identifiers)
+            ]
+
+            with tqdm(
+                total=total_uploads,
+                desc="Uploading media",
+                disable=rapidata_config.logging.silent_mode,
+            ) as pbar:
+                for future in as_completed(futures):
+                    try:
+                        successful_id, failed_id = future.result()
+                        if successful_id:
+                            successful_uploads.append(successful_id)
+                            pbar.update(1)
+                        if failed_id:
+                            failed_uploads.append(failed_id)
+                    except Exception as e:
+                        logger.error(f"Future execution failed: {str(e)}")
+
+        return successful_uploads, failed_uploads