rapidata 2.36.2__py3-none-any.whl → 2.38.0__py3-none-any.whl

rapidata/rapidata_client/benchmark/rapidata_benchmark.py

@@ -1,38 +1,41 @@
 import re
+import urllib.parse
+import webbrowser
+from colorama import Fore
 from typing import Literal, Optional, Sequence
-from rapidata.api_client.models.root_filter import RootFilter
-from rapidata.api_client.models.filter import Filter
-from rapidata.api_client.models.query_model import QueryModel
-from rapidata.api_client.models.page_info import PageInfo
-from rapidata.api_client.models.create_leaderboard_model import CreateLeaderboardModel
+
+from rapidata.api_client.models.and_user_filter_model_filters_inner import (
+    AndUserFilterModelFiltersInner,
+)
 from rapidata.api_client.models.create_benchmark_participant_model import (
     CreateBenchmarkParticipantModel,
 )
+from rapidata.api_client.models.create_leaderboard_model import CreateLeaderboardModel
+from rapidata.api_client.models.filter import Filter
+from rapidata.api_client.models.filter_operator import FilterOperator
+from rapidata.api_client.models.file_asset_model import FileAssetModel
+from rapidata.api_client.models.query_model import QueryModel
+from rapidata.api_client.models.page_info import PageInfo
+from rapidata.api_client.models.root_filter import RootFilter
+from rapidata.api_client.models.source_url_metadata_model import SourceUrlMetadataModel
 from rapidata.api_client.models.submit_prompt_model import SubmitPromptModel
 from rapidata.api_client.models.submit_prompt_model_prompt_asset import (
     SubmitPromptModelPromptAsset,
 )
 from rapidata.api_client.models.url_asset_input import UrlAssetInput
-from rapidata.api_client.models.file_asset_model import FileAssetModel
-from rapidata.api_client.models.source_url_metadata_model import SourceUrlMetadataModel
-from rapidata.api_client.models.and_user_filter_model_filters_inner import (
-    AndUserFilterModelFiltersInner,
-)
-from rapidata.api_client.models.filter_operator import FilterOperator
-
-from rapidata.rapidata_client.benchmark.participant._participant import (
-    BenchmarkParticipant,
-)
-from rapidata.rapidata_client.logging import logger
-from rapidata.service.openapi_service import OpenAPIService
 
+from rapidata.rapidata_client.benchmark._detail_mapper import DetailMapper
 from rapidata.rapidata_client.benchmark.leaderboard.rapidata_leaderboard import (
     RapidataLeaderboard,
 )
+from rapidata.rapidata_client.benchmark.participant._participant import (
+    BenchmarkParticipant,
+)
 from rapidata.rapidata_client.datapoints.assets import MediaAsset
-from rapidata.rapidata_client.benchmark._detail_mapper import DetailMapper
 from rapidata.rapidata_client.filter import RapidataFilter
+from rapidata.rapidata_client.config import logger, managed_print, tracer
 from rapidata.rapidata_client.settings import RapidataSetting
+from rapidata.service.openapi_service import OpenAPIService
 
 
 class RapidataBenchmark:
@@ -56,6 +59,9 @@ class RapidataBenchmark:
         self.__leaderboards: list[RapidataLeaderboard] = []
         self.__identifiers: list[str] = []
         self.__tags: list[list[str]] = []
+        self.__benchmark_page: str = (
+            f"https://app.{self.__openapi_service.environment}/mri/benchmarks/{self.id}"
+        )
 
     def __instantiate_prompts(self) -> None:
         current_page = 1
@@ -99,98 +105,104 @@ class RapidataBenchmark:
 
     @property
     def identifiers(self) -> list[str]:
-        if not self.__identifiers:
-            self.__instantiate_prompts()
+        with tracer.start_as_current_span("RapidataBenchmark.identifiers"):
+            if not self.__identifiers:
+                self.__instantiate_prompts()
 
-        return self.__identifiers
+            return self.__identifiers
 
     @property
     def prompts(self) -> list[str | None]:
         """
         Returns the prompts that are registered for the leaderboard.
         """
-        if not self.__prompts:
-            self.__instantiate_prompts()
+        with tracer.start_as_current_span("RapidataBenchmark.prompts"):
+            if not self.__prompts:
+                self.__instantiate_prompts()
 
-        return self.__prompts
+            return self.__prompts
 
     @property
     def prompt_assets(self) -> list[str | None]:
         """
         Returns the prompt assets that are registered for the benchmark.
         """
-        if not self.__prompt_assets:
-            self.__instantiate_prompts()
+        with tracer.start_as_current_span("RapidataBenchmark.prompt_assets"):
+            if not self.__prompt_assets:
+                self.__instantiate_prompts()
 
-        return self.__prompt_assets
+            return self.__prompt_assets
 
     @property
     def tags(self) -> list[list[str]]:
         """
         Returns the tags that are registered for the benchmark.
         """
-        if not self.__tags:
-            self.__instantiate_prompts()
+        with tracer.start_as_current_span("RapidataBenchmark.tags"):
+            if not self.__tags:
+                self.__instantiate_prompts()
 
-        return self.__tags
+            return self.__tags
 
     @property
     def leaderboards(self) -> list[RapidataLeaderboard]:
         """
         Returns the leaderboards that are registered for the benchmark.
         """
-        if not self.__leaderboards:
-            current_page = 1
-            total_pages = None
-
-            while True:
-                leaderboards_result = (
-                    self.__openapi_service.leaderboard_api.leaderboards_get(
-                        request=QueryModel(
-                            filter=RootFilter(
-                                filters=[
-                                    Filter(
-                                        field="BenchmarkId",
-                                        operator=FilterOperator.EQ,
-                                        value=self.id,
-                                    )
-                                ]
-                            ),
-                            page=PageInfo(index=current_page, size=100),
+        with tracer.start_as_current_span("RapidataBenchmark.leaderboards"):
+            if not self.__leaderboards:
+                current_page = 1
+                total_pages = None
+
+                while True:
+                    leaderboards_result = (
+                        self.__openapi_service.leaderboard_api.leaderboards_get(
+                            request=QueryModel(
+                                filter=RootFilter(
+                                    filters=[
+                                        Filter(
+                                            field="BenchmarkId",
+                                            operator=FilterOperator.EQ,
+                                            value=self.id,
+                                        )
+                                    ]
+                                ),
+                                page=PageInfo(index=current_page, size=100),
+                            )
                         )
                     )
-                )
-
-                if leaderboards_result.total_pages is None:
-                    raise ValueError(
-                        "An error occurred while fetching leaderboards: total_pages is None"
-                    )
 
-                total_pages = leaderboards_result.total_pages
-
-                self.__leaderboards.extend(
-                    [
-                        RapidataLeaderboard(
-                            leaderboard.name,
-                            leaderboard.instruction,
-                            leaderboard.show_prompt,
-                            leaderboard.show_prompt_asset,
-                            leaderboard.is_inversed,
-                            leaderboard.response_budget,
-                            leaderboard.min_responses,
-                            leaderboard.id,
-                            self.__openapi_service,
+                    if leaderboards_result.total_pages is None:
+                        raise ValueError(
+                            "An error occurred while fetching leaderboards: total_pages is None"
                         )
-                        for leaderboard in leaderboards_result.items
-                    ]
-                )
 
-                if current_page >= total_pages:
-                    break
+                    total_pages = leaderboards_result.total_pages
+
+                    self.__leaderboards.extend(
+                        [
+                            RapidataLeaderboard(
+                                leaderboard.name,
+                                leaderboard.instruction,
+                                leaderboard.show_prompt,
+                                leaderboard.show_prompt_asset,
+                                leaderboard.is_inversed,
+                                leaderboard.response_budget,
+                                leaderboard.min_responses,
+                                self.id,
+                                leaderboard.id,
+                                self.__openapi_service,
+                            )
+                            for leaderboard in leaderboards_result.items
+                        ]
+                    )
 
-                current_page += 1
+                    if current_page >= total_pages:
+                        break
 
-        return self.__leaderboards
+                    current_page += 1
+
+            return self.__leaderboards
 
     def add_prompt(
         self,
@@ -208,53 +220,66 @@ class RapidataBenchmark:
             asset: The asset that will be used to evaluate the model. Provided as a link to the asset.
             tags: The tags can be used to filter the leaderboard results. They will NOT be shown to the users.
         """
-        if tags is None:
-            tags = []
-
-        if not isinstance(identifier, str):
-            raise ValueError("Identifier must be a string.")
+        with tracer.start_as_current_span("RapidataBenchmark.add_prompt"):
+            if tags is None:
+                tags = []
 
-        if prompt is None and asset is None:
-            raise ValueError("Prompt or asset must be provided.")
+            if not isinstance(identifier, str):
+                raise ValueError("Identifier must be a string.")
 
-        if prompt is not None and not isinstance(prompt, str):
-            raise ValueError("Prompt must be a string.")
+            if prompt is None and asset is None:
+                raise ValueError("Prompt or asset must be provided.")
 
-        if asset is not None and not isinstance(asset, str):
-            raise ValueError("Asset must be a string. That is the link to the asset.")
+            if prompt is not None and not isinstance(prompt, str):
+                raise ValueError("Prompt must be a string.")
 
-        if identifier in self.identifiers:
-            raise ValueError("Identifier already exists in the benchmark.")
-
-        if asset is not None and not re.match(r"^https?://", asset):
-            raise ValueError("Asset must be a link to the asset.")
+            if asset is not None and not isinstance(asset, str):
+                raise ValueError(
+                    "Asset must be a string. That is the link to the asset."
+                )
 
-        if tags is not None and (
-            not isinstance(tags, list) or not all(isinstance(tag, str) for tag in tags)
-        ):
-            raise ValueError("Tags must be a list of strings.")
+            if identifier in self.identifiers:
+                raise ValueError("Identifier already exists in the benchmark.")
+
+            if asset is not None and not re.match(r"^https?://", asset):
+                raise ValueError("Asset must be a link to the asset.")
+
+            if tags is not None and (
+                not isinstance(tags, list)
+                or not all(isinstance(tag, str) for tag in tags)
+            ):
+                raise ValueError("Tags must be a list of strings.")
+
+            logger.info(
+                "Adding identifier %s with prompt %s, asset %s and tags %s to benchmark %s",
+                identifier,
+                prompt,
+                asset,
+                tags,
+                self.id,
+            )
 
-        self.__identifiers.append(identifier)
+            self.__identifiers.append(identifier)
 
-        self.__tags.append(tags)
-        self.__prompts.append(prompt)
-        self.__prompt_assets.append(asset)
+            self.__tags.append(tags)
+            self.__prompts.append(prompt)
+            self.__prompt_assets.append(asset)
 
-        self.__openapi_service.benchmark_api.benchmark_benchmark_id_prompt_post(
-            benchmark_id=self.id,
-            submit_prompt_model=SubmitPromptModel(
-                identifier=identifier,
-                prompt=prompt,
-                promptAsset=(
-                    SubmitPromptModelPromptAsset(
-                        UrlAssetInput(_t="UrlAssetInput", url=asset)
-                    )
-                    if asset is not None
-                    else None
+            self.__openapi_service.benchmark_api.benchmark_benchmark_id_prompt_post(
+                benchmark_id=self.id,
+                submit_prompt_model=SubmitPromptModel(
+                    identifier=identifier,
+                    prompt=prompt,
+                    promptAsset=(
+                        SubmitPromptModelPromptAsset(
+                            UrlAssetInput(_t="UrlAssetInput", url=asset)
+                        )
+                        if asset is not None
+                        else None
+                    ),
+                    tags=tags,
                 ),
-                tags=tags,
-            ),
-        )
+            )
 
     def create_leaderboard(
         self,
@@ -284,54 +309,74 @@ class RapidataBenchmark:
             filters: The filters that should be applied to the leaderboard. Will determine who can solve answer in the leaderboard. (default: [])
             settings: The settings that should be applied to the leaderboard. Will determine the behavior of the tasks on the leaderboard. (default: [])
         """
-        if not isinstance(min_responses_per_matchup, int):
-            raise ValueError("Min responses per matchup must be an integer")
-
-        if min_responses_per_matchup < 3:
-            raise ValueError("Min responses per matchup must be at least 3")
-
-        leaderboard_result = self.__openapi_service.leaderboard_api.leaderboard_post(
-            create_leaderboard_model=CreateLeaderboardModel(
-                benchmarkId=self.id,
-                name=name,
-                instruction=instruction,
-                showPrompt=show_prompt,
-                showPromptAsset=show_prompt_asset,
-                isInversed=inverse_ranking,
-                minResponses=min_responses_per_matchup,
-                responseBudget=DetailMapper.get_budget(level_of_detail),
-                validationSetId=validation_set_id,
-                filters=(
-                    [
-                        AndUserFilterModelFiltersInner(filter._to_model())
-                        for filter in filters
-                    ]
-                    if filters
-                    else None
-                ),
-                featureFlags=(
-                    [setting._to_feature_flag() for setting in settings]
-                    if settings
-                    else None
-                ),
+        with tracer.start_as_current_span("create_leaderboard"):
+            if not isinstance(min_responses_per_matchup, int):
+                raise ValueError("Min responses per matchup must be an integer")
+
+            if min_responses_per_matchup < 3:
+                raise ValueError("Min responses per matchup must be at least 3")
+
+            logger.info(
+                "Creating leaderboard %s with instruction %s, show_prompt %s, show_prompt_asset %s, inverse_ranking %s, level_of_detail %s, min_responses_per_matchup %s, validation_set_id %s, filters %s, settings %s",
+                name,
+                instruction,
+                show_prompt,
+                show_prompt_asset,
+                inverse_ranking,
+                level_of_detail,
+                min_responses_per_matchup,
+                validation_set_id,
+                filters,
+                settings,
             )
-        )
 
-        assert (
-            leaderboard_result.benchmark_id == self.id
-        ), "The leaderboard was not created for the correct benchmark."
-
-        return RapidataLeaderboard(
-            name,
-            instruction,
-            show_prompt,
-            show_prompt_asset,
-            inverse_ranking,
-            leaderboard_result.response_budget,
-            min_responses_per_matchup,
-            leaderboard_result.id,
-            self.__openapi_service,
-        )
+            leaderboard_result = (
+                self.__openapi_service.leaderboard_api.leaderboard_post(
+                    create_leaderboard_model=CreateLeaderboardModel(
+                        benchmarkId=self.id,
+                        name=name,
+                        instruction=instruction,
+                        showPrompt=show_prompt,
+                        showPromptAsset=show_prompt_asset,
+                        isInversed=inverse_ranking,
+                        minResponses=min_responses_per_matchup,
+                        responseBudget=DetailMapper.get_budget(level_of_detail),
+                        validationSetId=validation_set_id,
+                        filters=(
+                            [
+                                AndUserFilterModelFiltersInner(filter._to_model())
+                                for filter in filters
+                            ]
+                            if filters
+                            else None
+                        ),
+                        featureFlags=(
+                            [setting._to_feature_flag() for setting in settings]
+                            if settings
+                            else None
+                        ),
+                    )
+                )
+            )
+
+            assert (
+                leaderboard_result.benchmark_id == self.id
+            ), "The leaderboard was not created for the correct benchmark."
+
+            logger.info("Leaderboard created with id %s", leaderboard_result.id)
+
+            return RapidataLeaderboard(
+                name,
+                instruction,
+                show_prompt,
+                show_prompt_asset,
+                inverse_ranking,
+                leaderboard_result.response_budget,
+                min_responses_per_matchup,
+                self.id,
+                leaderboard_result.id,
+                self.__openapi_service,
+            )
 
     def evaluate_model(
         self, name: str, media: list[str], identifiers: list[str]
@@ -345,65 +390,89 @@ class RapidataBenchmark:
             identifiers: The identifiers that correspond to the media. The order of the identifiers must match the order of the media.
                 The identifiers that are used must be registered for the benchmark. To see the registered identifiers, use the identifiers property.
         """
-        if not media:
-            raise ValueError("Media must be a non-empty list of strings")
+        with tracer.start_as_current_span("evaluate_model"):
+            if not media:
+                raise ValueError("Media must be a non-empty list of strings")
+
+            if len(media) != len(identifiers):
+                raise ValueError("Media and identifiers must have the same length")
+
+            if not all(identifier in self.identifiers for identifier in identifiers):
+                raise ValueError(
+                    "All identifiers must be in the registered identifiers list. To see the registered identifiers, use the identifiers property.\
+            \nTo see the prompts that are associated with the identifiers, use the prompts property."
+                )
 
-        if len(media) != len(identifiers):
-            raise ValueError("Media and identifiers must have the same length")
+            # happens before the creation of the participant to ensure all media paths are valid
+            assets: list[MediaAsset] = []
+            for media_path in media:
+                assets.append(MediaAsset(media_path))
 
-        if not all(identifier in self.identifiers for identifier in identifiers):
-            raise ValueError(
-                "All identifiers must be in the registered identifiers list. To see the registered identifiers, use the identifiers property.\
-\nTo see the prompts that are associated with the identifiers, use the prompts property."
+            participant_result = self.__openapi_service.benchmark_api.benchmark_benchmark_id_participants_post(
+                benchmark_id=self.id,
+                create_benchmark_participant_model=CreateBenchmarkParticipantModel(
+                    name=name,
+                ),
             )
 
-        # happens before the creation of the participant to ensure all media paths are valid
-        assets: list[MediaAsset] = []
-        for media_path in media:
-            assets.append(MediaAsset(media_path))
+            logger.info(f"Participant created: {participant_result.participant_id}")
 
-        participant_result = self.__openapi_service.benchmark_api.benchmark_benchmark_id_participants_post(
-            benchmark_id=self.id,
-            create_benchmark_participant_model=CreateBenchmarkParticipantModel(
-                name=name,
-            ),
-        )
+            participant = BenchmarkParticipant(
+                name, participant_result.participant_id, self.__openapi_service
+            )
 
-        logger.info(f"Participant created: {participant_result.participant_id}")
+            with tracer.start_as_current_span("upload_media_for_participant"):
+                logger.info(
+                    f"Uploading {len(assets)} media assets to participant {participant.id}"
+                )
 
-        participant = BenchmarkParticipant(
-            name, participant_result.participant_id, self.__openapi_service
-        )
+                successful_uploads, failed_uploads = participant.upload_media(
+                    assets,
+                    identifiers,
+                )
 
-        successful_uploads, failed_uploads = participant.upload_media(
-            assets,
-            identifiers,
-        )
+                total_uploads = len(assets)
+                success_rate = (
+                    (len(successful_uploads) / total_uploads * 100)
+                    if total_uploads > 0
+                    else 0
+                )
+                logger.info(
+                    f"Upload complete: {len(successful_uploads)} successful, {len(failed_uploads)} failed ({success_rate:.1f}% success rate)"
+                )
 
-        total_uploads = len(assets)
-        success_rate = (
-            (len(successful_uploads) / total_uploads * 100) if total_uploads > 0 else 0
-        )
-        logger.info(
-            f"Upload complete: {len(successful_uploads)} successful, {len(failed_uploads)} failed ({success_rate:.1f}% success rate)"
-        )
+                if failed_uploads:
+                    logger.error(
+                        f"Failed uploads for media: {[asset.path for asset in failed_uploads]}"
+                    )
+                    logger.warning(
+                        "Some uploads failed. The model evaluation may be incomplete."
+                    )
 
-        if failed_uploads:
-            logger.error(
-                f"Failed uploads for media: {[asset.path for asset in failed_uploads]}"
-            )
-            logger.warning(
-                "Some uploads failed. The model evaluation may be incomplete."
-            )
+                if len(successful_uploads) == 0:
+                    raise RuntimeError(
+                        "No uploads were successful. The model evaluation will not be completed."
+                    )
 
-        if len(successful_uploads) == 0:
-            raise RuntimeError(
-                "No uploads were successful. The model evaluation will not be completed."
+            self.__openapi_service.participant_api.participants_participant_id_submit_post(
+                participant_id=participant_result.participant_id
             )
 
-        self.__openapi_service.participant_api.participants_participant_id_submit_post(
-            participant_id=participant_result.participant_id
-        )
+    def view(self) -> None:
+        """
+        Views the benchmark.
+        """
+        logger.info("Opening benchmark page in browser...")
+        could_open_browser = webbrowser.open(self.__benchmark_page)
+        if not could_open_browser:
+            encoded_url = urllib.parse.quote(
+                self.__benchmark_page, safe="%/:=&?~#+!$,;'@()*[]"
+            )
+            managed_print(
+                Fore.RED
+                + f"Please open this URL in your browser: '{encoded_url}'"
+                + Fore.RESET
+            )
 
     def __str__(self) -> str:
         return f"RapidataBenchmark(name={self.name}, id={self.id})"