rapidata 2.37.0__py3-none-any.whl → 2.39.0__py3-none-any.whl

rapidata/rapidata_client/benchmark/rapidata_benchmark.py

@@ -1,38 +1,41 @@
 import re
+import urllib.parse
+import webbrowser
+from colorama import Fore
 from typing import Literal, Optional, Sequence
-from rapidata.api_client.models.root_filter import RootFilter
-from rapidata.api_client.models.filter import Filter
-from rapidata.api_client.models.query_model import QueryModel
-from rapidata.api_client.models.page_info import PageInfo
-from rapidata.api_client.models.create_leaderboard_model import CreateLeaderboardModel
+
+from rapidata.api_client.models.and_user_filter_model_filters_inner import (
+    AndUserFilterModelFiltersInner,
+)
 from rapidata.api_client.models.create_benchmark_participant_model import (
     CreateBenchmarkParticipantModel,
 )
+from rapidata.api_client.models.create_leaderboard_model import CreateLeaderboardModel
+from rapidata.api_client.models.filter import Filter
+from rapidata.api_client.models.filter_operator import FilterOperator
+from rapidata.api_client.models.file_asset_model import FileAssetModel
+from rapidata.api_client.models.query_model import QueryModel
+from rapidata.api_client.models.page_info import PageInfo
+from rapidata.api_client.models.root_filter import RootFilter
+from rapidata.api_client.models.source_url_metadata_model import SourceUrlMetadataModel
 from rapidata.api_client.models.submit_prompt_model import SubmitPromptModel
 from rapidata.api_client.models.submit_prompt_model_prompt_asset import (
     SubmitPromptModelPromptAsset,
 )
 from rapidata.api_client.models.url_asset_input import UrlAssetInput
-from rapidata.api_client.models.file_asset_model import FileAssetModel
-from rapidata.api_client.models.source_url_metadata_model import SourceUrlMetadataModel
-from rapidata.api_client.models.and_user_filter_model_filters_inner import (
-    AndUserFilterModelFiltersInner,
-)
-from rapidata.api_client.models.filter_operator import FilterOperator
-
-from rapidata.rapidata_client.benchmark.participant._participant import (
-    BenchmarkParticipant,
-)
-from rapidata.rapidata_client.logging import logger
-from rapidata.service.openapi_service import OpenAPIService
 
+from rapidata.rapidata_client.benchmark._detail_mapper import DetailMapper
 from rapidata.rapidata_client.benchmark.leaderboard.rapidata_leaderboard import (
     RapidataLeaderboard,
 )
+from rapidata.rapidata_client.benchmark.participant._participant import (
+    BenchmarkParticipant,
+)
 from rapidata.rapidata_client.datapoints.assets import MediaAsset
-from rapidata.rapidata_client.benchmark._detail_mapper import DetailMapper
 from rapidata.rapidata_client.filter import RapidataFilter
+from rapidata.rapidata_client.config import logger, managed_print, tracer
 from rapidata.rapidata_client.settings import RapidataSetting
+from rapidata.service.openapi_service import OpenAPIService
 
 
 class RapidataBenchmark:
@@ -56,6 +59,9 @@ class RapidataBenchmark:
         self.__leaderboards: list[RapidataLeaderboard] = []
         self.__identifiers: list[str] = []
         self.__tags: list[list[str]] = []
+        self.__benchmark_page: str = (
+            f"https://app.{self.__openapi_service.environment}/mri/benchmarks/{self.id}"
+        )
 
     def __instantiate_prompts(self) -> None:
         current_page = 1
@@ -99,162 +105,192 @@ class RapidataBenchmark:
 
     @property
     def identifiers(self) -> list[str]:
-        if not self.__identifiers:
-            self.__instantiate_prompts()
+        with tracer.start_as_current_span("RapidataBenchmark.identifiers"):
+            if not self.__identifiers:
+                self.__instantiate_prompts()
 
-        return self.__identifiers
+            return self.__identifiers
 
     @property
     def prompts(self) -> list[str | None]:
         """
         Returns the prompts that are registered for the leaderboard.
         """
-        if not self.__prompts:
-            self.__instantiate_prompts()
+        with tracer.start_as_current_span("RapidataBenchmark.prompts"):
+            if not self.__prompts:
+                self.__instantiate_prompts()
 
-        return self.__prompts
+            return self.__prompts
 
     @property
     def prompt_assets(self) -> list[str | None]:
         """
         Returns the prompt assets that are registered for the benchmark.
         """
-        if not self.__prompt_assets:
-            self.__instantiate_prompts()
+        with tracer.start_as_current_span("RapidataBenchmark.prompt_assets"):
+            if not self.__prompt_assets:
+                self.__instantiate_prompts()
 
-        return self.__prompt_assets
+            return self.__prompt_assets
 
     @property
     def tags(self) -> list[list[str]]:
         """
         Returns the tags that are registered for the benchmark.
         """
-        if not self.__tags:
-            self.__instantiate_prompts()
+        with tracer.start_as_current_span("RapidataBenchmark.tags"):
+            if not self.__tags:
+                self.__instantiate_prompts()
 
-        return self.__tags
+            return self.__tags
 
     @property
     def leaderboards(self) -> list[RapidataLeaderboard]:
         """
         Returns the leaderboards that are registered for the benchmark.
         """
-        if not self.__leaderboards:
-            current_page = 1
-            total_pages = None
-
-            while True:
-                leaderboards_result = (
-                    self.__openapi_service.leaderboard_api.leaderboards_get(
-                        request=QueryModel(
-                            filter=RootFilter(
-                                filters=[
-                                    Filter(
-                                        field="BenchmarkId",
-                                        operator=FilterOperator.EQ,
-                                        value=self.id,
-                                    )
-                                ]
-                            ),
-                            page=PageInfo(index=current_page, size=100),
+        with tracer.start_as_current_span("RapidataBenchmark.leaderboards"):
+            if not self.__leaderboards:
+                current_page = 1
+                total_pages = None
+
+                while True:
+                    leaderboards_result = (
+                        self.__openapi_service.leaderboard_api.leaderboards_get(
+                            request=QueryModel(
+                                filter=RootFilter(
+                                    filters=[
+                                        Filter(
+                                            field="BenchmarkId",
+                                            operator=FilterOperator.EQ,
+                                            value=self.id,
+                                        )
+                                    ]
+                                ),
+                                page=PageInfo(index=current_page, size=100),
+                            )
                         )
                     )
-                )
-
-                if leaderboards_result.total_pages is None:
-                    raise ValueError(
-                        "An error occurred while fetching leaderboards: total_pages is None"
-                    )
 
-                total_pages = leaderboards_result.total_pages
-
-                self.__leaderboards.extend(
-                    [
-                        RapidataLeaderboard(
-                            leaderboard.name,
-                            leaderboard.instruction,
-                            leaderboard.show_prompt,
-                            leaderboard.show_prompt_asset,
-                            leaderboard.is_inversed,
-                            leaderboard.response_budget,
-                            leaderboard.min_responses,
-                            leaderboard.id,
-                            self.__openapi_service,
+                    if leaderboards_result.total_pages is None:
+                        raise ValueError(
+                            "An error occurred while fetching leaderboards: total_pages is None"
                         )
-                        for leaderboard in leaderboards_result.items
-                    ]
-                )
 
-                if current_page >= total_pages:
-                    break
+                    total_pages = leaderboards_result.total_pages
+
+                    self.__leaderboards.extend(
+                        [
+                            RapidataLeaderboard(
+                                leaderboard.name,
+                                leaderboard.instruction,
+                                leaderboard.show_prompt,
+                                leaderboard.show_prompt_asset,
+                                leaderboard.is_inversed,
+                                leaderboard.response_budget,
+                                leaderboard.min_responses,
+                                self.id,
+                                leaderboard.id,
+                                self.__openapi_service,
+                            )
+                            for leaderboard in leaderboards_result.items
+                        ]
+                    )
+
+                    if current_page >= total_pages:
+                        break
 
-                current_page += 1
+                    current_page += 1
 
-        return self.__leaderboards
+            return self.__leaderboards
 
     def add_prompt(
         self,
-        identifier: str,
+        identifier: str | None = None,
         prompt: str | None = None,
-        asset: str | None = None,
+        prompt_asset: str | None = None,
         tags: Optional[list[str]] = None,
     ):
         """
         Adds a prompt to the benchmark.
 
         Args:
-            identifier: The identifier of the prompt/asset/tags that will be used to match up the media.
+            identifier: The identifier of the prompt/asset/tags that will be used to match up the media. If not provided, it will use the prompt, asset or prompt + asset as the identifier.
             prompt: The prompt that will be used to evaluate the model.
-            asset: The asset that will be used to evaluate the model. Provided as a link to the asset.
+            prompt_asset: The prompt asset that will be used to evaluate the model. Provided as a link to the asset.
             tags: The tags can be used to filter the leaderboard results. They will NOT be shown to the users.
         """
-        if tags is None:
-            tags = []
+        with tracer.start_as_current_span("RapidataBenchmark.add_prompt"):
+            if tags is None:
+                tags = []
 
-        if not isinstance(identifier, str):
-            raise ValueError("Identifier must be a string.")
+            if prompt is None and prompt_asset is None:
+                raise ValueError("Prompt or prompt asset must be provided.")
 
-        if prompt is None and asset is None:
-            raise ValueError("Prompt or asset must be provided.")
+            if identifier is None and prompt is None:
+                raise ValueError("Identifier or prompt must be provided.")
 
-        if prompt is not None and not isinstance(prompt, str):
-            raise ValueError("Prompt must be a string.")
+            if identifier and not isinstance(identifier, str):
+                raise ValueError("Identifier must be a string.")
 
-        if asset is not None and not isinstance(asset, str):
-            raise ValueError("Asset must be a string. That is the link to the asset.")
+            if prompt and not isinstance(prompt, str):
+                raise ValueError("Prompt must be a string.")
 
-        if identifier in self.identifiers:
-            raise ValueError("Identifier already exists in the benchmark.")
-
-        if asset is not None and not re.match(r"^https?://", asset):
-            raise ValueError("Asset must be a link to the asset.")
+            if prompt_asset and not isinstance(prompt_asset, str):
+                raise ValueError(
+                    "Asset must be a string. That is the link to the asset."
+                )
 
-        if tags is not None and (
-            not isinstance(tags, list) or not all(isinstance(tag, str) for tag in tags)
-        ):
-            raise ValueError("Tags must be a list of strings.")
+            if identifier is None:
+                assert prompt is not None
+                if prompt in self.prompts:
+                    raise ValueError(
+                        "Prompts must be unique. Otherwise use identifiers."
+                    )
+                identifier = prompt
+
+            if identifier in self.identifiers:
+                raise ValueError("Identifier already exists in the benchmark.")
+
+            if prompt_asset is not None and not re.match(r"^https?://", prompt_asset):
+                raise ValueError("Prompt asset must be a link to the asset.")
+
+            if tags is not None and (
+                not isinstance(tags, list)
+                or not all(isinstance(tag, str) for tag in tags)
+            ):
+                raise ValueError("Tags must be a list of strings.")
+
+            logger.info(
+                "Adding identifier %s with prompt %s, prompt asset %s and tags %s to benchmark %s",
+                identifier,
+                prompt,
+                prompt_asset,
+                tags,
+                self.id,
+            )
 
-        self.__identifiers.append(identifier)
+            self.__identifiers.append(identifier)
 
-        self.__tags.append(tags)
-        self.__prompts.append(prompt)
-        self.__prompt_assets.append(asset)
+            self.__tags.append(tags)
+            self.__prompts.append(prompt)
+            self.__prompt_assets.append(prompt_asset)
 
-        self.__openapi_service.benchmark_api.benchmark_benchmark_id_prompt_post(
-            benchmark_id=self.id,
-            submit_prompt_model=SubmitPromptModel(
-                identifier=identifier,
-                prompt=prompt,
-                promptAsset=(
-                    SubmitPromptModelPromptAsset(
-                        UrlAssetInput(_t="UrlAssetInput", url=asset)
-                    )
-                    if asset is not None
-                    else None
+            self.__openapi_service.benchmark_api.benchmark_benchmark_id_prompt_post(
+                benchmark_id=self.id,
+                submit_prompt_model=SubmitPromptModel(
+                    identifier=identifier,
+                    prompt=prompt,
+                    promptAsset=(
+                        SubmitPromptModelPromptAsset(
+                            UrlAssetInput(_t="UrlAssetInput", url=prompt_asset)
+                        )
+                        if prompt_asset is not None
+                        else None
+                    ),
+                    tags=tags,
                 ),
-                tags=tags,
-            ),
-        )
+            )
 
     def create_leaderboard(
         self,
@@ -284,126 +320,190 @@ class RapidataBenchmark:
             filters: The filters that should be applied to the leaderboard. Will determine who can solve answer in the leaderboard. (default: [])
             settings: The settings that should be applied to the leaderboard. Will determine the behavior of the tasks on the leaderboard. (default: [])
         """
-        if not isinstance(min_responses_per_matchup, int):
-            raise ValueError("Min responses per matchup must be an integer")
-
-        if min_responses_per_matchup < 3:
-            raise ValueError("Min responses per matchup must be at least 3")
-
-        leaderboard_result = self.__openapi_service.leaderboard_api.leaderboard_post(
-            create_leaderboard_model=CreateLeaderboardModel(
-                benchmarkId=self.id,
-                name=name,
-                instruction=instruction,
-                showPrompt=show_prompt,
-                showPromptAsset=show_prompt_asset,
-                isInversed=inverse_ranking,
-                minResponses=min_responses_per_matchup,
-                responseBudget=DetailMapper.get_budget(level_of_detail),
-                validationSetId=validation_set_id,
-                filters=(
-                    [
-                        AndUserFilterModelFiltersInner(filter._to_model())
-                        for filter in filters
-                    ]
-                    if filters
-                    else None
-                ),
-                featureFlags=(
-                    [setting._to_feature_flag() for setting in settings]
-                    if settings
-                    else None
-                ),
+        with tracer.start_as_current_span("create_leaderboard"):
+            if not isinstance(min_responses_per_matchup, int):
+                raise ValueError("Min responses per matchup must be an integer")
+
+            if min_responses_per_matchup < 3:
+                raise ValueError("Min responses per matchup must be at least 3")
+
+            logger.info(
+                "Creating leaderboard %s with instruction %s, show_prompt %s, show_prompt_asset %s, inverse_ranking %s, level_of_detail %s, min_responses_per_matchup %s, validation_set_id %s, filters %s, settings %s",
+                name,
+                instruction,
+                show_prompt,
+                show_prompt_asset,
+                inverse_ranking,
+                level_of_detail,
+                min_responses_per_matchup,
+                validation_set_id,
+                filters,
+                settings,
             )
-        )
 
-        assert (
-            leaderboard_result.benchmark_id == self.id
-        ), "The leaderboard was not created for the correct benchmark."
-
-        return RapidataLeaderboard(
-            name,
-            instruction,
-            show_prompt,
-            show_prompt_asset,
-            inverse_ranking,
-            leaderboard_result.response_budget,
-            min_responses_per_matchup,
-            leaderboard_result.id,
-            self.__openapi_service,
-        )
+            leaderboard_result = (
+                self.__openapi_service.leaderboard_api.leaderboard_post(
+                    create_leaderboard_model=CreateLeaderboardModel(
+                        benchmarkId=self.id,
+                        name=name,
+                        instruction=instruction,
+                        showPrompt=show_prompt,
+                        showPromptAsset=show_prompt_asset,
+                        isInversed=inverse_ranking,
+                        minResponses=min_responses_per_matchup,
+                        responseBudget=DetailMapper.get_budget(level_of_detail),
+                        validationSetId=validation_set_id,
+                        filters=(
+                            [
+                                AndUserFilterModelFiltersInner(filter._to_model())
+                                for filter in filters
+                            ]
+                            if filters
+                            else None
+                        ),
+                        featureFlags=(
+                            [setting._to_feature_flag() for setting in settings]
+                            if settings
+                            else None
+                        ),
+                    )
+                )
+            )
+
+            assert (
+                leaderboard_result.benchmark_id == self.id
+            ), "The leaderboard was not created for the correct benchmark."
+
+            logger.info("Leaderboard created with id %s", leaderboard_result.id)
+
+            return RapidataLeaderboard(
+                name,
+                instruction,
+                show_prompt,
+                show_prompt_asset,
+                inverse_ranking,
+                leaderboard_result.response_budget,
+                min_responses_per_matchup,
+                self.id,
+                leaderboard_result.id,
+                self.__openapi_service,
+            )
 
     def evaluate_model(
-        self, name: str, media: list[str], identifiers: list[str]
+        self,
+        name: str,
+        media: list[str],
+        identifiers: list[str] | None = None,
+        prompts: list[str] | None = None,
     ) -> None:
         """
         Evaluates a model on the benchmark across all leaderboards.
 
+        prompts or identifiers must be provided to match the media.
+
         Args:
             name: The name of the model.
             media: The generated images/videos that will be used to evaluate the model.
-            identifiers: The identifiers that correspond to the media. The order of the identifiers must match the order of the media.
+            identifiers: The identifiers that correspond to the media. The order of the identifiers must match the order of the media.\n
                 The identifiers that are used must be registered for the benchmark. To see the registered identifiers, use the identifiers property.
+            prompts: The prompts that correspond to the media. The order of the prompts must match the order of the media.
         """
-        if not media:
-            raise ValueError("Media must be a non-empty list of strings")
+        with tracer.start_as_current_span("evaluate_model"):
+            if not media:
+                raise ValueError("Media must be a non-empty list of strings")
 
-        if len(media) != len(identifiers):
-            raise ValueError("Media and identifiers must have the same length")
+            if not identifiers and not prompts:
+                raise ValueError("Identifiers or prompts must be provided.")
 
-        if not all(identifier in self.identifiers for identifier in identifiers):
-            raise ValueError(
-                "All identifiers must be in the registered identifiers list. To see the registered identifiers, use the identifiers property.\
-\nTo see the prompts that are associated with the identifiers, use the prompts property."
-            )
+            if identifiers and prompts:
+                raise ValueError(
+                    "Identifiers and prompts cannot be provided at the same time. Use one or the other."
+                )
 
-        # happens before the creation of the participant to ensure all media paths are valid
-        assets: list[MediaAsset] = []
-        for media_path in media:
-            assets.append(MediaAsset(media_path))
+            if not identifiers:
+                assert prompts is not None
+                identifiers = prompts
 
-        participant_result = self.__openapi_service.benchmark_api.benchmark_benchmark_id_participants_post(
-            benchmark_id=self.id,
-            create_benchmark_participant_model=CreateBenchmarkParticipantModel(
-                name=name,
-            ),
-        )
+            if len(media) != len(identifiers):
+                raise ValueError(
+                    "Media and identifiers/prompts must have the same length"
+                )
 
-        logger.info(f"Participant created: {participant_result.participant_id}")
+            if not all(identifier in self.identifiers for identifier in identifiers):
+                raise ValueError(
+                    "All identifiers/prompts must be in the registered identifiers/prompts list. To see the registered identifiers/prompts, use the identifiers/prompts property."
+                )
 
-        participant = BenchmarkParticipant(
-            name, participant_result.participant_id, self.__openapi_service
-        )
+            # happens before the creation of the participant to ensure all media paths are valid
+            assets: list[MediaAsset] = []
+            for media_path in media:
+                assets.append(MediaAsset(media_path))
 
-        successful_uploads, failed_uploads = participant.upload_media(
-            assets,
-            identifiers,
-        )
+            participant_result = self.__openapi_service.benchmark_api.benchmark_benchmark_id_participants_post(
+                benchmark_id=self.id,
+                create_benchmark_participant_model=CreateBenchmarkParticipantModel(
+                    name=name,
+                ),
+            )
 
-        total_uploads = len(assets)
-        success_rate = (
-            (len(successful_uploads) / total_uploads * 100) if total_uploads > 0 else 0
-        )
-        logger.info(
-            f"Upload complete: {len(successful_uploads)} successful, {len(failed_uploads)} failed ({success_rate:.1f}% success rate)"
-        )
+            logger.info(f"Participant created: {participant_result.participant_id}")
 
-        if failed_uploads:
-            logger.error(
-                f"Failed uploads for media: {[asset.path for asset in failed_uploads]}"
-            )
-            logger.warning(
-                "Some uploads failed. The model evaluation may be incomplete."
+            participant = BenchmarkParticipant(
+                name, participant_result.participant_id, self.__openapi_service
             )
 
-        if len(successful_uploads) == 0:
-            raise RuntimeError(
-                "No uploads were successful. The model evaluation will not be completed."
+            with tracer.start_as_current_span("upload_media_for_participant"):
+                logger.info(
+                    f"Uploading {len(assets)} media assets to participant {participant.id}"
+                )
+
+                successful_uploads, failed_uploads = participant.upload_media(
+                    assets,
+                    identifiers,
+                )
+
+                total_uploads = len(assets)
+                success_rate = (
+                    (len(successful_uploads) / total_uploads * 100)
+                    if total_uploads > 0
+                    else 0
+                )
+                logger.info(
+                    f"Upload complete: {len(successful_uploads)} successful, {len(failed_uploads)} failed ({success_rate:.1f}% success rate)"
+                )
+
+                if failed_uploads:
+                    logger.error(
+                        f"Failed uploads for media: {[asset.path for asset in failed_uploads]}"
+                    )
+                    logger.warning(
+                        "Some uploads failed. The model evaluation may be incomplete."
+                    )
+
+                if len(successful_uploads) == 0:
+                    raise RuntimeError(
+                        "No uploads were successful. The model evaluation will not be completed."
+                    )
+
+            self.__openapi_service.participant_api.participants_participant_id_submit_post(
+                participant_id=participant_result.participant_id
             )
 
-        self.__openapi_service.participant_api.participants_participant_id_submit_post(
-            participant_id=participant_result.participant_id
-        )
+    def view(self) -> None:
+        """
+        Views the benchmark.
+        """
+        logger.info("Opening benchmark page in browser...")
+        could_open_browser = webbrowser.open(self.__benchmark_page)
+        if not could_open_browser:
+            encoded_url = urllib.parse.quote(
+                self.__benchmark_page, safe="%/:=&?~#+!$,;'@()*[]"
+            )
+            managed_print(
+                Fore.RED
+                + f"Please open this URL in your browser: '{encoded_url}'"
+                + Fore.RESET
+            )
 
     def __str__(self) -> str:
         return f"RapidataBenchmark(name={self.name}, id={self.id})"