rapidata 2.35.1__py3-none-any.whl → 3.0.0a8__py3-none-any.whl

rapidata/rapidata_client/benchmark/rapidata_benchmark.py

@@ -1,33 +1,19 @@
-import re
-from typing import Literal, Optional
-from rapidata.api_client.models.root_filter import RootFilter
-from rapidata.api_client.models.filter import Filter
-from rapidata.api_client.models.query_model import QueryModel
-from rapidata.api_client.models.page_info import PageInfo
-from rapidata.api_client.models.create_leaderboard_model import CreateLeaderboardModel
-from rapidata.api_client.models.create_benchmark_participant_model import (
-    CreateBenchmarkParticipantModel,
-)
-from rapidata.api_client.models.submit_prompt_model import SubmitPromptModel
-from rapidata.api_client.models.submit_prompt_model_prompt_asset import (
-    SubmitPromptModelPromptAsset,
-)
-from rapidata.api_client.models.url_asset_input import UrlAssetInput
-from rapidata.api_client.models.file_asset_model import FileAssetModel
-from rapidata.api_client.models.source_url_metadata_model import SourceUrlMetadataModel
-
-
-from rapidata.rapidata_client.benchmark.participant._participant import (
-    BenchmarkParticipant,
-)
-from rapidata.rapidata_client.logging import logger
-from rapidata.service.openapi_service import OpenAPIService
-
-from rapidata.rapidata_client.benchmark.leaderboard.rapidata_leaderboard import (
-    RapidataLeaderboard,
-)
-from rapidata.rapidata_client.datapoints.assets import MediaAsset
-from rapidata.rapidata_client.benchmark._detail_mapper import DetailMapper
+from __future__ import annotations
+import urllib.parse
+import webbrowser
+from colorama import Fore
+from typing import Optional, Sequence, TYPE_CHECKING
+from rapidata.rapidata_client.config import logger, managed_print, tracer
+from rapidata.rapidata_client.benchmark._detail_mapper import LevelOfDetail
+
+if TYPE_CHECKING:
+    import pandas as pd
+    from rapidata.rapidata_client.benchmark.leaderboard.rapidata_leaderboard import (
+        RapidataLeaderboard,
+    )
+    from rapidata.rapidata_client.filter import RapidataFilter
+    from rapidata.rapidata_client.settings import RapidataSetting
+    from rapidata.service.openapi_service import OpenAPIService
 
 
 class RapidataBenchmark:
@@ -43,54 +29,67 @@ class RapidataBenchmark:
     """
 
     def __init__(self, name: str, id: str, openapi_service: OpenAPIService):
+        from rapidata.rapidata_client.datapoints._asset_uploader import AssetUploader
+
         self.name = name
         self.id = id
-        self.__openapi_service = openapi_service
+        self._openapi_service = openapi_service
         self.__prompts: list[str | None] = []
         self.__prompt_assets: list[str | None] = []
-        self.__leaderboards: list[RapidataLeaderboard] = []
+        self.__leaderboards: list["RapidataLeaderboard"] = []
         self.__identifiers: list[str] = []
         self.__tags: list[list[str]] = []
+        self.__benchmark_page: str = (
+            f"https://app.{self._openapi_service.environment}/mri/benchmarks/{self.id}"
+        )
+        self._asset_uploader = AssetUploader(openapi_service)
 
     def __instantiate_prompts(self) -> None:
-        current_page = 1
-        total_pages = None
+        from rapidata.rapidata_client.config import tracer
+        from rapidata.api_client.models.query_model import QueryModel
+        from rapidata.api_client.models.page_info import PageInfo
+        from rapidata.api_client.models.file_asset_model import FileAssetModel
+        from rapidata.api_client.models.source_url_metadata_model import (
+            SourceUrlMetadataModel,
+        )
 
-        while True:
-            prompts_result = (
-                self.__openapi_service.benchmark_api.benchmark_benchmark_id_prompts_get(
+        with tracer.start_as_current_span("RapidataBenchmark.__instantiate_prompts"):
+            current_page = 1
+            total_pages = None
+
+            while True:
+                prompts_result = self._openapi_service.benchmark_api.benchmark_benchmark_id_prompts_get(
                     benchmark_id=self.id,
                     request=QueryModel(page=PageInfo(index=current_page, size=100)),
                 )
-            )
 
-            if prompts_result.total_pages is None:
-                raise ValueError(
-                    "An error occurred while fetching prompts: total_pages is None"
-                )
+                if prompts_result.total_pages is None:
+                    raise ValueError(
+                        "An error occurred while fetching prompts: total_pages is None"
+                    )
 
-            total_pages = prompts_result.total_pages
+                total_pages = prompts_result.total_pages
 
-            for prompt in prompts_result.items:
-                self.__prompts.append(prompt.prompt)
-                self.__identifiers.append(prompt.identifier)
-                if prompt.prompt_asset is None:
-                    self.__prompt_assets.append(None)
-                else:
-                    assert isinstance(
-                        prompt.prompt_asset.actual_instance, FileAssetModel
-                    )
-                    source_url = prompt.prompt_asset.actual_instance.metadata[
-                        "sourceUrl"
-                    ].actual_instance
-                    assert isinstance(source_url, SourceUrlMetadataModel)
-                    self.__prompt_assets.append(source_url.url)
+                for prompt in prompts_result.items:
+                    self.__prompts.append(prompt.prompt)
+                    self.__identifiers.append(prompt.identifier)
+                    if prompt.prompt_asset is None:
+                        self.__prompt_assets.append(None)
+                    else:
+                        assert isinstance(
+                            prompt.prompt_asset.actual_instance, FileAssetModel
+                        )
+                        source_url = prompt.prompt_asset.actual_instance.metadata[
+                            "sourceUrl"
+                        ].actual_instance
+                        assert isinstance(source_url, SourceUrlMetadataModel)
+                        self.__prompt_assets.append(source_url.url)
 
-                self.__tags.append(prompt.tags)
-            if current_page >= total_pages:
-                break
+                    self.__tags.append(prompt.tags)
+                if current_page >= total_pages:
+                    break
 
-            current_page += 1
+                current_page += 1
 
     @property
     def identifiers(self) -> list[str]:
@@ -134,122 +133,157 @@ class RapidataBenchmark:
         """
         Returns the leaderboards that are registered for the benchmark.
         """
-        if not self.__leaderboards:
-            current_page = 1
-            total_pages = None
+        from rapidata.api_client.models.query_model import QueryModel
+        from rapidata.api_client.models.page_info import PageInfo
+        from rapidata.rapidata_client.benchmark.leaderboard.rapidata_leaderboard import (
+            RapidataLeaderboard,
+        )
 
-            while True:
-                leaderboards_result = (
-                    self.__openapi_service.leaderboard_api.leaderboards_get(
+        with tracer.start_as_current_span("RapidataBenchmark.leaderboards"):
+            if not self.__leaderboards:
+                current_page = 1
+                total_pages = None
+
+                while True:
+                    leaderboards_result = self._openapi_service.benchmark_api.benchmark_benchmark_id_leaderboards_get(
+                        benchmark_id=self.id,
                         request=QueryModel(
-                            filter=RootFilter(
-                                filters=[
-                                    Filter(
-                                        field="BenchmarkId",
-                                        operator="Eq",
-                                        value=self.id,
-                                    )
-                                ]
-                            ),
                             page=PageInfo(index=current_page, size=100),
-                        )
+                        ),
                     )
-                )
 
-                if leaderboards_result.total_pages is None:
-                    raise ValueError(
-                        "An error occurred while fetching leaderboards: total_pages is None"
-                    )
-
-                total_pages = leaderboards_result.total_pages
-
-                self.__leaderboards.extend(
-                    [
-                        RapidataLeaderboard(
-                            leaderboard.name,
-                            leaderboard.instruction,
-                            leaderboard.show_prompt,
-                            leaderboard.show_prompt_asset,
-                            leaderboard.is_inversed,
-                            leaderboard.response_budget,
-                            leaderboard.min_responses,
-                            leaderboard.id,
-                            self.__openapi_service,
+                    if leaderboards_result.total_pages is None:
+                        raise ValueError(
+                            "An error occurred while fetching leaderboards: total_pages is None"
                         )
-                        for leaderboard in leaderboards_result.items
-                    ]
-                )
 
-                if current_page >= total_pages:
-                    break
+                    total_pages = leaderboards_result.total_pages
+
+                    self.__leaderboards.extend(
+                        [
+                            RapidataLeaderboard(
+                                leaderboard.name,
+                                leaderboard.instruction,
+                                leaderboard.show_prompt,
+                                leaderboard.show_prompt_asset,
+                                leaderboard.is_inversed,
+                                leaderboard.response_budget,
+                                leaderboard.min_responses,
+                                self.id,
+                                leaderboard.id,
+                                self._openapi_service,
+                            )
+                            for leaderboard in leaderboards_result.items
+                        ]
+                    )
 
-                current_page += 1
+                    if current_page >= total_pages:
+                        break
+
+                    current_page += 1
 
-        return self.__leaderboards
+            return self.__leaderboards
 
     def add_prompt(
         self,
-        identifier: str,
+        identifier: str | None = None,
         prompt: str | None = None,
-        asset: str | None = None,
+        prompt_asset: str | None = None,
         tags: Optional[list[str]] = None,
     ):
         """
         Adds a prompt to the benchmark.
 
         Args:
-            identifier: The identifier of the prompt/asset/tags that will be used to match up the media.
+            identifier: The identifier of the prompt/asset/tags that will be used to match up the media. If not provided, it will use the prompt, asset or prompt + asset as the identifier.
             prompt: The prompt that will be used to evaluate the model.
-            asset: The asset that will be used to evaluate the model. Provided as a link to the asset.
+            prompt_asset: The prompt asset that will be used to evaluate the model. Provided as a link to the asset.
             tags: The tags can be used to filter the leaderboard results. They will NOT be shown to the users.
         """
-        if tags is None:
-            tags = []
-
-        if not isinstance(identifier, str):
-            raise ValueError("Identifier must be a string.")
-
-        if prompt is None and asset is None:
-            raise ValueError("Prompt or asset must be provided.")
+        from rapidata.api_client.models.submit_prompt_model import SubmitPromptModel
+        from rapidata.api_client.models.i_asset_input_existing_asset_input import (
+            IAssetInputExistingAssetInput,
+        )
 
-        if prompt is not None and not isinstance(prompt, str):
-            raise ValueError("Prompt must be a string.")
+        from rapidata.api_client.models.i_asset_input import IAssetInput
 
-        if asset is not None and not isinstance(asset, str):
-            raise ValueError("Asset must be a string. That is the link to the asset.")
+        with tracer.start_as_current_span("RapidataBenchmark.add_prompt"):
+            if tags is None:
+                tags = []
 
-        if identifier in self.identifiers:
-            raise ValueError("Identifier already exists in the benchmark.")
+            if prompt is None and prompt_asset is None:
+                raise ValueError("Prompt or prompt asset must be provided.")
 
-        if asset is not None and not re.match(r"^https?://", asset):
-            raise ValueError("Asset must be a link to the asset.")
+            if identifier is None and prompt is None:
+                raise ValueError("Identifier or prompt must be provided.")
 
-        if tags is not None and (
-            not isinstance(tags, list) or not all(isinstance(tag, str) for tag in tags)
-        ):
-            raise ValueError("Tags must be a list of strings.")
+            if identifier and not isinstance(identifier, str):
+                raise ValueError("Identifier must be a string.")
 
-        self.__identifiers.append(identifier)
+            if prompt and not isinstance(prompt, str):
+                raise ValueError("Prompt must be a string.")
 
-        self.__tags.append(tags)
-        self.__prompts.append(prompt)
-        self.__prompt_assets.append(asset)
+            if prompt_asset and not isinstance(prompt_asset, str):
+                raise ValueError(
+                    "Asset must be a string. That is the link to the asset."
+                )
 
-        self.__openapi_service.benchmark_api.benchmark_benchmark_id_prompt_post(
-            benchmark_id=self.id,
-            submit_prompt_model=SubmitPromptModel(
-                identifier=identifier,
-                prompt=prompt,
-                promptAsset=(
-                    SubmitPromptModelPromptAsset(
-                        UrlAssetInput(_t="UrlAssetInput", url=asset)
+            if identifier is None:
+                assert prompt is not None
+                if prompt in self.prompts:
+                    raise ValueError(
+                        "Prompts must be unique. Otherwise use identifiers."
                     )
-                    if asset is not None
-                    else None
+                identifier = prompt
+
+            if identifier in self.identifiers:
+                raise ValueError("Identifier already exists in the benchmark.")
+
+            if tags is not None and (
+                not isinstance(tags, list)
+                or not all(isinstance(tag, str) for tag in tags)
+            ):
+                raise ValueError("Tags must be a list of strings.")
+
+            logger.info(
+                "Adding identifier %s with prompt %s, prompt asset %s and tags %s to benchmark %s",
+                identifier,
+                prompt,
+                prompt_asset,
+                tags,
+                self.id,
+            )
+
+            self.__identifiers.append(identifier)
+
+            self.__tags.append(tags)
+            self.__prompts.append(prompt)
+            self.__prompt_assets.append(prompt_asset)
+
+            self._openapi_service.benchmark_api.benchmark_benchmark_id_prompt_post(
+                benchmark_id=self.id,
+                submit_prompt_model=SubmitPromptModel(
+                    identifier=identifier,
+                    prompt=prompt,
+                    promptAsset=(
+                        IAssetInput(
+                            actual_instance=(
+                                IAssetInputExistingAssetInput(
+                                    _t="ExistingAssetInput",
+                                    name=self._asset_uploader.upload_asset(
+                                        prompt_asset
+                                    ),
+                                )
+                                if prompt_asset is not None
+                                else None
+                            )
+                        )
+                        if prompt_asset is not None
+                        else None
+                    ),
+                    tags=tags,
                 ),
-                tags=tags,
-            ),
-        )
+            )
 
     def create_leaderboard(
         self,
@@ -258,8 +292,11 @@ class RapidataBenchmark:
         show_prompt: bool = False,
         show_prompt_asset: bool = False,
         inverse_ranking: bool = False,
-        level_of_detail: Literal["low", "medium", "high", "very high"] = "low",
-        min_responses_per_matchup: int = 3,
+        level_of_detail: LevelOfDetail | None = None,
+        min_responses_per_matchup: int | None = None,
+        validation_set_id: str | None = None,
+        filters: Sequence["RapidataFilter"] | None = None,
+        settings: Sequence["RapidataSetting"] | None = None,
     ) -> RapidataLeaderboard:
         """
         Creates a new leaderboard for the benchmark.
@@ -272,113 +309,239 @@ class RapidataBenchmark:
             inverse_ranking: Whether to inverse the ranking of the leaderboard. (if the question is inversed, e.g. "Which video is worse?")
             level_of_detail: The level of detail of the leaderboard. This will effect how many comparisons are done per model evaluation. (default: "low")
             min_responses_per_matchup: The minimum number of responses required to be considered for the leaderboard. (default: 3)
+            validation_set_id: The id of the validation set that should be attached to the leaderboard. (default: None)
+            filters: The filters that should be applied to the leaderboard. Will determine who can solve answer in the leaderboard. (default: [])
+            settings: The settings that should be applied to the leaderboard. Will determine the behavior of the tasks on the leaderboard. (default: [])
         """
-        if not isinstance(min_responses_per_matchup, int):
-            raise ValueError("Min responses per matchup must be an integer")
-
-        if min_responses_per_matchup < 3:
-            raise ValueError("Min responses per matchup must be at least 3")
-
-        leaderboard_result = self.__openapi_service.leaderboard_api.leaderboard_post(
-            create_leaderboard_model=CreateLeaderboardModel(
-                benchmarkId=self.id,
-                name=name,
-                instruction=instruction,
-                showPrompt=show_prompt,
-                showPromptAsset=show_prompt_asset,
-                isInversed=inverse_ranking,
-                minResponses=min_responses_per_matchup,
-                responseBudget=DetailMapper.get_budget(level_of_detail),
-            )
+        from rapidata.api_client.models.create_leaderboard_model import (
+            CreateLeaderboardModel,
         )
-
-        assert (
-            leaderboard_result.benchmark_id == self.id
-        ), "The leaderboard was not created for the correct benchmark."
-
-        return RapidataLeaderboard(
-            name,
-            instruction,
-            show_prompt,
-            show_prompt_asset,
-            inverse_ranking,
-            leaderboard_result.response_budget,
-            min_responses_per_matchup,
-            leaderboard_result.id,
-            self.__openapi_service,
+        from rapidata.rapidata_client.benchmark._detail_mapper import DetailMapper
+        from rapidata.rapidata_client.benchmark.leaderboard.rapidata_leaderboard import (
+            RapidataLeaderboard,
         )
 
+        with tracer.start_as_current_span("create_leaderboard"):
+            if level_of_detail is not None and (
+                not isinstance(level_of_detail, str)
+                or level_of_detail not in ["low", "medium", "high", "very high"]
+            ):
+                raise ValueError(
+                    "Level of detail must be a string and one of: 'low', 'medium', 'high', 'very high'"
+                )
+
+            if min_responses_per_matchup is not None and (
+                not isinstance(min_responses_per_matchup, int)
+                or min_responses_per_matchup < 3
+            ):
+                raise ValueError(
+                    "Min responses per matchup must be an integer and at least 3"
+                )
+
+            logger.info(
+                "Creating leaderboard %s with instruction %s, show_prompt %s, show_prompt_asset %s, inverse_ranking %s, level_of_detail %s, min_responses_per_matchup %s, validation_set_id %s, filters %s, settings %s",
+                name,
+                instruction,
+                show_prompt,
+                show_prompt_asset,
+                inverse_ranking,
+                level_of_detail,
+                min_responses_per_matchup,
+                validation_set_id,
+                filters,
+                settings,
+            )
+
+            leaderboard_result = self._openapi_service.leaderboard_api.leaderboard_post(
+                create_leaderboard_model=CreateLeaderboardModel(
+                    benchmarkId=self.id,
+                    name=name,
+                    instruction=instruction,
+                    showPrompt=show_prompt,
+                    showPromptAsset=show_prompt_asset,
+                    isInversed=inverse_ranking,
+                    minResponses=min_responses_per_matchup,
+                    responseBudget=(
+                        DetailMapper.get_budget(level_of_detail)
+                        if level_of_detail is not None
+                        else None
+                    ),
+                    validationSetId=validation_set_id,
+                    filters=(
+                        [filter._to_model() for filter in filters] if filters else None
+                    ),
+                    featureFlags=(
+                        [setting._to_feature_flag() for setting in settings]  # type: ignore - until backend fixes generation
+                        if settings
+                        else None
+                    ),
+                )
+            )
+
+            assert (
+                leaderboard_result.benchmark_id == self.id
+            ), "The leaderboard was not created for the correct benchmark."
+
+            logger.info("Leaderboard created with id %s", leaderboard_result.id)
+
+            return RapidataLeaderboard(
+                name,
+                instruction,
+                show_prompt,
+                show_prompt_asset,
+                inverse_ranking,
+                leaderboard_result.response_budget,
+                leaderboard_result.min_responses,
+                self.id,
+                leaderboard_result.id,
+                self._openapi_service,
+            )
+
     def evaluate_model(
-        self, name: str, media: list[str], identifiers: list[str]
+        self,
+        name: str,
+        media: list[str],
+        identifiers: list[str] | None = None,
+        prompts: list[str] | None = None,
     ) -> None:
         """
         Evaluates a model on the benchmark across all leaderboards.
 
+        prompts or identifiers must be provided to match the media.
+
         Args:
             name: The name of the model.
             media: The generated images/videos that will be used to evaluate the model.
-            identifiers: The identifiers that correspond to the media. The order of the identifiers must match the order of the media.
+            identifiers: The identifiers that correspond to the media. The order of the identifiers must match the order of the media.\n
                 The identifiers that are used must be registered for the benchmark. To see the registered identifiers, use the identifiers property.
+            prompts: The prompts that correspond to the media. The order of the prompts must match the order of the media.
         """
-        if not media:
-            raise ValueError("Media must be a non-empty list of strings")
+        from rapidata.api_client.models.create_benchmark_participant_model import (
+            CreateBenchmarkParticipantModel,
+        )
+        from rapidata.rapidata_client.benchmark.participant._participant import (
+            BenchmarkParticipant,
+        )
+
+        with tracer.start_as_current_span("evaluate_model"):
+            if not media:
+                raise ValueError("Media must be a non-empty list of strings")
 
-        if len(media) != len(identifiers):
-            raise ValueError("Media and identifiers must have the same length")
+            if not identifiers and not prompts:
+                raise ValueError("Identifiers or prompts must be provided.")
 
-        if not all(identifier in self.identifiers for identifier in identifiers):
-            raise ValueError(
-                "All identifiers must be in the registered identifiers list. To see the registered identifiers, use the identifiers property.\
-\nTo see the prompts that are associated with the identifiers, use the prompts property."
+            if identifiers and prompts:
+                raise ValueError(
+                    "Identifiers and prompts cannot be provided at the same time. Use one or the other."
+                )
+
+            if not identifiers:
+                assert prompts is not None
+                identifiers = prompts
+
+            if len(media) != len(identifiers):
+                raise ValueError(
+                    "Media and identifiers/prompts must have the same length"
+                )
+
+            if not all(identifier in self.identifiers for identifier in identifiers):
+                raise ValueError(
+                    "All identifiers/prompts must be in the registered identifiers/prompts list. To see the registered identifiers/prompts, use the identifiers/prompts property."
+                )
+
+            participant_result = self._openapi_service.benchmark_api.benchmark_benchmark_id_participants_post(
+                benchmark_id=self.id,
+                create_benchmark_participant_model=CreateBenchmarkParticipantModel(
+                    name=name,
+                ),
             )
 
-        # happens before the creation of the participant to ensure all media paths are valid
-        assets: list[MediaAsset] = []
-        for media_path in media:
-            assets.append(MediaAsset(media_path))
+            logger.info(f"Participant created: {participant_result.participant_id}")
 
-        participant_result = self.__openapi_service.benchmark_api.benchmark_benchmark_id_participants_post(
-            benchmark_id=self.id,
-            create_benchmark_participant_model=CreateBenchmarkParticipantModel(
-                name=name,
-            ),
-        )
+            participant = BenchmarkParticipant(
+                name, participant_result.participant_id, self._openapi_service
+            )
 
-        logger.info(f"Participant created: {participant_result.participant_id}")
+            with tracer.start_as_current_span("upload_media_for_participant"):
+                logger.info(
+                    f"Uploading {len(media)} media assets to participant {participant.id}"
+                )
 
-        participant = BenchmarkParticipant(
-            name, participant_result.participant_id, self.__openapi_service
-        )
+                successful_uploads, failed_uploads = participant.upload_media(
+                    media,
+                    identifiers,
+                )
 
-        successful_uploads, failed_uploads = participant.upload_media(
-            assets,
-            identifiers,
-        )
+                total_uploads = len(media)
+                success_rate = (
+                    (len(successful_uploads) / total_uploads * 100)
+                    if total_uploads > 0
+                    else 0
+                )
+                logger.info(
+                    f"Upload complete: {len(successful_uploads)} successful, {len(failed_uploads)} failed ({success_rate:.1f}% success rate)"
+                )
 
-        total_uploads = len(assets)
-        success_rate = (
-            (len(successful_uploads) / total_uploads * 100) if total_uploads > 0 else 0
-        )
-        logger.info(
-            f"Upload complete: {len(successful_uploads)} successful, {len(failed_uploads)} failed ({success_rate:.1f}% success rate)"
-        )
+                if failed_uploads:
+                    logger.error(f"Failed uploads for media: {failed_uploads}")
+                    logger.warning(
+                        "Some uploads failed. The model evaluation may be incomplete."
+                    )
+
+                if len(successful_uploads) == 0:
+                    raise RuntimeError(
+                        "No uploads were successful. The model evaluation will not be completed."
+                    )
 
-        if failed_uploads:
-            logger.error(
-                f"Failed uploads for media: {[asset.path for asset in failed_uploads]}"
+                self._openapi_service.participant_api.participants_participant_id_submit_post(
+                    participant_id=participant_result.participant_id
+                )
+
+    def view(self) -> None:
+        """
+        Views the benchmark.
+        """
+
+        logger.info("Opening benchmark page in browser...")
+        could_open_browser = webbrowser.open(self.__benchmark_page)
+        if not could_open_browser:
+            encoded_url = urllib.parse.quote(
+                self.__benchmark_page, safe="%/:=&?~#+!$,;'@()*[]"
             )
-            logger.warning(
-                "Some uploads failed. The model evaluation may be incomplete."
+            managed_print(
+                Fore.RED
+                + f"Please open this URL in your browser: '{encoded_url}'"
+                + Fore.RESET
             )
 
-        if len(successful_uploads) == 0:
-            raise RuntimeError(
-                "No uploads were successful. The model evaluation will not be completed."
+    def get_overall_standings(self, tags: Optional[list[str]] = None) -> pd.DataFrame:
+        """
+        Returns an aggregated elo table of all leaderboards in the benchmark.
+        """
+        import pandas as pd
+
+        with tracer.start_as_current_span("get_overall_standings"):
+            participants = self._openapi_service.benchmark_api.benchmark_benchmark_id_standings_get(
+                benchmark_id=self.id,
+                tags=tags,
             )
 
-        self.__openapi_service.participant_api.participants_participant_id_submit_post(
-            participant_id=participant_result.participant_id
-        )
+            standings = []
+            for participant in participants.items:
+                standings.append(
+                    {
+                        "name": participant.name,
+                        "wins": participant.wins,
+                        "total_matches": participant.total_matches,
+                        "score": (
+                            round(participant.score, 2)
+                            if participant.score is not None
+                            else None
+                        ),
+                    }
+                )
+
+            return pd.DataFrame(standings)
 
     def __str__(self) -> str:
         return f"RapidataBenchmark(name={self.name}, id={self.id})"