rapidata 2.37.0__py3-none-any.whl → 3.0.0a8__py3-none-any.whl

rapidata/rapidata_client/audience/rapidata_audience_manager.py

@@ -0,0 +1,162 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+from rapidata.rapidata_client.config import tracer
+from rapidata.rapidata_client.config import logger
+
+if TYPE_CHECKING:
+    from rapidata.rapidata_client.audience.rapidata_audience import RapidataAudience
+    from rapidata.service.openapi_service import OpenAPIService
+    from rapidata.rapidata_client.filter import RapidataFilter
+
+
+class RapidataAudienceManager:
+    """Handles everything regarding audiences from creation to retrieval.
+
+    A manager for creating, retrieving, and searching for audiences.
+    Audiences are groups of annotators that can be recruited based on example tasks and assigned jobs.
+    """
+
+    def __init__(self, openapi_service: OpenAPIService):
+        self._openapi_service = openapi_service
+
+    def create_audience(
+        self,
+        name: str,
+        filters: list[RapidataFilter] | None = None,
+    ) -> RapidataAudience:
+        """Create a new audience.
+
+        Creates a new audience with the specified name and optional filters.
+
+        Args:
+            name (str): The name of the audience.
+            filters (list[RapidataFilter], optional): The list of filters to apply to the audience. Defaults to None (no filters).
+
+        Returns:
+            RapidataAudience: The created audience instance.
+        """
+        with tracer.start_as_current_span("RapidataAudienceManager.create_audience"):
+            from rapidata.rapidata_client.audience.rapidata_audience import (
+                RapidataAudience,
+            )
+            from rapidata.api_client.models.create_audience_request import (
+                CreateAudienceRequest,
+            )
+
+            logger.debug(f"Creating audience: {name}")
+            if filters is None:
+                filters = []
+            response = self._openapi_service.audience_api.audience_post(
+                create_audience_request=CreateAudienceRequest(
+                    name=name,
+                    filters=[filter._to_audience_model() for filter in filters],
+                ),
+            )
+            return RapidataAudience(
+                id=response.audience_id,
+                name=name,
+                filters=filters,
+                openapi_service=self._openapi_service,
+            )
+
+    def get_audience_by_id(self, audience_id: str) -> RapidataAudience:
+        """Get an audience by its ID.
+
+        Args:
+            audience_id (str): The unique identifier of the audience.
+
+        Returns:
+            RapidataAudience: The audience instance.
+        """
+        with tracer.start_as_current_span("RapidataAudienceManager.get_audience_by_id"):
+            from rapidata.rapidata_client.filter._backend_filter_mapper import (
+                BackendFilterMapper,
+            )
+            from rapidata.rapidata_client.audience.rapidata_audience import (
+                RapidataAudience,
+            )
+
+            logger.debug(f"Getting audience by id: {audience_id}")
+            response = self._openapi_service.audience_api.audience_audience_id_get(
+                audience_id=audience_id,
+            )
+            return RapidataAudience(
+                id=audience_id,
+                name=response.name,
+                filters=[
+                    BackendFilterMapper.backend_filter_from_rapidata_filter(filter)
+                    for filter in response.filters
+                ],
+                openapi_service=self._openapi_service,
+            )
+
+    def find_audiences(
+        self, name: str = "", amount: int = 10
+    ) -> list[RapidataAudience]:
+        """Find your audiences by name.
+
+        Args:
+            name (str, optional): Filter audiences by name (matching audiences will contain this string). Defaults to "" for any audience.
+            amount (int, optional): The maximum number of audiences to return. Defaults to 10.
+
+        Returns:
+            list[RapidataAudience]: A list of RapidataAudience instances.
+        """
+        with tracer.start_as_current_span("RapidataAudienceManager.find_audiences"):
+            from rapidata.rapidata_client.filter._backend_filter_mapper import (
+                BackendFilterMapper,
+            )
+            from rapidata.api_client.models.page_info import PageInfo
+            from rapidata.api_client.models.query_model import QueryModel
+            from rapidata.api_client.models.root_filter import RootFilter
+            from rapidata.api_client.models.filter import Filter
+            from rapidata.api_client.models.filter_operator import FilterOperator
+            from rapidata.api_client.models.sort_criterion import SortCriterion
+            from rapidata.api_client.models.sort_direction import SortDirection
+            from rapidata.rapidata_client.audience.rapidata_audience import (
+                RapidataAudience,
+            )
+
+            logger.debug(f"Finding audiences: {name}, {amount}")
+            response = self._openapi_service.audience_api.audiences_get(
+                request=QueryModel(
+                    page=PageInfo(index=1, size=amount),
+                    filter=RootFilter(
+                        filters=[
+                            Filter(
+                                field="Name",
+                                operator=FilterOperator.CONTAINS,
+                                value=name,
+                            )
+                        ]
+                    ),
+                    sortCriteria=[
+                        SortCriterion(
+                            direction=SortDirection.DESC, propertyName="CreatedAt"
+                        )
+                    ],
+                )
+            )
+            audiences = []
+            for item in response.items:
+                audiences.append(
+                    RapidataAudience(
+                        id=item.id,
+                        name=item.name,
+                        filters=[
+                            BackendFilterMapper.backend_filter_from_rapidata_filter(
+                                filter
+                            )
+                            for filter in item.filters
+                        ],
+                        openapi_service=self._openapi_service,
+                    )
+                )
+
+            return audiences
+
+    def __str__(self) -> str:
+        return "RapidataAudienceManager"
+
+    def __repr__(self) -> str:
+        return self.__str__()

rapidata/rapidata_client/benchmark/_detail_mapper.py

@@ -1,12 +1,14 @@
 from typing import Literal
 
+LevelOfDetail = Literal["debug", "low", "medium", "high", "very high"]
+
 
 class DetailMapper:
     @staticmethod
-    def get_budget(
-        level_of_detail: Literal["low", "medium", "high", "very high"]
-    ) -> int:
-        if level_of_detail == "low":
+    def get_budget(level_of_detail: LevelOfDetail) -> int:
+        if level_of_detail == "debug":
+            return 20
+        elif level_of_detail == "low":
             return 2_000
         elif level_of_detail == "medium":
             return 4_000
@@ -16,14 +18,14 @@ class DetailMapper:
             return 16_000
         else:
             raise ValueError(
-                "Invalid level of detail. Must be one of: 'low', 'medium', 'high', 'very high'"
+                "Invalid level of detail. Must be one of: 'debug', 'low', 'medium', 'high', 'very high'"
             )
 
     @staticmethod
-    def get_level_of_detail(
-        response_budget: int,
-    ) -> Literal["low", "medium", "high", "very high"]:
-        if response_budget < 4_000:
+    def get_level_of_detail(response_budget: int) -> LevelOfDetail:
+        if response_budget <= 20:
+            return "debug"
+        elif response_budget < 4_000:
             return "low"
         elif response_budget < 8_000:
             return "medium"

rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py

@@ -1,12 +1,15 @@
-import pandas as pd
-from typing import Literal, Optional
+import urllib.parse
+import webbrowser
+from colorama import Fore
+from typing import Literal, Optional, TYPE_CHECKING
 
-from rapidata.rapidata_client.logging import logger
-from rapidata.rapidata_client.benchmark._detail_mapper import DetailMapper
+from rapidata.rapidata_client.config import logger, managed_print, tracer
+from rapidata.rapidata_client.benchmark._detail_mapper import DetailMapper, LevelOfDetail
 from rapidata.service.openapi_service import OpenAPIService
-from rapidata.api_client.models.update_leaderboard_response_config_model import (
-    UpdateLeaderboardResponseConfigModel,
-)
+from rapidata.api_client.models.update_leaderboard_model import UpdateLeaderboardModel
+
+if TYPE_CHECKING:
+    import pandas as pd
 
 
 class RapidataLeaderboard:
@@ -32,6 +35,7 @@ class RapidataLeaderboard:
         inverse_ranking: bool,
         response_budget: int,
         min_responses_per_matchup: int,
+        benchmark_id: str,
         id: str,
         openapi_service: OpenAPIService,
     ):
@@ -43,31 +47,26 @@ class RapidataLeaderboard:
         self.__inverse_ranking = inverse_ranking
         self.__response_budget = response_budget
         self.__min_responses_per_matchup = min_responses_per_matchup
+        self.__benchmark_id = benchmark_id
         self.id = id
+        self.__leaderboard_page = f"https://app.{self.__openapi_service.environment}/mri/benchmarks/{self.__benchmark_id}/leaderboard/{self.id}"
 
     @property
-    def level_of_detail(self) -> Literal["low", "medium", "high", "very high"]:
+    def level_of_detail(self) -> LevelOfDetail:
         """
         Returns the level of detail of the leaderboard.
         """
         return DetailMapper.get_level_of_detail(self.__response_budget)
 
     @level_of_detail.setter
-    def level_of_detail(
-        self, level_of_detail: Literal["low", "medium", "high", "very high"]
-    ):
+    def level_of_detail(self, level_of_detail: LevelOfDetail):
         """
         Sets the level of detail of the leaderboard.
         """
-        logger.debug(f"Setting level of detail to {level_of_detail}")
-        self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_response_config_put(
-            leaderboard_id=self.id,
-            update_leaderboard_response_config_model=UpdateLeaderboardResponseConfigModel(
-                responseBudget=DetailMapper.get_budget(level_of_detail),
-                minResponses=self.__min_responses_per_matchup,
-            ),
-        )
-        self.__response_budget = DetailMapper.get_budget(level_of_detail)
+        with tracer.start_as_current_span("RapidataLeaderboard.level_of_detail.setter"):
+            logger.debug(f"Setting level of detail to {level_of_detail}")
+            self.__response_budget = DetailMapper.get_budget(level_of_detail)
+            self._update_config()
 
     @property
     def min_responses_per_matchup(self) -> int:
@@ -81,23 +80,20 @@ class RapidataLeaderboard:
         """
         Sets the minimum number of responses required to be considered for the leaderboard.
         """
-        if not isinstance(min_responses, int):
-            raise ValueError("Min responses per matchup must be an integer")
+        with tracer.start_as_current_span(
+            "RapidataLeaderboard.min_responses_per_matchup.setter"
+        ):
+            if not isinstance(min_responses, int):
+                raise ValueError("Min responses per matchup must be an integer")
 
-        if min_responses < 3:
-            raise ValueError("Min responses per matchup must be at least 3")
+            if min_responses < 3:
+                raise ValueError("Min responses per matchup must be at least 3")
 
-        logger.debug(
-            f"Setting min responses per matchup to {min_responses} for leaderboard {self.name}"
-        )
-        self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_response_config_put(
-            leaderboard_id=self.id,
-            update_leaderboard_response_config_model=UpdateLeaderboardResponseConfigModel(
-                responseBudget=self.__response_budget,
-                minResponses=min_responses,
-            ),
-        )
-        self.__min_responses_per_matchup = min_responses
+            logger.debug(
+                f"Setting min responses per matchup to {min_responses} for leaderboard {self.name}"
+            )
+            self.__min_responses_per_matchup = min_responses
+            self._update_config()
 
     @property
     def show_prompt_asset(self) -> bool:
@@ -134,7 +130,21 @@ class RapidataLeaderboard:
         """
         return self.__name
 
-    def get_standings(self, tags: Optional[list[str]] = None) -> pd.DataFrame:
+    @name.setter
+    def name(self, name: str):
+        """
+        Sets the name of the leaderboard.
+        """
+        with tracer.start_as_current_span("RapidataLeaderboard.name.setter"):
+            if not isinstance(name, str):
+                raise ValueError("Name must be a string")
+            if len(name) < 1:
+                raise ValueError("Name must be at least 1 character long")
+
+            self.__name = name
+            self._update_config()
+
+    def get_standings(self, tags: Optional[list[str]] = None) -> "pd.DataFrame":
         """
         Returns the standings of the leaderboard.
 
@@ -146,27 +156,61 @@ class RapidataLeaderboard:
         Returns:
             A pandas DataFrame containing the standings of the leaderboard.
         """
+        with tracer.start_as_current_span("RapidataLeaderboard.get_standings"):
+            participants = self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_standings_get(
+                leaderboard_id=self.id, tags=tags
+            )
 
-        participants = self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_standings_get(
-            leaderboard_id=self.id, tags=tags
-        )
-
-        standings = []
-        for participant in participants.items:
-            standings.append(
-                {
-                    "name": participant.name,
-                    "wins": participant.wins,
-                    "total_matches": participant.total_matches,
-                    "score": (
-                        round(participant.score, 2)
-                        if participant.score is not None
-                        else None
-                    ),
-                }
+            import pandas as pd
+
+            standings = []
+            for participant in participants.items:
+                standings.append(
+                    {
+                        "name": participant.name,
+                        "wins": participant.wins,
+                        "total_matches": participant.total_matches,
+                        "score": (
+                            round(participant.score, 2)
+                            if participant.score is not None
+                            else None
+                        ),
+                    }
+                )
+
+            return pd.DataFrame(standings)
+
+    def view(self) -> None:
+        """
+        Views the leaderboard.
+        """
+        logger.info("Opening leaderboard page in browser...")
+        could_open_browser = webbrowser.open(self.__leaderboard_page)
+        if not could_open_browser:
+            encoded_url = urllib.parse.quote(
+                self.__leaderboard_page, safe="%/:=&?~#+!$,;'@()*[]"
+            )
+            managed_print(
+                Fore.RED
+                + f"Please open this URL in your browser: '{encoded_url}'"
+                + Fore.RESET
             )
 
-        return pd.DataFrame(standings)
+    def _custom_config(self, response_budget: int, min_responses_per_matchup: int):
+        self.__response_budget = response_budget
+        self.__min_responses_per_matchup = min_responses_per_matchup
+        self._update_config()
+
+    def _update_config(self):
+        with tracer.start_as_current_span("RapidataLeaderboard._update_config"):
+            self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_patch(
+                leaderboard_id=self.id,
+                update_leaderboard_model=UpdateLeaderboardModel(
+                    name=self.__name,
+                    responseBudget=self.__response_budget,
+                    minResponses=self.__min_responses_per_matchup,
+                ),
+            )
 
     def __str__(self) -> str:
         return f"RapidataLeaderboard(name={self.name}, instruction={self.instruction}, show_prompt={self.show_prompt}, leaderboard_id={self.id})"

rapidata/rapidata_client/benchmark/participant/_participant.py

@@ -1,29 +1,34 @@
+from __future__ import annotations
+
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import time
 from tqdm import tqdm
 
-from rapidata.rapidata_client.datapoints.assets import MediaAsset
-from rapidata.rapidata_client.logging import logger
-from rapidata.rapidata_client.logging.output_manager import RapidataOutputManager
-from rapidata.api_client.models.create_sample_model import CreateSampleModel
-from rapidata.service.openapi_service import OpenAPIService
+from rapidata.rapidata_client.config import logger
 from rapidata.rapidata_client.config.rapidata_config import rapidata_config
-from rapidata.rapidata_client.api.rapidata_exception import (
+from rapidata.rapidata_client.api.rapidata_api_client import (
     suppress_rapidata_error_logging,
 )
 
+from opentelemetry import context as otel_context
+from rapidata.rapidata_client.datapoints._asset_uploader import AssetUploader
+
+
+from rapidata.service.openapi_service import OpenAPIService
+
 
 class BenchmarkParticipant:
     def __init__(self, name: str, id: str, openapi_service: OpenAPIService):
         self.name = name
         self.id = id
-        self.__openapi_service = openapi_service
+        self._openapi_service = openapi_service
+        self._asset_uploader = AssetUploader(openapi_service)
 
     def _process_single_sample_upload(
         self,
-        asset: MediaAsset,
+        asset: str,
         identifier: str,
-    ) -> tuple[MediaAsset | None, MediaAsset | None]:
+    ) -> tuple[str | None, str | None]:
         """
         Process single sample upload with retry logic and error tracking.
 
@@ -34,29 +39,35 @@ class BenchmarkParticipant:
         Returns:
             tuple[MediaAsset | None, MediaAsset | None]: (successful_asset, failed_asset)
         """
-        if asset.is_local():
-            files = [asset.to_file()]
-            urls = []
-        else:
-            files = []
-            urls = [asset.path]
+        from rapidata.api_client.models.create_sample_model import CreateSampleModel
+        from rapidata.api_client.models.i_asset_input_existing_asset_input import (
+            IAssetInputExistingAssetInput,
+        )
+
+        from rapidata.api_client.models.i_asset_input import IAssetInput
 
         last_exception = None
-        for attempt in range(rapidata_config.uploadMaxRetries):
+        for attempt in range(rapidata_config.upload.maxRetries):
             try:
                 with suppress_rapidata_error_logging():
-                    self.__openapi_service.participant_api.participant_participant_id_sample_post(
+                    self._openapi_service.participant_api.participant_participant_id_sample_post(
                         participant_id=self.id,
-                        model=CreateSampleModel(identifier=identifier),
-                        files=files,
-                        urls=urls,
+                        create_sample_model=CreateSampleModel(
+                            identifier=identifier,
+                            asset=IAssetInput(
+                                actual_instance=IAssetInputExistingAssetInput(
+                                    _t="ExistingAssetInput",
+                                    name=self._asset_uploader.upload_asset(asset),
+                                ),
+                            ),
+                        ),
                     )
 
                 return asset, None
 
             except Exception as e:
                 last_exception = e
-                if attempt < rapidata_config.uploadMaxRetries - 1:
+                if attempt < rapidata_config.upload.maxRetries - 1:
                     # Exponential backoff: wait 1s, then 2s, then 4s
                     retry_delay = 2**attempt
                     time.sleep(retry_delay)
@@ -64,7 +75,7 @@ class BenchmarkParticipant:
                     logger.debug(
                         "Retrying %s of %s...",
                         attempt + 1,
-                        rapidata_config.uploadMaxRetries,
+                        rapidata_config.upload.maxRetries,
                     )
 
         logger.error(f"Upload failed for {identifier}. Error: {str(last_exception)}")
@@ -72,29 +83,44 @@ class BenchmarkParticipant:
 
     def upload_media(
         self,
-        assets: list[MediaAsset],
+        assets: list[str],
         identifiers: list[str],
-    ) -> tuple[list[MediaAsset], list[MediaAsset]]:
+    ) -> tuple[list[str], list[str]]:
         """
         Upload samples concurrently with proper error handling and progress tracking.
 
         Args:
-            assets: List of MediaAsset objects to upload
+            assets: List of strings to upload
             identifiers: List of identifiers matching the assets
 
         Returns:
             tuple[list[str], list[str]]: Lists of successful and failed identifiers
         """
-        successful_uploads: list[MediaAsset] = []
-        failed_uploads: list[MediaAsset] = []
+
+        def upload_with_context(
+            context: otel_context.Context, asset: str, identifier: str
+        ) -> tuple[str | None, str | None]:
+            """Wrapper function that runs _process_single_sample_upload with the provided context."""
+            token = otel_context.attach(context)
+            try:
+                return self._process_single_sample_upload(asset, identifier)
+            finally:
+                otel_context.detach(token)
+
+        successful_uploads: list[str] = []
+        failed_uploads: list[str] = []
         total_uploads = len(assets)
 
+        # Capture the current OpenTelemetry context before creating threads
+        current_context = otel_context.get_current()
+
         with ThreadPoolExecutor(
-            max_workers=rapidata_config.maxUploadWorkers
+            max_workers=rapidata_config.upload.maxWorkers
         ) as executor:
             futures = [
                 executor.submit(
-                    self._process_single_sample_upload,
+                    upload_with_context,
+                    current_context,
                     asset,
                     identifier,
                 )
@@ -104,18 +130,17 @@ class BenchmarkParticipant:
             with tqdm(
                 total=total_uploads,
                 desc="Uploading media",
-                disable=RapidataOutputManager.silent_mode,
+                disable=rapidata_config.logging.silent_mode,
             ) as pbar:
                 for future in as_completed(futures):
                     try:
                         successful_id, failed_id = future.result()
                         if successful_id:
                             successful_uploads.append(successful_id)
+                            pbar.update(1)
                         if failed_id:
                             failed_uploads.append(failed_id)
                     except Exception as e:
                         logger.error(f"Future execution failed: {str(e)}")
 
-                    pbar.update(1)
-
         return successful_uploads, failed_uploads