rapidata 2.28.2__py3-none-any.whl → 2.44.3__py3-none-any.whl

rapidata/rapidata_client/__init__.py

@@ -6,17 +6,18 @@ from .selection import (
     ConditionalValidationSelection,
     CappedSelection,
     ShufflingSelection,
-    RetrievalMode,
-    EffortEstimationSelection,
+    RapidataRetrievalMode,
+    EffortSelection,
 )
-from .metadata import (
+from .datapoints import Datapoint
+from .datapoints.metadata import (
     PrivateTextMetadata,
     PublicTextMetadata,
     PromptMetadata,
     SelectWordsMetadata,
 )
 from .settings import (
-    RapidataSettings, 
+    RapidataSettings,
     TranslationBehaviourOptions,
     AlertOnFastResponse,
     TranslationBehaviour,
@@ -25,13 +26,8 @@ from .settings import (
     PlayVideoUntilTheEnd,
     CustomSetting,
     AllowNeitherBoth,
-    )
-from .country_codes import CountryCodes
-from .assets import (
-    MediaAsset, 
-    TextAsset, 
-    MultiAsset, 
-    RapidataDataTypes
+    SwapContextInstruction,
+    MuteVideo,
 )
 from .filter import (
     CountryFilter,
@@ -47,11 +43,6 @@ from .filter import (
     OrFilter,
     ResponseCountFilter,
 )
-
-from .logging import (
-    configure_logger, 
-    logger,
-    RapidataOutputManager
-)
-
 from .validation import Box
+from .exceptions import FailedUploadException
+from .config import rapidata_config, logger, managed_print

rapidata/rapidata_client/api/rapidata_api_client.py

@@ -0,0 +1,253 @@
+from typing import Optional, Any
+from rapidata.api_client.api_client import (
+    ApiClient,
+    rest,
+    ApiResponse,
+    ApiResponseT,
+)
+from rapidata.api_client.exceptions import ApiException
+import json
+import threading
+from contextlib import contextmanager
+from rapidata.rapidata_client.config import logger, tracer
+from opentelemetry import trace
+from opentelemetry.trace import format_trace_id, format_span_id, Link, SpanContext
+from opentelemetry.sdk.trace.id_generator import RandomIdGenerator
+
+
+# Thread-local storage for controlling error logging
+_thread_local = threading.local()
+
+
+@contextmanager
+def suppress_rapidata_error_logging():
+    """Context manager to suppress error logging for RapidataApiClient calls."""
+    old_value = getattr(_thread_local, "suppress_error_logging", False)
+    _thread_local.suppress_error_logging = True
+    try:
+        yield
+    finally:
+        _thread_local.suppress_error_logging = old_value
+
+
+def _should_suppress_error_logging() -> bool:
+    """Check if error logging should be suppressed for the current thread."""
+    return getattr(_thread_local, "suppress_error_logging", False)
+
+
+class RapidataError(Exception):
+    """Custom error class for Rapidata API errors."""
+
+    def __init__(
+        self,
+        status_code: Optional[int] = None,
+        message: str | None = None,
+        original_exception: Exception | None = None,
+        details: Any = None,
+    ):
+        self.status_code = status_code
+        self.message = message
+        self.original_exception = original_exception
+        self.details = details
+
+        # Create a nice error message
+        error_msg = "Rapidata API Error"
+        if status_code:
+            error_msg += f" ({status_code})"
+        if message:
+            error_msg += f": {message}"
+
+        super().__init__(error_msg)
+
+    def __str__(self):
+        """Return a string representation of the error."""
+        # Extract information from message if available
+        title = None
+        errors = None
+        trace_id = None
+
+        # Try to extract from details if available and is a dict
+        if self.details and isinstance(self.details, dict):
+            title = self.details.get("title")
+            errors = self.details.get("errors")
+            trace_id = self.details.get("traceId")
+
+        # Build the error string
+        error_parts = []
+
+        # Main error line
+        if title:
+            error_parts.append(f"{title}")
+        else:
+            error_parts.append(f"{self.message or 'Unknown error'}")
+
+        # Reasons
+        if errors:
+            if isinstance(errors, dict):
+                error_parts.append(f"Reasons: {json.dumps({'errors': errors})}")
+            else:
+                error_parts.append(f"Reasons: {errors}")
+
+        # Trace ID
+        if trace_id:
+            error_parts.append(f"Trace Id: {trace_id}")
+        else:
+            error_parts.append("Trace Id: N/A")
+
+        return "\n".join(error_parts)
+
+
+class RapidataApiClient(ApiClient):
+    """Custom API client that wraps errors in RapidataError."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.id_generator = RandomIdGenerator()
+
+    def call_api(
+        self,
+        method,
+        url,
+        header_params=None,
+        body=None,
+        post_params=None,
+        _request_timeout=None,
+    ) -> rest.RESTResponse:
+        # Get the current span from OpenTelemetry
+        current_span = trace.get_current_span()
+
+        # Initialize header_params if it's None
+        if header_params is None:
+            header_params = {}
+
+        # Add tracing headers if we have a valid span
+        if not current_span.is_recording():
+            return super().call_api(
+                method,
+                url,
+                header_params,
+                body,
+                post_params,
+                _request_timeout,
+            )
+
+        current_span_context = current_span.get_span_context()
+
+        # Generate a new trace ID for backend communication
+        # This separates the backend trace from the SDK trace
+        backend_trace_id = self.id_generator.generate_trace_id()
+        backend_span_id = self.id_generator.generate_span_id()
+
+        # Create a new span context for the backend trace
+        backend_span_context = SpanContext(
+            trace_id=backend_trace_id,
+            span_id=backend_span_id,
+            is_remote=True,
+            trace_flags=current_span_context.trace_flags,
+        )
+
+        # Create a link from current SDK span to the backend trace
+        link_to_backend = Link(backend_span_context)
+
+        # Create a link from backend trace back to the original SDK span
+        link_back_to_sdk = Link(current_span_context)
+
+        # Create a span in the current SDK trace that links to the backend
+        with tracer.start_span(
+            f"{str(method).upper()} {url}",
+            links=[link_to_backend],
+        ) as sdk_request_span:
+            # Set attributes on the SDK span
+            sdk_request_span.set_attribute("http.method", method)
+            sdk_request_span.set_attribute("http.target", url)
+            sdk_request_span.set_attribute(
+                "rapidata.backend_trace_id", format_trace_id(backend_trace_id)
+            )
+            sdk_request_span.set_attribute(
+                "rapidata.original_trace_id",
+                format_trace_id(current_span_context.trace_id),
+            )
+
+            # Now create the initial span for the backend trace that will be sent
+            # This span will be the starting point for the backend trace
+            with tracer.start_span(
+                f"{str(method).upper()} {url}",
+                context=trace.set_span_in_context(
+                    trace.NonRecordingSpan(backend_span_context)
+                ),
+                links=[link_back_to_sdk],
+            ) as backend_initial_span:
+                # Set attributes on the backend initial span
+                backend_initial_span.set_attribute("http.method", method)
+                backend_initial_span.set_attribute("http.target", url)
+                backend_initial_span.set_attribute(
+                    "rapidata.trace_type", "backend_start"
+                )
+                backend_initial_span.set_attribute(
+                    "rapidata.sdk_trace_id",
+                    format_trace_id(current_span_context.trace_id),
+                )
+
+                # Format the traceparent header with the backend trace ID
+                # The backend will receive this and continue the trace
+                header_params["traceparent"] = (
+                    "00-"
+                    + format_trace_id(backend_trace_id)
+                    + "-"
+                    + format_span_id(backend_initial_span.get_span_context().span_id)
+                    + "-"
+                    + f"{backend_span_context.trace_flags:02x}"
+                )
+
+                return super().call_api(
+                    method,
+                    url,
+                    header_params,
+                    body,
+                    post_params,
+                    _request_timeout,
+                )
+
+    def response_deserialize(
+        self,
+        response_data: rest.RESTResponse,
+        response_types_map: Optional[dict[str, ApiResponseT]] = None,
+    ) -> ApiResponse[ApiResponseT]:
+        """Override the response_deserialize method to catch and convert exceptions."""
+        try:
+            return super().response_deserialize(response_data, response_types_map)
+        except ApiException as e:
+            status_code = getattr(e, "status", None)
+            message = str(e)
+            details = None
+
+            # Extract more detailed error message from response body if available
+            if hasattr(e, "body") and e.body:
+                try:
+                    body_json = json.loads(e.body)
+                    if isinstance(body_json, dict):
+                        if "message" in body_json:
+                            message = body_json["message"]
+                        elif "error" in body_json:
+                            message = body_json["error"]
+
+                        # Store the full error details for debugging
+                        details = body_json
+                except (json.JSONDecodeError, AttributeError):
+                    # If we can't parse the body as JSON, use the original message
+                    pass
+
+            error_formatted = RapidataError(
+                status_code=status_code,
+                message=message,
+                original_exception=e,
+                details=details,
+            )
+
+            # Only log error if not suppressed
+            if not _should_suppress_error_logging():
+                logger.error("Error: %s", error_formatted)
+            else:
+                logger.debug("Suppressed Error: %s", error_formatted)
+
+            raise error_formatted from None

rapidata/rapidata_client/benchmark/_detail_mapper.py

@@ -0,0 +1,32 @@
+from typing import Literal
+
+
+class DetailMapper:
+    @staticmethod
+    def get_budget(
+        level_of_detail: Literal["low", "medium", "high", "very high"],
+    ) -> int:
+        if level_of_detail == "low":
+            return 2_000
+        elif level_of_detail == "medium":
+            return 4_000
+        elif level_of_detail == "high":
+            return 8_000
+        elif level_of_detail == "very high":
+            return 16_000
+        else:
+            raise ValueError(
+                "Invalid level of detail. Must be one of: 'low', 'medium', 'high', 'very high'"
+            )
+
+    @staticmethod
+    def get_level_of_detail(
+        response_budget: int,
+    ) -> Literal["low", "medium", "high", "very high"]:
+        if response_budget < 4_000:
+            return "low"
+        elif response_budget < 8_000:
+            return "medium"
+        elif response_budget < 16_000:
+            return "high"
+        return "very high"

rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py

@@ -0,0 +1,221 @@
+import urllib.parse
+import webbrowser
+from colorama import Fore
+from typing import Literal, Optional, TYPE_CHECKING
+
+from rapidata.rapidata_client.config import logger, managed_print, tracer
+from rapidata.rapidata_client.benchmark._detail_mapper import DetailMapper
+from rapidata.service.openapi_service import OpenAPIService
+from rapidata.api_client.models.update_leaderboard_model import UpdateLeaderboardModel
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+
+class RapidataLeaderboard:
+    """
+    An instance of a Rapidata leaderboard.
+
+    Used to interact with a specific leaderboard in the Rapidata system, such as retrieving prompts and evaluating models.
+
+    Args:
+        name: The name that will be used to identify the leaderboard on the overview.
+        instruction: The instruction that will determine what how the models will be evaluated.
+        show_prompt: Whether to show the prompt to the users.
+        id: The ID of the leaderboard.
+        openapi_service: The OpenAPIService instance for API interaction.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        instruction: str,
+        show_prompt: bool,
+        show_prompt_asset: bool,
+        inverse_ranking: bool,
+        response_budget: int,
+        min_responses_per_matchup: int,
+        benchmark_id: str,
+        id: str,
+        openapi_service: OpenAPIService,
+    ):
+        self.__openapi_service = openapi_service
+        self.__name = name
+        self.__instruction = instruction
+        self.__show_prompt = show_prompt
+        self.__show_prompt_asset = show_prompt_asset
+        self.__inverse_ranking = inverse_ranking
+        self.__response_budget = response_budget
+        self.__min_responses_per_matchup = min_responses_per_matchup
+        self.__benchmark_id = benchmark_id
+        self.id = id
+        self.__leaderboard_page = f"https://app.{self.__openapi_service.environment}/mri/benchmarks/{self.__benchmark_id}/leaderboard/{self.id}"
+
+    @property
+    def level_of_detail(self) -> Literal["low", "medium", "high", "very high"]:
+        """
+        Returns the level of detail of the leaderboard.
+        """
+        return DetailMapper.get_level_of_detail(self.__response_budget)
+
+    @level_of_detail.setter
+    def level_of_detail(
+        self, level_of_detail: Literal["low", "medium", "high", "very high"]
+    ):
+        """
+        Sets the level of detail of the leaderboard.
+        """
+        with tracer.start_as_current_span("RapidataLeaderboard.level_of_detail.setter"):
+            logger.debug(f"Setting level of detail to {level_of_detail}")
+            self.__response_budget = DetailMapper.get_budget(level_of_detail)
+            self._update_config()
+
+    @property
+    def min_responses_per_matchup(self) -> int:
+        """
+        Returns the minimum number of responses required to be considered for the leaderboard.
+        """
+        return self.__min_responses_per_matchup
+
+    @min_responses_per_matchup.setter
+    def min_responses_per_matchup(self, min_responses: int):
+        """
+        Sets the minimum number of responses required to be considered for the leaderboard.
+        """
+        with tracer.start_as_current_span(
+            "RapidataLeaderboard.min_responses_per_matchup.setter"
+        ):
+            if not isinstance(min_responses, int):
+                raise ValueError("Min responses per matchup must be an integer")
+
+            if min_responses < 3:
+                raise ValueError("Min responses per matchup must be at least 3")
+
+            logger.debug(
+                f"Setting min responses per matchup to {min_responses} for leaderboard {self.name}"
+            )
+            self.__min_responses_per_matchup = min_responses
+            self._update_config()
+
+    @property
+    def show_prompt_asset(self) -> bool:
+        """
+        Returns whether the prompt asset is shown to the users.
+        """
+        return self.__show_prompt_asset
+
+    @property
+    def inverse_ranking(self) -> bool:
+        """
+        Returns whether the ranking is inverse.
+        """
+        return self.__inverse_ranking
+
+    @property
+    def show_prompt(self) -> bool:
+        """
+        Returns whether the prompt is shown to the users.
+        """
+        return self.__show_prompt
+
+    @property
+    def instruction(self) -> str:
+        """
+        Returns the instruction of the leaderboard.
+        """
+        return self.__instruction
+
+    @property
+    def name(self) -> str:
+        """
+        Returns the name of the leaderboard.
+        """
+        return self.__name
+
+    @name.setter
+    def name(self, name: str):
+        """
+        Sets the name of the leaderboard.
+        """
+        with tracer.start_as_current_span("RapidataLeaderboard.name.setter"):
+            if not isinstance(name, str):
+                raise ValueError("Name must be a string")
+            if len(name) < 1:
+                raise ValueError("Name must be at least 1 character long")
+
+            self.__name = name
+            self._update_config()
+
+    def get_standings(self, tags: Optional[list[str]] = None) -> "pd.DataFrame":
+        """
+        Returns the standings of the leaderboard.
+
+        Args:
+            tags: The matchups with these tags should be used to create the standings.
+                If tags are None, all matchups will be considered.
+                If tags are empty, no matchups will be considered.
+
+        Returns:
+            A pandas DataFrame containing the standings of the leaderboard.
+        """
+        with tracer.start_as_current_span("RapidataLeaderboard.get_standings"):
+            participants = self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_standings_get(
+                leaderboard_id=self.id, tags=tags
+            )
+
+            import pandas as pd
+
+            standings = []
+            for participant in participants.items:
+                standings.append(
+                    {
+                        "name": participant.name,
+                        "wins": participant.wins,
+                        "total_matches": participant.total_matches,
+                        "score": (
+                            round(participant.score, 2)
+                            if participant.score is not None
+                            else None
+                        ),
+                    }
+                )
+
+            return pd.DataFrame(standings)
+
+    def view(self) -> None:
+        """
+        Views the leaderboard.
+        """
+        logger.info("Opening leaderboard page in browser...")
+        could_open_browser = webbrowser.open(self.__leaderboard_page)
+        if not could_open_browser:
+            encoded_url = urllib.parse.quote(
+                self.__leaderboard_page, safe="%/:=&?~#+!$,;'@()*[]"
+            )
+            managed_print(
+                Fore.RED
+                + f"Please open this URL in your browser: '{encoded_url}'"
+                + Fore.RESET
+            )
+
+    def _custom_config(self, response_budget: int, min_responses_per_matchup: int):
+        self.__response_budget = response_budget
+        self.__min_responses_per_matchup = min_responses_per_matchup
+        self._update_config()
+
+    def _update_config(self):
+        with tracer.start_as_current_span("RapidataLeaderboard._update_config"):
+            self.__openapi_service.leaderboard_api.leaderboard_leaderboard_id_patch(
+                leaderboard_id=self.id,
+                update_leaderboard_model=UpdateLeaderboardModel(
+                    name=self.__name,
+                    responseBudget=self.__response_budget,
+                    minResponses=self.__min_responses_per_matchup,
+                ),
+            )
+
+    def __str__(self) -> str:
+        return f"RapidataLeaderboard(name={self.name}, instruction={self.instruction}, show_prompt={self.show_prompt}, leaderboard_id={self.id})"
+
+    def __repr__(self) -> str:
+        return self.__str__()