rapidata 1.8.3__py3-none-any.whl → 2.44.4__py3-none-any.whl

rapidata/rapidata_client/order/rapidata_results.py

@@ -0,0 +1,251 @@
+from typing import TYPE_CHECKING, Any
+from rapidata.rapidata_client.config import managed_print
+import json
+
+if TYPE_CHECKING:
+    import pandas as pd
+    from pandas.core.indexes.base import Index
+
+
+class RapidataResults(dict):
+    """
+    A specialized dictionary class for handling Rapidata API results.
+    Extends the built-in dict class with specialized methods.
+    """
+
+    def to_pandas(self, split_details: bool = False) -> "pd.DataFrame":
+        """
+        Warning:
+            This method is currently under development. The structure of the results may change in the future.
+
+        Converts the results to a pandas DataFrame.
+
+        For Compare results, creates standardized A/B columns for metrics.
+        For regular results, flattens nested dictionaries into columns with underscore-separated names.
+
+        Args:
+            split_details: If True, splits each datapoint by its detailed results,
+                          creating a row for each response with global metrics copied.
+
+        Returns:
+            pd.DataFrame: A DataFrame containing the processed results
+
+        Raises:
+            ValueError: If split_details is True but no detailed results are found
+        """
+        import pandas as pd
+        from pandas.core.indexes.base import Index
+
+        if "results" not in self or not self["results"]:
+            return pd.DataFrame()
+
+        if self["info"].get("orderType") is None:
+            managed_print(
+                "Warning: Results are old and Order type is not specified. Dataframe might be wrong."
+            )
+
+        # Check for detailed results if split_details is True
+        if split_details:
+            if not self._has_detailed_results():
+                raise ValueError("No detailed results found in the data")
+            return self._to_pandas_with_detailed_results()
+
+        if (
+            self["info"].get("orderType") == "Compare"
+            or self["info"].get("orderType") == "Ranking"
+        ):
+            return self._compare_to_pandas()
+
+        # Get the structure from first item
+        first_item = self["results"][0]
+        columns = []
+        path_map = {}  # Maps flattened column names to paths to reach the values
+
+        # Build the column structure once
+        self._build_column_structure(first_item, columns, path_map)
+
+        # Extract data using the known structure
+        data = []
+        for item in self["results"]:
+            row = []
+            for path in path_map.values():
+                value = self._get_value_from_path(item, path)
+                row.append(value)
+            data.append(row)
+
+        return pd.DataFrame(data, columns=Index(columns))
+
+    def _has_detailed_results(self) -> bool:
+        """
+        Checks if the results contain detailed results.
+
+        Returns:
+            bool: True if detailed results exist, False otherwise
+        """
+        if not self.get("results"):
+            return False
+
+        first_result = self["results"][0]
+        return "detailedResults" in first_result and isinstance(
+            first_result["detailedResults"], list
+        )
+
+    def _to_pandas_with_detailed_results(self) -> "pd.DataFrame":
+        """
+        Converts results to a pandas DataFrame with detailed results split into separate rows.
+
+        Returns:
+            pd.DataFrame: A DataFrame with one row per detailed result
+        """
+        import pandas as pd
+
+        rows = []
+
+        for result in self["results"]:
+            # Get all non-detailed results fields
+            base_data = {k: v for k, v in result.items() if k != "detailedResults"}
+
+            # Process each detailed result
+            for detailed_result in result["detailedResults"]:
+                row = base_data.copy()  # Copy base data for each detailed result
+
+                # Add flattened detailed result data
+                flattened = self._flatten_dict(detailed_result)
+                for key, value in flattened.items():
+                    row[key] = value
+
+                rows.append(row)
+
+        return pd.DataFrame(rows)
+
+    def _flatten_dict(self, d: dict[str, Any], parent_key: str = "") -> dict[str, Any]:
+        """
+        Flattens a nested dictionary into a single-level dictionary with underscore-separated keys.
+
+        Args:
+            d: The dictionary to flatten
+            parent_key: The parent key for nested dictionaries
+
+        Returns:
+            dict: A flattened dictionary
+        """
+        items: list[tuple[str, Any]] = []
+
+        for key, value in d.items():
+            new_key = f"{parent_key}_{key}" if parent_key else key
+
+            if isinstance(value, dict):
+                items.extend(self._flatten_dict(value, new_key).items())
+            else:
+                items.append((new_key, value))
+
+        return dict(items)
+
+    def _build_column_structure(
+        self,
+        d: dict[str, Any],
+        columns: list[str],
+        path_map: dict[str, list[str]],
+        parent_key: str = "",
+        current_path: list[str] | None = None,
+    ) -> None:
+        """
+        Builds the column structure and paths to reach values in nested dictionaries.
+
+        Args:
+            d: The dictionary to analyze
+            columns: List to store column names
+            path_map: Dictionary mapping column names to paths for accessing values
+            parent_key: The parent key for nested dictionaries
+            current_path: The current path in the dictionary structure
+        """
+        if current_path is None:
+            current_path = []
+
+        for key, value in d.items():
+            new_key = f"{parent_key}_{key}" if parent_key else key
+            new_path: list[str] = current_path + [key]
+
+            if isinstance(value, dict):
+                self._build_column_structure(
+                    value, columns, path_map, new_key, new_path
+                )
+            else:
+                columns.append(new_key)
+                path_map[new_key] = new_path
+
+    def _get_value_from_path(self, d: dict[str, Any], path: list[str]) -> Any:
+        """
+        Retrieves a value from a nested dictionary using a path list.
+
+        Args:
+            d: The dictionary to retrieve the value from
+            path: List of keys forming the path to the desired value
+
+        Returns:
+            The value at the specified path, or None if the path doesn't exist
+        """
+        for key in path[:-1]:
+            d = d.get(key, {})
+        return d.get(path[-1])
+
+    def _compare_to_pandas(self) -> "pd.DataFrame":
+        """
+        Converts Compare results to a pandas DataFrame dynamically.
+        """
+        import pandas as pd
+
+        if not self.get("results"):
+            return pd.DataFrame()
+
+        rows = []
+        for result in self["results"]:
+            # Get all asset names from the first metric we find
+            assets = []
+            for key in result:
+                if isinstance(result[key], dict) and len(result[key]) >= 2:
+                    assets = list(result[key].keys())
+                    break
+            else:
+                continue
+
+            assets = [asset for asset in assets if asset not in ["Both", "Neither"]]
+
+            # Initialize row with non-comparative fields
+            row = {
+                key: value
+                for key, value in result.items()
+                if not isinstance(value, dict)
+            }
+            row["assetA"] = assets[0]
+            row["assetB"] = assets[1]
+
+            # Handle comparative metrics
+            for key, values in result.items():
+                if isinstance(values, dict) and len(values) >= 2:
+                    # Add main asset columns
+                    for i, asset in enumerate(
+                        assets[:2]
+                    ):  # Limit to first 2 main assets
+                        column_prefix = "A_" if i == 0 else "B_"
+                        row[f"{column_prefix}{key}"] = values.get(asset, 0)
+
+                    # Add special option columns if they exist
+                    if "Both" in values:
+                        row[f"Both_{key}"] = values.get("Both", 0)
+                    if "Neither" in values:
+                        row[f"Neither_{key}"] = values.get("Neither", 0)
+
+            rows.append(row)
+
+        return pd.DataFrame(rows)
+
+    def to_json(self, path: str = "./results.json") -> None:
+        """
+        Saves the results to a JSON file.
+
+        Args:
+            path: The file path where the JSON should be saved. Defaults to "./results.json".
+        """
+        with open(path, "w") as f:
+            json.dump(self, f)

rapidata/rapidata_client/rapidata_client.py

@@ -1,207 +1,152 @@
-from rapidata.rapidata_client.dataset.rapidata_validation_set import RapidataValidationSet
-from rapidata.rapidata_client.dataset.validation_set_builder import ValidationSetBuilder
-from rapidata.rapidata_client.order.rapidata_order_builder import RapidataOrderBuilder
+import json
+from typing import Any
+import requests
+from packaging import version
+from rapidata import __version__
+import uuid
+import random
 from rapidata.service.openapi_service import OpenAPIService
-from rapidata.rapidata_client.order.rapidata_order import RapidataOrder
-from rapidata.rapidata_client.dataset.rapidata_dataset import RapidataDataset
 
-from rapidata.rapidata_client.simple_builders.simple_classification_builders import ClassificationQuestionBuilder
-from rapidata.rapidata_client.simple_builders.simple_compare_builders import CompareCriteriaBuilder
-from rapidata.rapidata_client.simple_builders.simple_free_text_builders import FreeTextQuestionBuilder
-from rapidata.rapidata_client.simple_builders.simple_transcription_builders import TranscriptionInstructionBuilder
+from rapidata.rapidata_client.order.rapidata_order_manager import RapidataOrderManager
+from rapidata.rapidata_client.benchmark.rapidata_benchmark_manager import (
+    RapidataBenchmarkManager,
+)
 
-from rapidata.rapidata_client.dataset.rapid_builders import BaseRapidBuilder
+from rapidata.rapidata_client.validation.validation_set_manager import (
+    ValidationSetManager,
+)
 
-from rapidata.api_client.exceptions import BadRequestException
-from urllib3._collections import HTTPHeaderDict
+from rapidata.rapidata_client.demographic.demographic_manager import DemographicManager
 
-from rapidata.api_client.models.query_model import QueryModel
-from rapidata.api_client.models.page_info import PageInfo
-from rapidata.api_client.models.root_filter import RootFilter
-from rapidata.api_client.models.filter import Filter
-from rapidata.api_client.models.sort_criterion import SortCriterion
-
-from rapidata.api_client.models.query_validation_set_model import QueryValidationSetModel
-
-from deprecated import deprecated
+from rapidata.rapidata_client.config import (
+    logger,
+    tracer,
+    managed_print,
+    rapidata_config,
+)
 
 
 class RapidataClient:
     """The Rapidata client is the main entry point for interacting with the Rapidata API. It allows you to create orders and validation sets."""
 
-    rapid_builder = BaseRapidBuilder()
-    
     def __init__(
         self,
         client_id: str | None = None,
         client_secret: str | None = None,
-        enviroment: str = "rapidata.ai",
-        oauth_scope: str = "openid",
+        environment: str = "rapidata.ai",
+        oauth_scope: str = "openid roles",
         cert_path: str | None = None,
+        token: dict | None = None,
+        leeway: int = 60,
     ):
-        """Initialize the RapidataClient. Best practice is to store the client ID and client secret in environment variables. Ask your Rapidata representative for the client ID and client secret.
+        """Initialize the RapidataClient. If both the client_id and client_secret are None, it will try using your credentials under "~/.config/rapidata/credentials.json".
+        If this is not successful, it will open a browser window and ask you to log in, then save your new credentials in said json file.
 
         Args:
             client_id (str): The client ID for authentication.
             client_secret (str): The client secret for authentication.
-            endpoint (str, optional): The API endpoint URL.
+            environment (str, optional): The API endpoint.
+            oauth_scope (str, optional): The scopes to use for authentication. In general this does not need to be changed.
+            cert_path (str, optional): An optional path to a certificate file useful for development.
+            token (dict, optional): If you already have a token that the client should use for authentication. Important, if set, this needs to be the complete token object containing the access token, token type and expiration time.
+            leeway (int, optional): An optional leeway to use to determine if a token is expired. Defaults to 60 seconds.
+
+        Attributes:
+            order (RapidataOrderManager): The RapidataOrderManager instance.
+            validation (ValidationSetManager): The ValidationSetManager instance.
+            demographic (DemographicManager): The DemographicManager instance.
+            mri (RapidataBenchmarkManager): The RapidataBenchmarkManager instance.
         """
-        self.openapi_service = OpenAPIService(
-            client_id=client_id,
-            client_secret=client_secret,
-            enviroment=enviroment,
-            oauth_scope=oauth_scope,
-            cert_path=cert_path
+        tracer.set_session_id(
+            uuid.UUID(int=random.Random().getrandbits(128), version=4).hex
         )
-    
-    @deprecated("Use the specific builder methods instead.")
-    def new_order(self, name: str) -> RapidataOrderBuilder:
-        """Create a new order using a RapidataOrderBuilder instance.
-
-        Args:
-            name (str): The name of the order.
-
-        Returns:
-            RapidataOrderBuilder: A RapidataOrderBuilder instance.
-        """
-        return RapidataOrderBuilder(openapi_service=self.openapi_service, name=name)
-
-    def new_validation_set(self, name: str) -> ValidationSetBuilder:
-        """Create a new validation set using a ValidationDatasetBuilder instance.
-
-        Args:
-            name (str): The name of the validation set.
-
-        Returns:
-            ValidationSetBuilder: A ValidationDatasetBuilder instance.
-        """
-        return ValidationSetBuilder(name=name, openapi_service=self.openapi_service)
-
-    def get_order(self, order_id: str) -> RapidataOrder:
-        """Get an order by ID.
-
-        Args:
-            order_id (str): The ID of the order.
 
-        Returns:
-            RapidataOrder: The Order instance.
-        """
-
-        # TODO: check the pipeline for the dataset id - not really necessary atm
-        # order = self.openapi_service.order_api.order_get_by_id_get(order_id)
-        # pipeline = self.openapi_service..pipeline_get_by_id_get(order.pipeline_id)
+        with tracer.start_as_current_span("RapidataClient.__init__"):
+            logger.debug("Checking version")
+            self._check_version()
+            if environment != "rapidata.ai":
+                rapidata_config.logging.enable_otlp = False
+
+            logger.debug("Initializing OpenAPIService")
+            self._openapi_service = OpenAPIService(
+                client_id=client_id,
+                client_secret=client_secret,
+                environment=environment,
+                oauth_scope=oauth_scope,
+                cert_path=cert_path,
+                token=token,
+                leeway=leeway,
+            )
+
+            logger.debug("Initializing RapidataOrderManager")
+            self.order = RapidataOrderManager(openapi_service=self._openapi_service)
+
+            logger.debug("Initializing ValidationSetManager")
+            self.validation = ValidationSetManager(
+                openapi_service=self._openapi_service
+            )
+
+            logger.debug("Initializing DemographicManager")
+            self._demographic = DemographicManager(
+                openapi_service=self._openapi_service
+            )
+
+            logger.debug("Initializing RapidataBenchmarkManager")
+            self.mri = RapidataBenchmarkManager(openapi_service=self._openapi_service)
+
+        self._check_beta_features()  # can't be in the trace for some reason
+
+    def reset_credentials(self):
+        """Reset the credentials saved in the configuration file for the current environment."""
+        self._openapi_service.reset_credentials()
+
+    def clear_all_caches(self):
+        """Clear all caches for the client."""
+        self.order._asset_uploader.clear_cache()
+        logger.info("All caches cleared")
+
+    def _check_beta_features(self):
+        """Enable beta features for the client."""
+        with tracer.start_as_current_span("RapidataClient.check_beta_features"):
+            result: dict[str, Any] = json.loads(
+                self._openapi_service.api_client.call_api(
+                    "GET",
+                    f"https://auth.{self._openapi_service.environment}/connect/userinfo",
+                )
+                .read()
+                .decode("utf-8")
+            )
+            logger.debug("Userinfo: %s", result)
+            if "Admin" not in result.get("role", []):
+                logger.debug("User is not an admin, not enabling beta features")
+                return
+
+            logger.debug("User is an admin, enabling beta features")
+            rapidata_config.enableBetaFeatures = True
+
+    def _check_version(self):
         try:
-            order = self.openapi_service.order_api.order_get_by_id_get(order_id)
-        except Exception:
-            raise ValueError(f"Order with ID {order_id} not found.")
-
-        temp_dataset = RapidataDataset("temp", self.openapi_service)
-        return RapidataOrder(
-            dataset=temp_dataset, 
-            order_id=order_id, 
-            name=order.order_name,
-            openapi_service=self.openapi_service)
-
-    def find_orders(self, name: str = "", amount: int = 1) -> list[RapidataOrder]:
-        """Find your recent orders given criteria. If nothing is provided, it will return the most recent order.
-
-        Args:
-            name (str, optional): The name of the order - matching order will contain the name. Defaults to "" for any order.
-            amount (int, optional): The amount of orders to return. Defaults to 1.
-
-        Returns:
-            list[RapidataOrder]: A list of RapidataOrder instances.
-        """
-        try:
-            order_page_result = self.openapi_service.order_api.order_query_get(QueryModel(
-                page=PageInfo(index=1, size=amount),
-                filter=RootFilter(filters=[Filter(field="OrderName", operator="Contains", value=name)]),
-                sortCriteria=[SortCriterion(direction="Desc", propertyName="OrderDate")]
-                ))
-
-        except BadRequestException as e:
-            raise ValueError(f"Error occured during request. \nError: {e.body} \nTraceid: {e.headers.get('X-Trace-Id') if isinstance(e.headers, HTTPHeaderDict) else 'Unknown'}")
-
+            response = requests.get(
+                "https://api.github.com/repos/RapidataAI/rapidata-python-sdk/releases/latest",
+                headers={"Accept": "application/vnd.github.v3+json"},
+                timeout=3,
+            )
+            if response.status_code == 200:
+                latest_version = response.json()["tag_name"].lstrip("v")
+                if version.parse(latest_version) > version.parse(__version__):
+                    managed_print(
+                        f"""A new version of the Rapidata SDK is available: {latest_version}
+Your current version is: {__version__}"""
+                    )
+                else:
+                    logger.debug(
+                        "Current version is up to date. Version: %s", __version__
+                    )
         except Exception as e:
-            raise ValueError(f"Unknown error occured: {e}")
-
-        orders = [self.get_order(order.id) for order in order_page_result.items]
-        return orders
-    
-    def get_validation_set(self, validation_set_id: str) -> RapidataValidationSet:
-        """Get a validation set by ID.
+            logger.debug("Failed to check for updates: %s", e)
 
-        Args:
-            validation_set_id (str): The ID of the validation set.
-
-        Returns:
-            RapidataValidationSet: The ValidationSet instance.
-        """
-        try:
-            validation_set = self.openapi_service.validation_api.validation_get_by_id_get(id=validation_set_id)
-        except Exception:
-            raise ValueError(f"ValidationSet with ID {validation_set_id} not found.")
-        
-        return RapidataValidationSet(validation_set_id, self.openapi_service, validation_set.name)
-    
-    def find_validation_sets(self, name: str = "", amount: int = 1) -> list[RapidataValidationSet]:
-        try:
-            validation_page_result = self.openapi_service.validation_api.validation_query_validation_sets_get(QueryValidationSetModel(
-                pageInfo=PageInfo(index=1, size=amount),
-                filter=RootFilter(filters=[Filter(field="Name", operator="Contains", value=name)]),
-                sortCriteria=[SortCriterion(direction="Desc", propertyName="CreatedAt")]
-                ))
-
-        except BadRequestException as e:
-            raise ValueError(f"Error occured during request. \nError: {e.body} \nTraceid: {e.headers.get('X-Trace-Id') if isinstance(e.headers, HTTPHeaderDict) else 'Unknown'}")
-
-        except Exception as e:
-            raise ValueError(f"Unknown error occured: {e}")
-
-        orders = [self.get_validation_set(validation_set.id) for validation_set in validation_page_result.items] # type: ignore # will be fixed with the next backend deployment
-        return orders
-
-    def create_classify_order(self, name: str) -> ClassificationQuestionBuilder:
-        """Create a new classification order where people are asked to classify an image.
-
-        Args:
-            name (str): The name of the order.
-
-        Returns:
-            ClassificationQuestionBuilder: A ClassificationQuestionBuilder instance.
-        """
-        return ClassificationQuestionBuilder(name=name, openapi_service=self.openapi_service)
-
-    def create_compare_order(self, name: str) -> CompareCriteriaBuilder:
-        """Create a new comparison order where people are asked to compare two images.
-
-        Args:
-            name (str): The name of the order.
-
-        Returns:
-            CompareQuestionBuilder: A CompareQuestionBuilder instance.
-        """
-        return CompareCriteriaBuilder(name=name, openapi_service=self.openapi_service)
-    
-    def create_free_text_order(self, name: str) -> FreeTextQuestionBuilder:
-        """Create a new free text order where people are asked to provide a free text answer.
-
-        Args:
-            name (str): The name of the order.
-
-        Returns:
-            FreeTextQuestionBuilder: A FreeTextQuestionBuilder instance.
-        """
-        return FreeTextQuestionBuilder(name=name, openapi_service=self.openapi_service)
-    
-    def create_transcription_order(self, name: str) -> TranscriptionInstructionBuilder:
-        """Create a new transcription order where people are asked to transcribe an audio file.
-
-        Args:
-            name (str): The name of the order.
-
-        Returns:
-            TranscriptionInstructionBuilder: A TranscriptionInstructionBuilder instance.
-        """
-        return TranscriptionInstructionBuilder(name=name, openapi_service=self.openapi_service)
+    def __str__(self) -> str:
+        return f"RapidataClient(environment={self._openapi_service.environment})"
 
+    def __repr__(self) -> str:
+        return self.__str__()

rapidata/rapidata_client/referee/__init__.py

@@ -1,3 +1,3 @@
-from .base_referee import Referee
-from .naive_referee import NaiveReferee #as MaxVoteReferee
-from .early_stopping_referee import EarlyStoppingReferee
+from ._base_referee import Referee
+from ._naive_referee import NaiveReferee  # as MaxVoteReferee
+from ._early_stopping_referee import EarlyStoppingReferee

rapidata/rapidata_client/referee/{base_referee.py → _base_referee.py}

@@ -1,20 +1,24 @@
 from abc import ABC, abstractmethod
-from typing import Any, Mapping
+from typing import Mapping
+
+from rapidata.api_client.models.i_referee_model import IRefereeModel
+
 
 class Referee(ABC):
     """
-    The referee defines when a rapid is considered complete. 
+    The referee defines when a rapid is considered complete.
     """
+
     @abstractmethod
-    def to_dict(self) -> Mapping[str, str | int | float]:
+    def _to_dict(self) -> Mapping[str, str | int | float]:
         """
         Convert the referee to a referee configuration dict.
         """
         pass
 
     @abstractmethod
-    def to_model(self) -> Any:
+    def _to_model(self) -> IRefereeModel:
         """
         Convert the referee to a referee configuration model.
         """
-        pass
+        pass