rapidata 2.28.5__py3-none-any.whl → 2.29.1__py3-none-any.whl

rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py

@@ -0,0 +1,83 @@
+from rapidata.rapidata_client.benchmark.rapidata_benchmark import RapidataBenchmark
+from rapidata.api_client.models.create_benchmark_model import CreateBenchmarkModel
+from rapidata.service.openapi_service import OpenAPIService
+from rapidata.api_client.models.query_model import QueryModel
+from rapidata.api_client.models.page_info import PageInfo
+from rapidata.api_client.models.root_filter import RootFilter
+from rapidata.api_client.models.filter import Filter
+from rapidata.api_client.models.sort_criterion import SortCriterion
+
+class RapidataBenchmarkManager:
+    """
+    A manager for benchmarks.
+
+    Used to create and retrieve benchmarks.
+
+    A benchmark is a collection of leaderboards.
+
+    Args:
+        openapi_service: The OpenAPIService instance for API interaction.
+    """
+    def __init__(self, openapi_service: OpenAPIService):
+        self.__openapi_service = openapi_service
+
+    def create_new_benchmark(self, 
+                             name: str,
+                             identifiers: list[str],
+                             prompts: list[str],
+                             ) -> RapidataBenchmark:
+        """
+        Creates a new benchmark with the given name, prompts, and leaderboards.
+
+        Args:
+            name: The name of the benchmark.
+            prompts: The prompts that will be registered for the benchmark.
+        """
+        if not isinstance(name, str):
+            raise ValueError("Name must be a string.")
+        
+        if not isinstance(prompts, list) or not all(isinstance(prompt, str) for prompt in prompts):
+            raise ValueError("Prompts must be a list of strings.")
+        
+        if not isinstance(identifiers, list) or not all(isinstance(identifier, str) for identifier in identifiers):
+            raise ValueError("Identifiers must be a list of strings.")
+        
+        if len(identifiers) != len(prompts):
+            raise ValueError("Identifiers and prompts must have the same length.")
+        
+        if len(set(identifiers)) != len(identifiers):
+            raise ValueError("Identifiers must be unique.")
+
+        benchmark_result = self.__openapi_service.benchmark_api.benchmark_post(
+            create_benchmark_model=CreateBenchmarkModel(
+                name=name,
+            )
+        )
+
+        benchmark = RapidataBenchmark(name, benchmark_result.id, self.__openapi_service)
+        for identifier, prompt in zip(identifiers, prompts):
+            benchmark.add_prompt(identifier, prompt)
+
+        return benchmark
+    
+    def get_benchmark_by_id(self, id: str) -> RapidataBenchmark:
+        """
+        Returns a benchmark by its ID.
+        """
+        benchmark_result = self.__openapi_service.benchmark_api.benchmark_benchmark_id_get(
+            benchmark_id=id
+        )
+        return RapidataBenchmark(benchmark_result.name, benchmark_result.id, self.__openapi_service)
+    
+    def find_benchmarks(self, name: str = "", amount: int = 10) -> list[RapidataBenchmark]:
+        """
+        Returns a list of benchmarks by their name.
+        """
+        benchmark_result = self.__openapi_service.benchmark_api.benchmarks_get(
+            QueryModel(
+                page=PageInfo(index=1, size=amount),
+                filter=RootFilter(filters=[Filter(field="Name", operator="Contains", value=name)]),
+                sortCriteria=[SortCriterion(direction="Desc", propertyName="CreatedAt")]
+            )
+        )
+        return [RapidataBenchmark(benchmark.name, benchmark.id, self.__openapi_service) for benchmark in benchmark_result.items]

rapidata/rapidata_client/filter/not_filter.py

@@ -1,7 +1,7 @@
 from typing import Any
 from rapidata.rapidata_client.filter._base_filter import RapidataFilter
 from rapidata.api_client.models.not_user_filter_model import NotUserFilterModel
-from rapidata.api_client.models.create_order_model_user_filters_inner import CreateOrderModelUserFiltersInner
+from rapidata.api_client.models.and_user_filter_model_filters_inner import AndUserFilterModelFiltersInner
 
 
 class NotFilter(RapidataFilter):
@@ -27,4 +27,4 @@ class NotFilter(RapidataFilter):
         self.filter = filter
 
     def _to_model(self):
-        return NotUserFilterModel(_t="NotFilter", filter=CreateOrderModelUserFiltersInner(self.filter._to_model()))
+        return NotUserFilterModel(_t="NotFilter", filter=AndUserFilterModelFiltersInner(self.filter._to_model()))

rapidata/rapidata_client/filter/or_filter.py

@@ -1,7 +1,7 @@
 from typing import Any
 from rapidata.rapidata_client.filter._base_filter import RapidataFilter
 from rapidata.api_client.models.or_user_filter_model import OrUserFilterModel
-from rapidata.api_client.models.create_order_model_user_filters_inner import CreateOrderModelUserFiltersInner
+from rapidata.api_client.models.and_user_filter_model_filters_inner import AndUserFilterModelFiltersInner
 
 
 class OrFilter(RapidataFilter):
@@ -27,4 +27,4 @@ class OrFilter(RapidataFilter):
         self.filters = filters
 
     def _to_model(self):
-        return OrUserFilterModel(_t="OrFilter", filters=[CreateOrderModelUserFiltersInner(filter._to_model()) for filter in self.filters])
+        return OrUserFilterModel(_t="OrFilter", filters=[AndUserFilterModelFiltersInner(filter._to_model()) for filter in self.filters])

rapidata/rapidata_client/metadata/__init__.py

@@ -4,3 +4,4 @@ from ._public_text_metadata import PublicTextMetadata
 from ._prompt_metadata import PromptMetadata
 from ._select_words_metadata import SelectWordsMetadata
 from ._media_asset_metadata import MediaAssetMetadata
+from ._prompt_identifier_metadata import PromptIdentifierMetadata

rapidata/rapidata_client/metadata/_media_asset_metadata.py

@@ -1,6 +1,7 @@
 from rapidata.api_client.models.prompt_asset_metadata_input import PromptAssetMetadataInput
 from rapidata.api_client.models.url_asset_input import UrlAssetInput
 from rapidata.rapidata_client.metadata._base_metadata import Metadata
+from rapidata.api_client.models.prompt_asset_metadata_input_asset import PromptAssetMetadataInputAsset
 
 
 class MediaAssetMetadata(Metadata):
@@ -11,5 +12,11 @@ class MediaAssetMetadata(Metadata):
 
     def to_model(self):
         return PromptAssetMetadataInput(
-            _t="PromptAssetMetadataInput", asset=UrlAssetInput(_t="UrlAssetInput", url=self._url)
+            _t="PromptAssetMetadataInput", 
+            asset=PromptAssetMetadataInputAsset(
+                actual_instance=UrlAssetInput(
+                    _t="UrlAssetInput", 
+                    url=self._url
+                )
+            )
         )

rapidata/rapidata_client/metadata/_prompt_identifier_metadata.py

@@ -0,0 +1,15 @@
+from rapidata.rapidata_client.metadata._base_metadata import Metadata
+from rapidata.api_client.models.private_text_metadata_input import (
+    PrivateTextMetadataInput,
+)
+
+
+class PromptIdentifierMetadata(Metadata):
+    def __init__(self, identifier: str):
+        super().__init__()
+        self._identifier = identifier
+
+    def to_model(self):
+        return PrivateTextMetadataInput(
+            _t="PrivateTextMetadataInput", identifier="prompt-id", text=self._identifier
+        )

rapidata/rapidata_client/order/_rapidata_dataset.py

@@ -22,7 +22,7 @@ def chunk_list(lst: list, chunk_size: int) -> Generator:
 class RapidataDataset:
 
     def __init__(self, dataset_id: str, openapi_service: OpenAPIService):
-        self.dataset_id = dataset_id
+        self.id = dataset_id
         self.openapi_service = openapi_service
         self.local_file_service = LocalFileService()
 
@@ -96,7 +96,7 @@ class RapidataDataset:
                 metadata=metadata,
             )
 
-            self.openapi_service.dataset_api.dataset_dataset_id_datapoints_texts_post(dataset_id=self.dataset_id, create_datapoint_from_text_sources_model=model)
+            self.openapi_service.dataset_api.dataset_dataset_id_datapoints_texts_post(dataset_id=self.id, create_datapoint_from_text_sources_model=model)
 
         total_uploads = len(text_assets)
         with ThreadPoolExecutor(max_workers=max_workers) as executor:
@@ -161,7 +161,7 @@ class RapidataDataset:
         for attempt in range(max_retries):
             try:
                 self.openapi_service.dataset_api.dataset_dataset_id_datapoints_post(
-                    dataset_id=self.dataset_id,
+                    dataset_id=self.id,
                     file=local_paths,
                     url=urls,
                     metadata=metadata,
@@ -222,7 +222,7 @@ class RapidataDataset:
                     
                     while not stop_event.is_set() or not all_uploads_complete.is_set():
                         try:
-                            current_progress = self.openapi_service.dataset_api.dataset_dataset_id_progress_get(self.dataset_id)
+                            current_progress = self.openapi_service.dataset_api.dataset_dataset_id_progress_get(self.id)
                             
                             # Calculate items completed since our initialization
                             completed_ready = current_progress.ready
@@ -365,7 +365,7 @@ class RapidataDataset:
         """
         try:            
             # Get final progress
-            final_progress = self.openapi_service.dataset_api.dataset_dataset_id_progress_get(self.dataset_id)
+            final_progress = self.openapi_service.dataset_api.dataset_dataset_id_progress_get(self.id)
             total_ready = final_progress.ready
             total_failed = final_progress.failed
             
@@ -373,7 +373,7 @@ class RapidataDataset:
             if total_ready + total_failed < total_uploads:
                 # Try one more time after a longer wait
                 time.sleep(5 * progress_poll_interval)
-                final_progress = self.openapi_service.dataset_api.dataset_dataset_id_progress_get(self.dataset_id)
+                final_progress = self.openapi_service.dataset_api.dataset_dataset_id_progress_get(self.id)
                 total_ready = final_progress.ready
                 total_failed = final_progress.failed
             

rapidata/rapidata_client/order/_rapidata_order_builder.py

@@ -5,8 +5,8 @@ from rapidata.api_client.models.create_order_model import CreateOrderModel
 from rapidata.api_client.models.create_order_model_referee import (
     CreateOrderModelReferee,
 )
-from rapidata.api_client.models.create_order_model_user_filters_inner import (
-    CreateOrderModelUserFiltersInner,
+from rapidata.api_client.models.and_user_filter_model_filters_inner import (
+    AndUserFilterModelFiltersInner,
 )
 from rapidata.api_client.models.create_order_model_workflow import (
     CreateOrderModelWorkflow,
@@ -83,7 +83,7 @@ class RapidataOrderBuilder:
             orderName=self._name,
             workflow=CreateOrderModelWorkflow(self.__workflow._to_model()),
             userFilters=[
-                CreateOrderModelUserFiltersInner(user_filter._to_model())
+                AndUserFilterModelFiltersInner(user_filter._to_model())
                 for user_filter in self.__user_filters
             ],
             referee=CreateOrderModelReferee(self.__referee._to_model()),
@@ -136,7 +136,7 @@ class RapidataOrderBuilder:
             else None
         )
         if self.__dataset:
-            logger.debug(f"Dataset created with ID: {self.__dataset.dataset_id}")
+            logger.debug(f"Dataset created with ID: {self.__dataset.id}")
         else:
             logger.warning("No dataset created for this order.")
 

rapidata/rapidata_client/rapidata_client.py

@@ -5,7 +5,7 @@ from rapidata import __version__
 from rapidata.service.openapi_service import OpenAPIService
 
 from rapidata.rapidata_client.order.rapidata_order_manager import RapidataOrderManager
-from rapidata.rapidata_client.leaderboard.rapidata_leaderboard_manager import RapidataLeaderboardManager
+from rapidata.rapidata_client.benchmark.rapidata_benchmark_manager import RapidataBenchmarkManager
 
 from rapidata.rapidata_client.validation.validation_set_manager import (
     ValidationSetManager,
@@ -67,8 +67,8 @@ class RapidataClient:
         logger.debug("Initializing DemographicManager")
         self._demographic = DemographicManager(openapi_service=self._openapi_service)
 
-        logger.debug("Initializing RapidataLeaderboardManager")
-        self.mri = RapidataLeaderboardManager(openapi_service=self._openapi_service)
+        logger.debug("Initializing RapidataBenchmarkManager")
+        self.mri = RapidataBenchmarkManager(openapi_service=self._openapi_service)
         
     def reset_credentials(self):
         """Reset the credentials saved in the configuration file for the current environment."""

rapidata/service/openapi_service.py

@@ -3,6 +3,7 @@ from importlib.metadata import version, PackageNotFoundError
 
 from rapidata.api_client.api.campaign_api import CampaignApi
 from rapidata.api_client.api.dataset_api import DatasetApi
+from rapidata.api_client.api.benchmark_api import BenchmarkApi
 from rapidata.api_client.api.order_api import OrderApi
 from rapidata.api_client.api.pipeline_api import PipelineApi
 from rapidata.api_client.api.rapid_api import RapidApi
@@ -112,6 +113,10 @@ class OpenAPIService:
     @property
     def leaderboard_api(self) -> LeaderboardApi:
         return LeaderboardApi(self.api_client)
+    
+    @property
+    def benchmark_api(self) -> BenchmarkApi:
+        return BenchmarkApi(self.api_client)
 
     def _get_rapidata_package_version(self):
         """

{rapidata-2.28.5.dist-info → rapidata-2.29.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rapidata
-Version: 2.28.5
+Version: 2.29.1
 Summary: Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way.
 License: Apache-2.0
 Author: Rapidata AG