rapidata 2.35.2__tar.gz → 2.35.3__tar.gz

{rapidata-2.35.2 → rapidata-2.35.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rapidata
-Version: 2.35.2
+Version: 2.35.3
 Summary: Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way.
 License: Apache-2.0
 Author: Rapidata AG

{rapidata-2.35.2 → rapidata-2.35.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rapidata"
-version = "2.35.2"
+version = "2.35.3"
 description = "Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way."
 authors = ["Rapidata AG <info@rapidata.ai>"]
 license = "Apache-2.0"

{rapidata-2.35.2 → rapidata-2.35.3}/src/rapidata/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.35.2"
+__version__ = "2.35.3"
 
 from .rapidata_client import (
     RapidataClient,
@@ -42,4 +42,5 @@ from .rapidata_client import (
     configure_logger,
     RapidataOutputManager,
     FailedUploadException,
+    rapidata_config,
 )

{rapidata-2.35.2 → rapidata-2.35.3}/src/rapidata/rapidata_client/__init__.py

@@ -16,14 +16,9 @@ from .datapoints.metadata import (
     PromptMetadata,
     SelectWordsMetadata,
 )
-from .datapoints.assets import (
-    MediaAsset, 
-    TextAsset, 
-    MultiAsset, 
-    RapidataDataTypes
-)
+from .datapoints.assets import MediaAsset, TextAsset, MultiAsset, RapidataDataTypes
 from .settings import (
-    RapidataSettings, 
+    RapidataSettings,
     TranslationBehaviourOptions,
     AlertOnFastResponse,
     TranslationBehaviour,
@@ -32,7 +27,7 @@ from .settings import (
     PlayVideoUntilTheEnd,
     CustomSetting,
     AllowNeitherBoth,
-    )
+)
 from .country_codes import CountryCodes
 from .filter import (
     CountryFilter,
@@ -49,11 +44,8 @@ from .filter import (
     ResponseCountFilter,
 )
 
-from .logging import (
-    configure_logger, 
-    logger,
-    RapidataOutputManager
-)
+from .logging import configure_logger, logger, RapidataOutputManager
 
 from .validation import Box
 from .exceptions import FailedUploadException
+from .config import rapidata_config

{rapidata-2.35.2 → rapidata-2.35.3}/src/rapidata/rapidata_client/benchmark/participant/_participant.py

@@ -1,4 +1,5 @@
 from concurrent.futures import ThreadPoolExecutor, as_completed
+import time
 from tqdm import tqdm
 
 from rapidata.rapidata_client.datapoints.assets import MediaAsset
@@ -6,6 +7,10 @@ from rapidata.rapidata_client.logging import logger
 from rapidata.rapidata_client.logging.output_manager import RapidataOutputManager
 from rapidata.api_client.models.create_sample_model import CreateSampleModel
 from rapidata.service.openapi_service import OpenAPIService
+from rapidata.rapidata_client.config.config import rapidata_config
+from rapidata.rapidata_client.api.rapidata_exception import (
+    suppress_rapidata_error_logging,
+)
 
 
 class BenchmarkParticipant:
@@ -21,11 +26,11 @@ class BenchmarkParticipant:
     ) -> tuple[MediaAsset | None, MediaAsset | None]:
         """
         Process single sample upload with retry logic and error tracking.
-        
+
         Args:
             asset: MediaAsset to upload
             identifier: Identifier for the sample
-            
+
         Returns:
             tuple[MediaAsset | None, MediaAsset | None]: (successful_asset, failed_asset)
         """
@@ -37,20 +42,30 @@ class BenchmarkParticipant:
             urls = [asset.path]
 
         last_exception = None
-        try:
-            self.__openapi_service.participant_api.participant_participant_id_sample_post(
-                participant_id=self.id,
-                model=CreateSampleModel(
-                    identifier=identifier
-                ),
-                files=files,
-                urls=urls
-            )
-            
-            return asset, None
-            
-        except Exception as e:
-            last_exception = e
+        for attempt in range(rapidata_config.upload_max_retries):
+            try:
+                with suppress_rapidata_error_logging():
+                    self.__openapi_service.participant_api.participant_participant_id_sample_post(
+                        participant_id=self.id,
+                        model=CreateSampleModel(identifier=identifier),
+                        files=files,
+                        urls=urls,
+                    )
+
+                return asset, None
+
+            except Exception as e:
+                last_exception = e
+                if attempt < rapidata_config.upload_max_retries - 1:
+                    # Exponential backoff: wait 1s, then 2s, then 4s
+                    retry_delay = 2**attempt
+                    time.sleep(retry_delay)
+                    logger.debug("Error: %s", str(last_exception))
+                    logger.debug(
+                        "Retrying %s of %s...",
+                        attempt + 1,
+                        rapidata_config.upload_max_retries,
+                    )
 
         logger.error(f"Upload failed for {identifier}. Error: {str(last_exception)}")
         return None, asset
@@ -59,24 +74,24 @@ class BenchmarkParticipant:
         self,
         assets: list[MediaAsset],
         identifiers: list[str],
-        max_workers: int = 10,
     ) -> tuple[list[MediaAsset], list[MediaAsset]]:
         """
         Upload samples concurrently with proper error handling and progress tracking.
-        
+
         Args:
             assets: List of MediaAsset objects to upload
             identifiers: List of identifiers matching the assets
-            max_workers: Maximum number of concurrent upload workers
-            
+
         Returns:
             tuple[list[str], list[str]]: Lists of successful and failed identifiers
         """
         successful_uploads: list[MediaAsset] = []
         failed_uploads: list[MediaAsset] = []
         total_uploads = len(assets)
-        
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+
+        with ThreadPoolExecutor(
+            max_workers=rapidata_config.max_upload_workers
+        ) as executor:
             futures = [
                 executor.submit(
                     self._process_single_sample_upload,
@@ -85,8 +100,12 @@ class BenchmarkParticipant:
                 )
                 for asset, identifier in zip(assets, identifiers)
             ]
-            
-            with tqdm(total=total_uploads, desc="Uploading media", disable=RapidataOutputManager.silent_mode) as pbar:
+
+            with tqdm(
+                total=total_uploads,
+                desc="Uploading media",
+                disable=RapidataOutputManager.silent_mode,
+            ) as pbar:
                 for future in as_completed(futures):
                     try:
                         successful_id, failed_id = future.result()
@@ -96,7 +115,7 @@ class BenchmarkParticipant:
                             failed_uploads.append(failed_id)
                     except Exception as e:
                         logger.error(f"Future execution failed: {str(e)}")
-                        
+
                     pbar.update(1)
-        
+
         return successful_uploads, failed_uploads

{rapidata-2.35.2 → rapidata-2.35.3}/src/rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py

@@ -8,6 +8,7 @@ from rapidata.api_client.models.root_filter import RootFilter
 from rapidata.api_client.models.filter import Filter
 from rapidata.api_client.models.sort_criterion import SortCriterion
 
+
 class RapidataBenchmarkManager:
     """
     A manager for benchmarks.
@@ -19,16 +20,18 @@ class RapidataBenchmarkManager:
     Args:
         openapi_service: The OpenAPIService instance for API interaction.
     """
+
     def __init__(self, openapi_service: OpenAPIService):
         self.__openapi_service = openapi_service
 
-    def create_new_benchmark(self, 
-                             name: str,
-                             identifiers: list[str],
-                             prompts: Optional[list[str | None]] = None,
-                             prompt_assets: Optional[list[str | None]] = None,
-                             tags: Optional[list[list[str] | None]] = None,
-                             ) -> RapidataBenchmark:
+    def create_new_benchmark(
+        self,
+        name: str,
+        identifiers: list[str],
+        prompts: Optional[list[str | None]] = None,
+        prompt_assets: Optional[list[str | None]] = None,
+        tags: Optional[list[list[str] | None]] = None,
+    ) -> RapidataBenchmark:
         """
         Creates a new benchmark with the given name, identifiers, prompts, and media assets.
         Everything is matched up by the indexes of the lists.
@@ -41,31 +44,54 @@ class RapidataBenchmarkManager:
             prompts: The prompts that will be registered for the benchmark.
             prompt_assets: The prompt assets that will be registered for the benchmark.
             tags: The tags that will be associated with the prompts to use for filtering the leaderboard results. They will NOT be shown to the users.
+
+        Example:
+            ```python
+            name = "Example Benchmark"
+            identifiers = ["id1", "id2", "id3"]
+            prompts = ["prompt 1", "prompt 2", "prompt 3"]
+            prompt_assets = ["https://assets.rapidata.ai/prompt_1.jpg", "https://assets.rapidata.ai/prompt_2.jpg", "https://assets.rapidata.ai/prompt_3.jpg"]
+            tags = [["tag1", "tag2"], ["tag2"], ["tag2", "tag3"]]
+
+            benchmark = create_new_benchmark(name=name, identifiers=identifiers, prompts=prompts, prompt_assets=prompt_assets, tags=tags)
+            ```
         """
         if not isinstance(name, str):
             raise ValueError("Name must be a string.")
-        
-        if prompts and (not isinstance(prompts, list) or not all(isinstance(prompt, str) or prompt is None for prompt in prompts)):
+
+        if prompts and (
+            not isinstance(prompts, list)
+            or not all(isinstance(prompt, str) or prompt is None for prompt in prompts)
+        ):
             raise ValueError("Prompts must be a list of strings or None.")
-        
-        if prompt_assets and (not isinstance(prompt_assets, list) or not all(isinstance(asset, str) or asset is None for asset in prompt_assets)):
+
+        if prompt_assets and (
+            not isinstance(prompt_assets, list)
+            or not all(
+                isinstance(asset, str) or asset is None for asset in prompt_assets
+            )
+        ):
             raise ValueError("Media assets must be a list of strings or None.")
-        
-        if not isinstance(identifiers, list) or not all(isinstance(identifier, str) for identifier in identifiers):
+
+        if not isinstance(identifiers, list) or not all(
+            isinstance(identifier, str) for identifier in identifiers
+        ):
             raise ValueError("Identifiers must be a list of strings.")
-        
+
         if prompts and len(identifiers) != len(prompts):
             raise ValueError("Identifiers and prompts must have the same length.")
-        
+
         if prompt_assets and len(identifiers) != len(prompt_assets):
             raise ValueError("Identifiers and media assets must have the same length.")
-        
+
         if not prompts and not prompt_assets:
-            raise ValueError("At least one of prompts or media assets must be provided.")
-        
+            raise ValueError(
+                "At least one of prompts or media assets must be provided."
+            )
+
         if len(set(identifiers)) != len(identifiers):
             raise ValueError("Identifiers must be unique.")
-        
+
         if tags and len(identifiers) != len(tags):
             raise ValueError("Identifiers and tags must have the same length.")
 
@@ -78,32 +104,49 @@ class RapidataBenchmarkManager:
         benchmark = RapidataBenchmark(name, benchmark_result.id, self.__openapi_service)
 
         prompts_list = prompts if prompts is not None else [None] * len(identifiers)
-        media_assets_list = prompt_assets if prompt_assets is not None else [None] * len(identifiers)
+        media_assets_list = (
+            prompt_assets if prompt_assets is not None else [None] * len(identifiers)
+        )
         tags_list = tags if tags is not None else [None] * len(identifiers)
 
-        for identifier, prompt, asset, tag in zip(identifiers, prompts_list, media_assets_list, tags_list):
+        for identifier, prompt, asset, tag in zip(
+            identifiers, prompts_list, media_assets_list, tags_list
+        ):
             benchmark.add_prompt(identifier, prompt, asset, tag)
 
         return benchmark
-    
+
     def get_benchmark_by_id(self, id: str) -> RapidataBenchmark:
         """
         Returns a benchmark by its ID.
         """
-        benchmark_result = self.__openapi_service.benchmark_api.benchmark_benchmark_id_get(
-            benchmark_id=id
+        benchmark_result = (
+            self.__openapi_service.benchmark_api.benchmark_benchmark_id_get(
+                benchmark_id=id
+            )
         )
-        return RapidataBenchmark(benchmark_result.name, benchmark_result.id, self.__openapi_service)
-    
-    def find_benchmarks(self, name: str = "", amount: int = 10) -> list[RapidataBenchmark]:
+        return RapidataBenchmark(
+            benchmark_result.name, benchmark_result.id, self.__openapi_service
+        )
+
+    def find_benchmarks(
+        self, name: str = "", amount: int = 10
+    ) -> list[RapidataBenchmark]:
         """
         Returns a list of benchmarks by their name.
         """
         benchmark_result = self.__openapi_service.benchmark_api.benchmarks_get(
             QueryModel(
                 page=PageInfo(index=1, size=amount),
-                filter=RootFilter(filters=[Filter(field="Name", operator="Contains", value=name)]),
-                sortCriteria=[SortCriterion(direction="Desc", propertyName="CreatedAt")]
+                filter=RootFilter(
+                    filters=[Filter(field="Name", operator="Contains", value=name)]
+                ),
+                sortCriteria=[
+                    SortCriterion(direction="Desc", propertyName="CreatedAt")
+                ],
             )
         )
-        return [RapidataBenchmark(benchmark.name, benchmark.id, self.__openapi_service) for benchmark in benchmark_result.items]
+        return [
+            RapidataBenchmark(benchmark.name, benchmark.id, self.__openapi_service)
+            for benchmark in benchmark_result.items
+        ]

rapidata-2.35.3/src/rapidata/rapidata_client/config/__init__.py

@@ -0,0 +1 @@
+from .config import rapidata_config

rapidata-2.35.3/src/rapidata/rapidata_client/config/config.py

@@ -0,0 +1,33 @@
+class _RapidataConfig:
+    def __init__(self):
+        self.__maxUploadWorkers: int = 10
+        self.__uploadMaxRetries: int = 3
+
+    @property
+    def max_upload_workers(self) -> int:
+        return self.__maxUploadWorkers
+
+    @max_upload_workers.setter
+    def max_upload_workers(self, value: int) -> None:
+        if value < 1:
+            raise ValueError("max_upload_workers must be greater than 0")
+        self.__maxUploadWorkers = value
+
+    @property
+    def upload_max_retries(self) -> int:
+        return self.__uploadMaxRetries
+
+    @upload_max_retries.setter
+    def upload_max_retries(self, value: int) -> None:
+        if value < 1:
+            raise ValueError("upload_max_retries must be greater than 0")
+        self.__uploadMaxRetries = value
+
+    def __str__(self) -> str:
+        return f"RapidataConfig(max_upload_workers={self.__maxUploadWorkers}, upload_max_retries={self.__uploadMaxRetries})"
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
+
+rapidata_config = _RapidataConfig()

{rapidata-2.35.2 → rapidata-2.35.3}/src/rapidata/rapidata_client/datapoints/assets/_sessions.py

@@ -2,34 +2,39 @@ import requests
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
+from rapidata.rapidata_client.config.config import rapidata_config
+
+
 class SessionManager:
     _session = None
-    
+
     @classmethod
-    def get_session(cls, ) -> requests.Session:
+    def get_session(
+        cls,
+    ) -> requests.Session:
         """Get a singleton requests session with retry logic.
 
         Returns:
             requests.Session: A singleton requests session with retry logic.
         """
         if cls._session is None:
-            max_retries: int = 5 
-            max_workers: int = 10
+            max_retries: int = rapidata_config.upload_max_retries
+            max_workers: int = rapidata_config.max_upload_workers
             cls._session = requests.Session()
             retries = Retry(
                 total=max_retries,
                 backoff_factor=1,
                 status_forcelist=[500, 502, 503, 504],
                 allowed_methods=["GET"],
-                respect_retry_after_header=True
+                respect_retry_after_header=True,
             )
 
             adapter = HTTPAdapter(
                 pool_connections=max_workers * 2,
                 pool_maxsize=max_workers * 4,
-                max_retries=retries
+                max_retries=retries,
             )
-            cls._session.mount('http://', adapter)
-            cls._session.mount('https://', adapter)
+            cls._session.mount("http://", adapter)
+            cls._session.mount("https://", adapter)
 
         return cls._session

{rapidata-2.35.2 → rapidata-2.35.3}/src/rapidata/rapidata_client/order/_rapidata_dataset.py

@@ -1,25 +1,11 @@
-from itertools import zip_longest
-
-from rapidata.api_client.models.create_datapoint_from_text_sources_model import (
-    CreateDatapointFromTextSourcesModel,
-)
-from rapidata.api_client.models.dataset_dataset_id_datapoints_post_request_metadata_inner import (
-    DatasetDatasetIdDatapointsPostRequestMetadataInner,
-)
 from rapidata.rapidata_client.datapoints.datapoint import Datapoint
-from rapidata.rapidata_client.datapoints.metadata import Metadata
-from rapidata.rapidata_client.datapoints.assets import (
-    TextAsset,
-    MediaAsset,
-    MultiAsset,
-    BaseAsset,
-)
+from rapidata.rapidata_client.datapoints.assets import TextAsset, MediaAsset
 from rapidata.service import LocalFileService
 from rapidata.service.openapi_service import OpenAPIService
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from tqdm import tqdm
 
-from typing import cast, Sequence, Generator
+from typing import Generator
 from rapidata.rapidata_client.logging import (
     logger,
     managed_print,
@@ -30,6 +16,7 @@ import threading
 from rapidata.rapidata_client.api.rapidata_exception import (
     suppress_rapidata_error_logging,
 )
+from rapidata.rapidata_client.config.config import rapidata_config
 
 
 def chunk_list(lst: list, chunk_size: int) -> Generator:
@@ -52,17 +39,19 @@ class RapidataDataset:
 
         effective_asset_type = datapoints[0]._get_effective_asset_type()
 
+        logger.debug(f"Config for datapoint upload: {rapidata_config}")
+
         if issubclass(effective_asset_type, MediaAsset):
-            return self._add_media_from_paths(datapoints)
+            return self._add_media_from_paths(
+                datapoints,
+            )
         elif issubclass(effective_asset_type, TextAsset):
             return self._add_texts(datapoints)
         else:
             raise ValueError(f"Unsupported asset type: {effective_asset_type}")
 
     def _add_texts(
-        self,
-        datapoints: list[Datapoint],
-        max_workers: int = 10,
+        self, datapoints: list[Datapoint]
     ) -> tuple[list[Datapoint], list[Datapoint]]:
 
         def upload_text_datapoint(datapoint: Datapoint, index: int) -> Datapoint:
@@ -77,7 +66,9 @@ class RapidataDataset:
         failed_uploads: list[Datapoint] = []
 
         total_uploads = len(datapoints)
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        with ThreadPoolExecutor(
+            max_workers=rapidata_config.max_upload_workers
+        ) as executor:
             future_to_datapoint = {
                 executor.submit(upload_text_datapoint, datapoint, index=i): datapoint
                 for i, datapoint in enumerate(datapoints)
@@ -104,7 +95,6 @@ class RapidataDataset:
         self,
         datapoint: Datapoint,
         index: int,
-        max_retries: int = 3,
     ) -> tuple[list[Datapoint], list[Datapoint]]:
         """
         Process single upload with retry logic and error tracking.
@@ -129,7 +119,7 @@ class RapidataDataset:
         urls = datapoint.get_urls()
 
         last_exception = None
-        for attempt in range(max_retries):
+        for attempt in range(rapidata_config.upload_max_retries):
             try:
                 with suppress_rapidata_error_logging():
                     self.openapi_service.dataset_api.dataset_dataset_id_datapoints_post(
@@ -146,17 +136,21 @@ class RapidataDataset:
 
             except Exception as e:
                 last_exception = e
-                if attempt < max_retries - 1:
+                if attempt < rapidata_config.upload_max_retries - 1:
                     # Exponential backoff: wait 1s, then 2s, then 4s
                     retry_delay = 2**attempt
                     time.sleep(retry_delay)
                     logger.debug("Error: %s", str(last_exception))
-                    logger.debug("Retrying %s of %s...", attempt + 1, max_retries)
+                    logger.debug(
+                        "Retrying %s of %s...",
+                        attempt + 1,
+                        rapidata_config.upload_max_retries,
+                    )
 
         # If we get here, all retries failed
         local_failed.append(datapoint)
         tqdm.write(
-            f"Upload failed for {datapoint} after {max_retries} attempts. \nFinal error: \n{str(last_exception)}"
+            f"Upload failed for {datapoint} after {rapidata_config.upload_max_retries} attempts. \nFinal error: \n{str(last_exception)}"
         )
 
         return local_successful, local_failed
@@ -277,7 +271,6 @@ class RapidataDataset:
     def _process_uploads_in_chunks(
         self,
         datapoints: list[Datapoint],
-        max_workers: int,
         chunk_size: int,
         stop_progress_tracking: threading.Event,
         progress_tracking_error: threading.Event,
@@ -288,7 +281,6 @@ class RapidataDataset:
         Args:
             media_paths: List of assets to upload
             multi_metadata: Optional sequence of sequences of metadata
-            max_workers: Maximum number of concurrent workers
             chunk_size: Number of items to process in each batch
             stop_progress_tracking: Event to signal progress tracking to stop
             progress_tracking_error: Event to detect progress tracking errors
@@ -300,7 +292,9 @@ class RapidataDataset:
         failed_uploads: list[Datapoint] = []
 
         try:
-            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            with ThreadPoolExecutor(
+                max_workers=rapidata_config.max_upload_workers
+            ) as executor:
                 # Process uploads in chunks to avoid overwhelming the system
                 for chunk_idx, chunk in enumerate(chunk_list(datapoints, chunk_size)):
                     futures = [
@@ -395,7 +389,6 @@ class RapidataDataset:
     def _add_media_from_paths(
         self,
         datapoints: list[Datapoint],
-        max_workers: int = 10,
         chunk_size: int = 50,
         progress_poll_interval: float = 0.5,
     ) -> tuple[list[Datapoint], list[Datapoint]]:
@@ -404,10 +397,8 @@ class RapidataDataset:
 
         Args:
             datapoints: List of Datapoint objects to upload
-            max_workers: Maximum number of concurrent upload workers
             chunk_size: Number of items to process in each batch
             progress_poll_interval: Time in seconds between progress checks
-
         Returns:
             tuple[list[Datapoint], list[Datapoint]]: Lists of successful and failed datapoints
 
@@ -435,7 +426,6 @@ class RapidataDataset:
         try:
             successful_uploads, failed_uploads = self._process_uploads_in_chunks(
                 datapoints,
-                max_workers,
                 chunk_size,
                 stop_progress_tracking,
                 progress_tracking_error,