rapidata 2.27.4__tar.gz → 2.27.5__tar.gz

{rapidata-2.27.4 → rapidata-2.27.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rapidata
-Version: 2.27.4
+Version: 2.27.5
 Summary: Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way.
 License: Apache-2.0
 Author: Rapidata AG

{rapidata-2.27.4 → rapidata-2.27.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rapidata"
-version = "2.27.4"
+version = "2.27.5"
 description = "Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way."
 authors = ["Rapidata AG <info@rapidata.ai>"]
 license = "Apache-2.0"

{rapidata-2.27.4 → rapidata-2.27.5}/src/rapidata/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.27.4"
+__version__ = "2.27.5"
 
 from .rapidata_client import (
     RapidataClient,

{rapidata-2.27.4 → rapidata-2.27.5}/src/rapidata/rapidata_client/order/_rapidata_dataset.py

@@ -3,20 +3,19 @@ from itertools import zip_longest
 from rapidata.api_client.models.create_datapoint_from_text_sources_model import CreateDatapointFromTextSourcesModel
 from rapidata.api_client.models.dataset_dataset_id_datapoints_post_request_metadata_inner import DatasetDatasetIdDatapointsPostRequestMetadataInner
 from rapidata.rapidata_client.metadata._base_metadata import Metadata
-from rapidata.rapidata_client.assets import TextAsset, MediaAsset, MultiAsset
+from rapidata.rapidata_client.assets import TextAsset, MediaAsset, MultiAsset, BaseAsset
 from rapidata.service import LocalFileService
 from rapidata.service.openapi_service import OpenAPIService
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from tqdm import tqdm
 
 from typing import cast, Sequence, Generator
-from rapidata.rapidata_client.logging import logger, RapidataOutputManager
+from rapidata.rapidata_client.logging import logger, managed_print, RapidataOutputManager
 import time
 import threading
 
 
 def chunk_list(lst: list, chunk_size: int) -> Generator:
-    """Split list into chunks to prevent resource exhaustion"""
     for i in range(0, len(lst), chunk_size):
         yield lst[i:i + chunk_size]
 
@@ -27,6 +26,43 @@ class RapidataDataset:
         self.openapi_service = openapi_service
         self.local_file_service = LocalFileService()
 
+    def _get_effective_asset_type(self, datapoints: Sequence[BaseAsset]) -> type:
+        if not datapoints:
+            raise ValueError("Cannot determine asset type from empty datapoints list.")
+            
+        first_item = datapoints[0]
+        
+        if isinstance(first_item, MultiAsset):
+            if not first_item.assets:
+                raise ValueError("MultiAsset cannot be empty.")
+            return type(first_item.assets[0])
+        
+        return type(first_item)
+
+    def _add_datapoints(
+        self,
+        datapoints: Sequence[BaseAsset],
+        metadata_list: Sequence[Sequence[Metadata]] | None = None,
+        max_workers: int = 10,
+    ):
+        effective_asset_type = self._get_effective_asset_type(datapoints)
+        
+        for item in datapoints:
+            if isinstance(item, MultiAsset):
+                if not all(isinstance(asset, effective_asset_type) for asset in item.assets):
+                    raise ValueError("All MultiAssets must contain the same type of assets.")
+            elif not isinstance(item, (MediaAsset, TextAsset, MultiAsset)):
+                raise ValueError("All datapoints must be MediaAsset, TextAsset, or MultiAsset.")
+        
+        if issubclass(effective_asset_type, MediaAsset):
+            media_datapoints = cast(list[MediaAsset] | list[MultiAsset], datapoints)
+            self._add_media_from_paths(media_datapoints, metadata_list, max_workers)
+        elif issubclass(effective_asset_type, TextAsset):
+            text_datapoints = cast(list[TextAsset] | list[MultiAsset], datapoints)
+            self._add_texts(text_datapoints, metadata_list)
+        else:
+            raise ValueError(f"Unsupported asset type: {effective_asset_type}")
+
     def _add_texts(
         self,
         text_assets: list[TextAsset] | list[MultiAsset],
@@ -60,7 +96,7 @@ class RapidataDataset:
                 metadata=metadata,
             )
 
-            upload_response = self.openapi_service.dataset_api.dataset_dataset_id_datapoints_texts_post(dataset_id=self.dataset_id, create_datapoint_from_text_sources_model=model)
+            self.openapi_service.dataset_api.dataset_dataset_id_datapoints_texts_post(dataset_id=self.dataset_id, create_datapoint_from_text_sources_model=model)
 
         total_uploads = len(text_assets)
         with ThreadPoolExecutor(max_workers=max_workers) as executor:
@@ -79,14 +115,16 @@ class RapidataDataset:
         media_asset: MediaAsset | MultiAsset, 
         meta_list: Sequence[Metadata] | None, 
         index: int,
+        max_retries: int = 3,
     ) -> tuple[list[str], list[str]]:
         """
-        Process single upload with error tracking.
+        Process single upload with retry logic and error tracking.
         
         Args:
             media_asset: MediaAsset or MultiAsset to upload
             meta_list: Optional sequence of metadata for the asset
             index: Sort index for the upload
+            max_retries: Maximum number of retry attempts (default: 3)
             
         Returns:
             tuple[list[str], list[str]]: Lists of successful and failed identifiers
@@ -95,44 +133,56 @@ class RapidataDataset:
         local_failed: list[str] = []
         identifiers_to_track: list[str] = []
         
-        try:
-            # Get identifier for this upload (URL or file path)
-            if isinstance(media_asset, MediaAsset):
-                assets = [media_asset]
-                identifier = media_asset._url if media_asset._url else media_asset.path
-                identifiers_to_track = [identifier] if identifier else []
-            elif isinstance(media_asset, MultiAsset):
-                assets = cast(list[MediaAsset], media_asset.assets)
-                identifiers_to_track = [
-                    (asset._url if asset._url else cast(str, asset.path)) 
-                    for asset in assets
-                ]
-            else:
-                raise ValueError(f"Unsupported asset type: {type(media_asset)}")
-
-            metadata: list[DatasetDatasetIdDatapointsPostRequestMetadataInner] = []
-            if meta_list:
-                for meta in meta_list:
-                    meta_model = meta.to_model() if meta else None
-                    if meta_model:
-                        metadata.append(DatasetDatasetIdDatapointsPostRequestMetadataInner(meta_model))
-
-            local_paths = [asset.to_file() for asset in assets if asset.is_local()]
-            urls = [asset.path for asset in assets if not asset.is_local()]
+        # Get identifier for this upload (URL or file path)
+        if isinstance(media_asset, MediaAsset):
+            assets = [media_asset]
+            identifier = media_asset._url if media_asset._url else media_asset.path
+            identifiers_to_track = [identifier] if identifier else []
+        elif isinstance(media_asset, MultiAsset):
+            assets = cast(list[MediaAsset], media_asset.assets)
+            identifiers_to_track = [
+                (asset._url if asset._url else cast(str, asset.path)) 
+                for asset in assets
+            ]
+        else:
+            raise ValueError(f"Unsupported asset type: {type(media_asset)}")
 
-            self.openapi_service.dataset_api.dataset_dataset_id_datapoints_post(
-                dataset_id=self.dataset_id,
-                file=local_paths,
-                url=urls,
-                metadata=metadata,
-                sort_index=index,
-            )
+        metadata: list[DatasetDatasetIdDatapointsPostRequestMetadataInner] = []
+        if meta_list:
+            for meta in meta_list:
+                meta_model = meta.to_model() if meta else None
+                if meta_model:
+                    metadata.append(DatasetDatasetIdDatapointsPostRequestMetadataInner(meta_model))
 
-            local_successful.extend(identifiers_to_track)
+        local_paths = [asset.to_file() for asset in assets if asset.is_local()]
+        urls = [asset.path for asset in assets if not asset.is_local()]
 
-        except Exception as e:
-            logger.error(f"\nUpload failed for {identifiers_to_track}: {str(e)}") # \n to avoid same line as tqdm
-            local_failed.extend(identifiers_to_track)
+        last_exception = None
+        for attempt in range(max_retries):
+            try:
+                self.openapi_service.dataset_api.dataset_dataset_id_datapoints_post(
+                    dataset_id=self.dataset_id,
+                    file=local_paths,
+                    url=urls,
+                    metadata=metadata,
+                    sort_index=index,
+                )
+                
+                # If we get here, the upload was successful
+                local_successful.extend(identifiers_to_track)
+                return local_successful, local_failed
+                
+            except Exception as e:
+                last_exception = e
+                if attempt < max_retries - 1:
+                    # Exponential backoff: wait 1s, then 2s, then 4s
+                    retry_delay = 2 ** attempt
+                    time.sleep(retry_delay)
+                    managed_print(f"\nRetrying {attempt + 1} of {max_retries}...\n")
+                    
+        # If we get here, all retries failed
+        logger.error(f"\nUpload failed for {identifiers_to_track} after {max_retries} attempts. Final error: {str(last_exception)}")
+        local_failed.extend(identifiers_to_track)
 
         return local_successful, local_failed
 
@@ -341,7 +391,7 @@ class RapidataDataset:
         self,
         media_paths: list[MediaAsset] | list[MultiAsset],
         multi_metadata: Sequence[Sequence[Metadata]] | None = None,
-        max_workers: int = 5,
+        max_workers: int = 10,
         chunk_size: int = 50,
         progress_poll_interval: float = 0.5,
     ) -> tuple[list[str], list[str]]:

{rapidata-2.27.4 → rapidata-2.27.5}/src/rapidata/rapidata_client/order/_rapidata_order_builder.py

@@ -149,50 +149,8 @@ class RapidataOrderBuilder:
         logger.debug(f"Order created: {order}")
         logger.debug("Adding media to the order.")
 
-        if all(isinstance(item, MediaAsset) for item in self.__assets) and self.__dataset:
-            assets = cast(list[MediaAsset], self.__assets)
-            self.__dataset._add_media_from_paths(assets, self.__multi_metadata, max_upload_workers)
-
-        elif (
-            all(isinstance(item, TextAsset) for item in self.__assets) and self.__dataset
-        ):
-            assets = cast(list[TextAsset], self.__assets)
-            self.__dataset._add_texts(assets, self.__multi_metadata)
-
-        elif (
-            all(isinstance(item, MultiAsset) for item in self.__assets) and self.__dataset
-        ):
-            multi_assets = cast(list[MultiAsset], self.__assets)
-
-            # Check if all MultiAssets contain the same type of assets
-            first_asset_type = type(multi_assets[0].assets[0])
-            if not all(
-                isinstance(asset, first_asset_type)
-                for multi_asset in multi_assets
-                for asset in multi_asset.assets
-            ):
-                raise ValueError(
-                    "All MultiAssets must contain the same type of assets (either all MediaAssets or all TextAssets)."
-                )
-
-            # Process based on the asset type
-            if issubclass(first_asset_type, MediaAsset):
-                self.__dataset._add_media_from_paths(
-                    multi_assets, self.__multi_metadata, max_upload_workers
-                )
-
-            elif issubclass(first_asset_type, TextAsset):
-                self.__dataset._add_texts(multi_assets, self.__multi_metadata)
-
-            else:
-                raise ValueError(
-                    "MultiAsset must contain MediaAssets or TextAssets objects."
-                )
-
-        elif self.__dataset:
-            raise ValueError(
-                "Media paths must all be of the same type: MediaAsset, TextAsset, or MultiAsset."
-            )
+        if self.__dataset:
+            self.__dataset._add_datapoints(self.__assets, self.__multi_metadata, max_upload_workers)
         
         logger.debug("Media added to the order.")
         logger.debug("Setting order to preview")