rapidata 2.27.4__py3-none-any.whl → 2.27.5__py3-none-any.whl

rapidata/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.27.4"
+__version__ = "2.27.5"
 
 from .rapidata_client import (
     RapidataClient,

rapidata/rapidata_client/order/_rapidata_dataset.py

@@ -3,20 +3,19 @@ from itertools import zip_longest
 from rapidata.api_client.models.create_datapoint_from_text_sources_model import CreateDatapointFromTextSourcesModel
 from rapidata.api_client.models.dataset_dataset_id_datapoints_post_request_metadata_inner import DatasetDatasetIdDatapointsPostRequestMetadataInner
 from rapidata.rapidata_client.metadata._base_metadata import Metadata
-from rapidata.rapidata_client.assets import TextAsset, MediaAsset, MultiAsset
+from rapidata.rapidata_client.assets import TextAsset, MediaAsset, MultiAsset, BaseAsset
 from rapidata.service import LocalFileService
 from rapidata.service.openapi_service import OpenAPIService
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from tqdm import tqdm
 
 from typing import cast, Sequence, Generator
-from rapidata.rapidata_client.logging import logger, RapidataOutputManager
+from rapidata.rapidata_client.logging import logger, managed_print, RapidataOutputManager
 import time
 import threading
 
 
 def chunk_list(lst: list, chunk_size: int) -> Generator:
-    """Split list into chunks to prevent resource exhaustion"""
     for i in range(0, len(lst), chunk_size):
         yield lst[i:i + chunk_size]
 
@@ -27,6 +26,43 @@ class RapidataDataset:
         self.openapi_service = openapi_service
         self.local_file_service = LocalFileService()
 
+    def _get_effective_asset_type(self, datapoints: Sequence[BaseAsset]) -> type:
+        if not datapoints:
+            raise ValueError("Cannot determine asset type from empty datapoints list.")
+            
+        first_item = datapoints[0]
+        
+        if isinstance(first_item, MultiAsset):
+            if not first_item.assets:
+                raise ValueError("MultiAsset cannot be empty.")
+            return type(first_item.assets[0])
+        
+        return type(first_item)
+
+    def _add_datapoints(
+        self,
+        datapoints: Sequence[BaseAsset],
+        metadata_list: Sequence[Sequence[Metadata]] | None = None,
+        max_workers: int = 10,
+    ):
+        effective_asset_type = self._get_effective_asset_type(datapoints)
+        
+        for item in datapoints:
+            if isinstance(item, MultiAsset):
+                if not all(isinstance(asset, effective_asset_type) for asset in item.assets):
+                    raise ValueError("All MultiAssets must contain the same type of assets.")
+            elif not isinstance(item, (MediaAsset, TextAsset, MultiAsset)):
+                raise ValueError("All datapoints must be MediaAsset, TextAsset, or MultiAsset.")
+        
+        if issubclass(effective_asset_type, MediaAsset):
+            media_datapoints = cast(list[MediaAsset] | list[MultiAsset], datapoints)
+            self._add_media_from_paths(media_datapoints, metadata_list, max_workers)
+        elif issubclass(effective_asset_type, TextAsset):
+            text_datapoints = cast(list[TextAsset] | list[MultiAsset], datapoints)
+            self._add_texts(text_datapoints, metadata_list)
+        else:
+            raise ValueError(f"Unsupported asset type: {effective_asset_type}")
+
     def _add_texts(
         self,
         text_assets: list[TextAsset] | list[MultiAsset],
@@ -60,7 +96,7 @@ class RapidataDataset:
                 metadata=metadata,
             )
 
-            upload_response = self.openapi_service.dataset_api.dataset_dataset_id_datapoints_texts_post(dataset_id=self.dataset_id, create_datapoint_from_text_sources_model=model)
+            self.openapi_service.dataset_api.dataset_dataset_id_datapoints_texts_post(dataset_id=self.dataset_id, create_datapoint_from_text_sources_model=model)
 
         total_uploads = len(text_assets)
         with ThreadPoolExecutor(max_workers=max_workers) as executor:
@@ -79,14 +115,16 @@ class RapidataDataset:
         media_asset: MediaAsset | MultiAsset, 
         meta_list: Sequence[Metadata] | None, 
         index: int,
+        max_retries: int = 3,
     ) -> tuple[list[str], list[str]]:
         """
-        Process single upload with error tracking.
+        Process single upload with retry logic and error tracking.
         
         Args:
             media_asset: MediaAsset or MultiAsset to upload
             meta_list: Optional sequence of metadata for the asset
             index: Sort index for the upload
+            max_retries: Maximum number of retry attempts (default: 3)
             
         Returns:
             tuple[list[str], list[str]]: Lists of successful and failed identifiers
@@ -95,44 +133,56 @@ class RapidataDataset:
         local_failed: list[str] = []
         identifiers_to_track: list[str] = []
         
-        try:
-            # Get identifier for this upload (URL or file path)
-            if isinstance(media_asset, MediaAsset):
-                assets = [media_asset]
-                identifier = media_asset._url if media_asset._url else media_asset.path
-                identifiers_to_track = [identifier] if identifier else []
-            elif isinstance(media_asset, MultiAsset):
-                assets = cast(list[MediaAsset], media_asset.assets)
-                identifiers_to_track = [
-                    (asset._url if asset._url else cast(str, asset.path)) 
-                    for asset in assets
-                ]
-            else:
-                raise ValueError(f"Unsupported asset type: {type(media_asset)}")
-
-            metadata: list[DatasetDatasetIdDatapointsPostRequestMetadataInner] = []
-            if meta_list:
-                for meta in meta_list:
-                    meta_model = meta.to_model() if meta else None
-                    if meta_model:
-                        metadata.append(DatasetDatasetIdDatapointsPostRequestMetadataInner(meta_model))
-
-            local_paths = [asset.to_file() for asset in assets if asset.is_local()]
-            urls = [asset.path for asset in assets if not asset.is_local()]
+        # Get identifier for this upload (URL or file path)
+        if isinstance(media_asset, MediaAsset):
+            assets = [media_asset]
+            identifier = media_asset._url if media_asset._url else media_asset.path
+            identifiers_to_track = [identifier] if identifier else []
+        elif isinstance(media_asset, MultiAsset):
+            assets = cast(list[MediaAsset], media_asset.assets)
+            identifiers_to_track = [
+                (asset._url if asset._url else cast(str, asset.path)) 
+                for asset in assets
+            ]
+        else:
+            raise ValueError(f"Unsupported asset type: {type(media_asset)}")
 
-            self.openapi_service.dataset_api.dataset_dataset_id_datapoints_post(
-                dataset_id=self.dataset_id,
-                file=local_paths,
-                url=urls,
-                metadata=metadata,
-                sort_index=index,
-            )
+        metadata: list[DatasetDatasetIdDatapointsPostRequestMetadataInner] = []
+        if meta_list:
+            for meta in meta_list:
+                meta_model = meta.to_model() if meta else None
+                if meta_model:
+                    metadata.append(DatasetDatasetIdDatapointsPostRequestMetadataInner(meta_model))
 
-            local_successful.extend(identifiers_to_track)
+        local_paths = [asset.to_file() for asset in assets if asset.is_local()]
+        urls = [asset.path for asset in assets if not asset.is_local()]
 
-        except Exception as e:
-            logger.error(f"\nUpload failed for {identifiers_to_track}: {str(e)}") # \n to avoid same line as tqdm
-            local_failed.extend(identifiers_to_track)
+        last_exception = None
+        for attempt in range(max_retries):
+            try:
+                self.openapi_service.dataset_api.dataset_dataset_id_datapoints_post(
+                    dataset_id=self.dataset_id,
+                    file=local_paths,
+                    url=urls,
+                    metadata=metadata,
+                    sort_index=index,
+                )
+                
+                # If we get here, the upload was successful
+                local_successful.extend(identifiers_to_track)
+                return local_successful, local_failed
+                
+            except Exception as e:
+                last_exception = e
+                if attempt < max_retries - 1:
+                    # Exponential backoff: wait 1s, then 2s, then 4s
+                    retry_delay = 2 ** attempt
+                    time.sleep(retry_delay)
+                    managed_print(f"\nRetrying {attempt + 1} of {max_retries}...\n")
+                    
+        # If we get here, all retries failed
+        logger.error(f"\nUpload failed for {identifiers_to_track} after {max_retries} attempts. Final error: {str(last_exception)}")
+        local_failed.extend(identifiers_to_track)
 
         return local_successful, local_failed
 
@@ -341,7 +391,7 @@ class RapidataDataset:
         self,
         media_paths: list[MediaAsset] | list[MultiAsset],
         multi_metadata: Sequence[Sequence[Metadata]] | None = None,
-        max_workers: int = 5,
+        max_workers: int = 10,
         chunk_size: int = 50,
         progress_poll_interval: float = 0.5,
     ) -> tuple[list[str], list[str]]:

rapidata/rapidata_client/order/_rapidata_order_builder.py

@@ -149,50 +149,8 @@ class RapidataOrderBuilder:
         logger.debug(f"Order created: {order}")
         logger.debug("Adding media to the order.")
 
-        if all(isinstance(item, MediaAsset) for item in self.__assets) and self.__dataset:
-            assets = cast(list[MediaAsset], self.__assets)
-            self.__dataset._add_media_from_paths(assets, self.__multi_metadata, max_upload_workers)
-
-        elif (
-            all(isinstance(item, TextAsset) for item in self.__assets) and self.__dataset
-        ):
-            assets = cast(list[TextAsset], self.__assets)
-            self.__dataset._add_texts(assets, self.__multi_metadata)
-
-        elif (
-            all(isinstance(item, MultiAsset) for item in self.__assets) and self.__dataset
-        ):
-            multi_assets = cast(list[MultiAsset], self.__assets)
-
-            # Check if all MultiAssets contain the same type of assets
-            first_asset_type = type(multi_assets[0].assets[0])
-            if not all(
-                isinstance(asset, first_asset_type)
-                for multi_asset in multi_assets
-                for asset in multi_asset.assets
-            ):
-                raise ValueError(
-                    "All MultiAssets must contain the same type of assets (either all MediaAssets or all TextAssets)."
-                )
-
-            # Process based on the asset type
-            if issubclass(first_asset_type, MediaAsset):
-                self.__dataset._add_media_from_paths(
-                    multi_assets, self.__multi_metadata, max_upload_workers
-                )
-
-            elif issubclass(first_asset_type, TextAsset):
-                self.__dataset._add_texts(multi_assets, self.__multi_metadata)
-
-            else:
-                raise ValueError(
-                    "MultiAsset must contain MediaAssets or TextAssets objects."
-                )
-
-        elif self.__dataset:
-            raise ValueError(
-                "Media paths must all be of the same type: MediaAsset, TextAsset, or MultiAsset."
-            )
+        if self.__dataset:
+            self.__dataset._add_datapoints(self.__assets, self.__multi_metadata, max_upload_workers)
         
         logger.debug("Media added to the order.")
         logger.debug("Setting order to preview")

{rapidata-2.27.4.dist-info → rapidata-2.27.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rapidata
-Version: 2.27.4
+Version: 2.27.5
 Summary: Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way.
 License: Apache-2.0
 Author: Rapidata AG

{rapidata-2.27.4.dist-info → rapidata-2.27.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rapidata/__init__.py,sha256=uidfaMpj0lDuv2SvZct-SvOTbMCiPvsW_0Tpb5nbl2A,812
+rapidata/__init__.py,sha256=tzVc51y8cqb39UGrgV8v880SavSz0vU49N4Fgcl5NU0,812
 rapidata/api_client/__init__.py,sha256=W-O8FZAfCEJR4U25Uw43g6PxmJTCAhB6v7SKcXwNuGA,28865
 rapidata/api_client/api/__init__.py,sha256=Dv6v1tCJS4BLVM5BN9k5iRMNMyglhqZ4n8vyoqkLZZw,1292
 rapidata/api_client/api/campaign_api.py,sha256=ZEYXEp8_mzsElbklLXBLGnKEfPB1mx8-G5CXfSnibq0,80791
@@ -489,8 +489,8 @@ rapidata/rapidata_client/metadata/_prompt_metadata.py,sha256=ecycAq_t2HCEptxgNxy
 rapidata/rapidata_client/metadata/_public_text_metadata.py,sha256=uXavDp1ucy_9u5n0girqWD_SkFr7tplGMK_2aqyyHIA,529
 rapidata/rapidata_client/metadata/_select_words_metadata.py,sha256=-MK5yQDi_G3BKEes6aaVyCcobB-sEy29b6bfo5f4pic,594
 rapidata/rapidata_client/order/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-rapidata/rapidata_client/order/_rapidata_dataset.py,sha256=JVGsiBa_BL0E7UZOHZ4uABr1jDVpbTiAQFO9pF2isVA,18891
-rapidata/rapidata_client/order/_rapidata_order_builder.py,sha256=x-2lpW6Jwlq-9XRz91beWNv2TgdEA-q4b_RhOSr7vhQ,14405
+rapidata/rapidata_client/order/_rapidata_dataset.py,sha256=v5b86EDF0ITIOV4k4QU8gQ4eFPz2ow-4HV_mmC9tb4c,21264
+rapidata/rapidata_client/order/_rapidata_order_builder.py,sha256=ioNGmWQF4KMdzvm-GIfAeflK8AgKaczZ1FfKkrZ1xXY,12649
 rapidata/rapidata_client/order/rapidata_order.py,sha256=uk2p6Hx2KTN4Oq2S35esdip7yLR44y-kkamS-5TBPFE,12752
 rapidata/rapidata_client/order/rapidata_order_manager.py,sha256=K9Nc66UmBfFQjSe78SMdZTOa4z3j5TRyvBnEX0Cs0u4,38306
 rapidata/rapidata_client/order/rapidata_results.py,sha256=UllYpuqpm2inKdRNhClaUwApuxsMLrvrGDsrHA5KqbY,8111
@@ -544,7 +544,7 @@ rapidata/service/__init__.py,sha256=s9bS1AJZaWIhLtJX_ZA40_CK39rAAkwdAmymTMbeWl4,
 rapidata/service/credential_manager.py,sha256=pUEEtp6VrFWYhfUUtyqmS0AlRqe2Y0kFkY6o22IT4KM,8682
 rapidata/service/local_file_service.py,sha256=pgorvlWcx52Uh3cEG6VrdMK_t__7dacQ_5AnfY14BW8,877
 rapidata/service/openapi_service.py,sha256=J07TB4P3cz9KCU7k_fwuMQwGXlq_nJx_m1_xHbZoCg0,4867
-rapidata-2.27.4.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-rapidata-2.27.4.dist-info/METADATA,sha256=7IhQtevjPvtNarZYThRc4704zkbIazsscmR5XmRUfg0,1264
-rapidata-2.27.4.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-rapidata-2.27.4.dist-info/RECORD,,
+rapidata-2.27.5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+rapidata-2.27.5.dist-info/METADATA,sha256=-bVQ6j4mkAw7lgIa9u2WCRiKHIuedG1J84ekG5LEcyE,1264
+rapidata-2.27.5.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+rapidata-2.27.5.dist-info/RECORD,,