rapidata 0.1.25__tar.gz → 0.2.0__tar.gz

{rapidata-0.1.25 → rapidata-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rapidata
-Version: 0.1.25
+Version: 0.2.0
 Summary: Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way.
 License: Apache-2.0
 Author: Rapidata AG

{rapidata-0.1.25 → rapidata-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rapidata"
-version = "0.1.25"
+version = "0.2.0"
 description = "Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way."
 authors = ["Rapidata AG <info@rapidata.ai>"]
 license = "Apache-2.0"

{rapidata-0.1.25 → rapidata-0.2.0}/src/rapidata/rapidata_client/dataset/rapidata_dataset.py

@@ -11,6 +11,8 @@ from rapidata.api_client.models.upload_text_sources_to_dataset_model import (
 from rapidata.rapidata_client.metadata.base_metadata import Metadata
 from rapidata.service import LocalFileService
 from rapidata.service.openapi_service import OpenAPIService
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
 
 
 class RapidataDataset:
@@ -32,19 +34,18 @@ class RapidataDataset:
         self,
         image_paths: list[str | list[str]],
         metadata: list[Metadata] | None = None,
+        max_workers: int = 10,
     ):
         if metadata is not None and len(metadata) != len(image_paths):
             raise ValueError(
                 "metadata must be None or have the same length as image_paths"
             )
 
-        for media_paths_rapid, meta in zip_longest(image_paths, metadata or []):
-
+        def upload_datapoint(media_paths_rapid: str | list[str], meta: Metadata | None) -> None:
             if isinstance(media_paths_rapid, list) and not all(
                 os.path.exists(media_path) for media_path in media_paths_rapid
             ):
                 raise FileNotFoundError(f"File not found: {media_paths_rapid}")
-
             elif isinstance(media_paths_rapid, str) and not os.path.exists(
                 media_paths_rapid
             ):
@@ -60,4 +61,19 @@ class RapidataDataset:
                 ),
             )
 
-            self.openapi_service.dataset_api.dataset_create_datapoint_post(model=model, files=media_paths_rapid if isinstance(media_paths_rapid, list) else [media_paths_rapid])  # type: ignore
+            self.openapi_service.dataset_api.dataset_create_datapoint_post(
+                model=model,
+                files=media_paths_rapid if isinstance(media_paths_rapid, list) else [media_paths_rapid] # type: ignore
+            )
+
+        total_uploads = len(image_paths)
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = [
+                executor.submit(upload_datapoint, media_paths, meta)
+                for media_paths, meta in zip_longest(image_paths, metadata or [])
+            ]
+
+            with tqdm(total=total_uploads, desc="Uploading datapoints") as pbar:
+                for future in as_completed(futures):
+                    future.result()  # This will raise any exceptions that occurred during execution
+                    pbar.update(1)

{rapidata-0.1.25 → rapidata-0.2.0}/src/rapidata/rapidata_client/order/rapidata_order_builder.py

@@ -87,7 +87,7 @@ class RapidataOrderBuilder:
             ],
         )
 
-    def create(self, submit=True) -> RapidataOrder:
+    def create(self, submit=True, max_workers=10) -> RapidataOrder:
         """Actually makes the API calls to create the order based on how the order builder was configured.
 
         Args:
@@ -113,7 +113,7 @@ class RapidataOrderBuilder:
             openapi_service=self._openapi_service,
         )
 
-        order.dataset.add_media_from_paths(self._media_paths, self._metadata)
+        order.dataset.add_media_from_paths(self._media_paths, self._metadata, max_workers)
 
         if submit:
             order.submit()