rapidata 0.1.25__py3-none-any.whl → 0.2.0__py3-none-any.whl

rapidata/rapidata_client/dataset/rapidata_dataset.py

@@ -11,6 +11,8 @@ from rapidata.api_client.models.upload_text_sources_to_dataset_model import (
 from rapidata.rapidata_client.metadata.base_metadata import Metadata
 from rapidata.service import LocalFileService
 from rapidata.service.openapi_service import OpenAPIService
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
 
 
 class RapidataDataset:
@@ -32,19 +34,18 @@ class RapidataDataset:
         self,
         image_paths: list[str | list[str]],
         metadata: list[Metadata] | None = None,
+        max_workers: int = 10,
     ):
         if metadata is not None and len(metadata) != len(image_paths):
             raise ValueError(
                 "metadata must be None or have the same length as image_paths"
             )
 
-        for media_paths_rapid, meta in zip_longest(image_paths, metadata or []):
-
+        def upload_datapoint(media_paths_rapid: str | list[str], meta: Metadata | None) -> None:
             if isinstance(media_paths_rapid, list) and not all(
                 os.path.exists(media_path) for media_path in media_paths_rapid
             ):
                 raise FileNotFoundError(f"File not found: {media_paths_rapid}")
-
             elif isinstance(media_paths_rapid, str) and not os.path.exists(
                 media_paths_rapid
             ):
@@ -60,4 +61,19 @@ class RapidataDataset:
                 ),
             )
 
-            self.openapi_service.dataset_api.dataset_create_datapoint_post(model=model, files=media_paths_rapid if isinstance(media_paths_rapid, list) else [media_paths_rapid])  # type: ignore
+            self.openapi_service.dataset_api.dataset_create_datapoint_post(
+                model=model,
+                files=media_paths_rapid if isinstance(media_paths_rapid, list) else [media_paths_rapid] # type: ignore
+            )
+
+        total_uploads = len(image_paths)
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = [
+                executor.submit(upload_datapoint, media_paths, meta)
+                for media_paths, meta in zip_longest(image_paths, metadata or [])
+            ]
+
+            with tqdm(total=total_uploads, desc="Uploading datapoints") as pbar:
+                for future in as_completed(futures):
+                    future.result()  # This will raise any exceptions that occurred during execution
+                    pbar.update(1)

rapidata/rapidata_client/order/rapidata_order_builder.py

@@ -87,7 +87,7 @@ class RapidataOrderBuilder:
             ],
         )
 
-    def create(self, submit=True) -> RapidataOrder:
+    def create(self, submit=True, max_workers=10) -> RapidataOrder:
         """Actually makes the API calls to create the order based on how the order builder was configured.
 
         Args:
@@ -113,7 +113,7 @@ class RapidataOrderBuilder:
             openapi_service=self._openapi_service,
         )
 
-        order.dataset.add_media_from_paths(self._media_paths, self._metadata)
+        order.dataset.add_media_from_paths(self._media_paths, self._metadata, max_workers)
 
         if submit:
             order.submit()

{rapidata-0.1.25.dist-info → rapidata-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rapidata
-Version: 0.1.25
+Version: 0.2.0
 Summary: Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way.
 License: Apache-2.0
 Author: Rapidata AG

{rapidata-0.1.25.dist-info → rapidata-0.2.0.dist-info}/RECORD

@@ -222,7 +222,7 @@ rapidata/rapidata_client/config.py,sha256=tQLgN6k_ATOX1GzZh38At2rgBDLStV6rJ6z0vs
 rapidata/rapidata_client/country_codes/__init__.py,sha256=Y8qeG2IMjvMGvhaPydq0nhwRQHb6dQqilctlEXu0_PE,55
 rapidata/rapidata_client/country_codes/country_codes.py,sha256=Q0HMX7uHJQDeLCFPP5bq4iYi6pgcDWEcl2ONGhjgoeU,286
 rapidata/rapidata_client/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-rapidata/rapidata_client/dataset/rapidata_dataset.py,sha256=J5jlSIbdswhqXHOq8Qf9gtzVF70cImq9sioDG7S9Mxs,2457
+rapidata/rapidata_client/dataset/rapidata_dataset.py,sha256=QDsl7ZCZxuG02yfEpBSphfSDZh_qHz6m5HUCGwZflfw,3205
 rapidata/rapidata_client/dataset/rapidata_validation_set.py,sha256=YrnUzia9AXgq2z917FtztFxj4fD5EgTWXVPBzLVIujY,9374
 rapidata/rapidata_client/dataset/validation_rapid_parts.py,sha256=SIeQesEXPPOW5kclxYLNWaKllBXHm7DQKBdMU-GXnfc,2104
 rapidata/rapidata_client/dataset/validation_set_builder.py,sha256=B9D-uNCo_PO0NCUHju_7dsWtz_KcOmvFIsxUgQ67Q2M,7471
@@ -235,7 +235,7 @@ rapidata/rapidata_client/metadata/prompt_metadata.py,sha256=_FypjKWrC3iKUO_G2CVw
 rapidata/rapidata_client/metadata/transcription_metadata.py,sha256=THtDEVCON4UlcXHmXrjilaOLHys4TrktUOPGWnXaCcc,631
 rapidata/rapidata_client/order/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 rapidata/rapidata_client/order/rapidata_order.py,sha256=VRDLTPBf2k6UihF0DnWq3nfBLWExfWHzh3T-cJgFF1w,2437
-rapidata/rapidata_client/order/rapidata_order_builder.py,sha256=qGfdmO3wdQMDwlW5NTM8x5ll8YmC0UUuUkGtyzGMObU,8105
+rapidata/rapidata_client/order/rapidata_order_builder.py,sha256=Wx7lhThTd6SwJNSbXzhGCsEgMhizjsriGz2zMjbQyEI,8134
 rapidata/rapidata_client/rapidata_client.py,sha256=z3vz5_GNivnShj7kqii-eUff16rvwSy62zwi8WZqAWo,2776
 rapidata/rapidata_client/referee/__init__.py,sha256=x0AxGCsR6TlDjfqQ00lB9V7QVS9EZCJzweNEIzx42PI,207
 rapidata/rapidata_client/referee/base_referee.py,sha256=bMy7cw0a-pGNbFu6u_1_Jplu0A483Ubj4oDQzh8vu8k,493
@@ -259,7 +259,7 @@ rapidata/service/local_file_service.py,sha256=pgorvlWcx52Uh3cEG6VrdMK_t__7dacQ_5
 rapidata/service/openapi_service.py,sha256=-vrM2jEzQxr9KAerOYkVhpvMEeHwjzRwm9L_VFyzOT0,1537
 rapidata/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 rapidata/utils/image_utils.py,sha256=TldO3eJWG8IhfJjm5MfNGO0mEDm1mQTsRoA0HLU1Uxs,404
-rapidata-0.1.25.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-rapidata-0.1.25.dist-info/METADATA,sha256=bWrZHRwA8LYarK3LDoP4wLlV4vlDYqMsiuIXemQ6C-s,924
-rapidata-0.1.25.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-rapidata-0.1.25.dist-info/RECORD,,
+rapidata-0.2.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+rapidata-0.2.0.dist-info/METADATA,sha256=Oymi81UjDR9fM9f-axaTKUWOCnlExpeA1i1voXC7ruU,923
+rapidata-0.2.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+rapidata-0.2.0.dist-info/RECORD,,