synapse-sdk 1.0.0b14__py3-none-any.whl → 1.0.0b16__py3-none-any.whl

synapse_sdk/clients/backend/data_collection.py

@@ -89,7 +89,7 @@ class DataCollectionClientMixin(BaseClient):
         path = 'data_files/'
         if use_chunked_upload:
             chunked_upload = self.create_chunked_upload(file_path)
-            data = {'chunked_upload': chunked_upload['id']}
+            data = {'chunked_upload': chunked_upload['id'], 'meta': {'filename': file_path.name}}
             return self._post(path, data=data)
         else:
             return self._post(path, files={'file': file_path})
@@ -164,7 +164,7 @@ class DataCollectionClientMixin(BaseClient):
 
                 self.create_tasks(tasks_data)
 
-    def upload_data_file(self, data: Dict, data_collection_id: int) -> Dict:
+    def upload_data_file(self, data: Dict, data_collection_id: int, use_chunked_upload: bool = False) -> Dict:
         """Upload files to synapse-backend.
 
         Args:
@@ -173,12 +173,14 @@ class DataCollectionClientMixin(BaseClient):
                     - files: The files to upload. (key: file name, value: file pathlib object)
                     - meta: The meta data to upload.
             data_collection_id: The data_collection id to upload the data to.
+            use_chunked_upload: Whether to use chunked upload for large files.(default False)
+                Automatically determined based on file size threshold in upload plugin (default 50MB).
 
         Returns:
             Dict: The result of the upload.
         """
         for name, path in data['files'].items():
-            data_file = self.create_data_file(path)
+            data_file = self.create_data_file(path, use_chunked_upload)
             data['data_collection'] = data_collection_id
             data['files'][name] = {'checksum': data_file['checksum'], 'path': str(path)}
         return data

synapse_sdk/plugins/categories/upload/actions/upload.py

@@ -415,6 +415,8 @@ class UploadParams(BaseModel):
     project: int | None
     excel_metadata_path: str | None = None
     is_recursive: bool = False
+    max_file_size_mb: int = 50
+    creating_data_unit_batch_size: int = 100
 
     @field_validator('storage', mode='before')
     @classmethod
@@ -901,7 +903,9 @@ class UploadAction(Action):
         if not uploaded_files:
             self.run.log_message_with_code('NO_FILES_UPLOADED')
             raise ActionError('Upload is aborted due to no uploaded files.')
-        generated_data_units = self._generate_data_units(uploaded_files)
+        generated_data_units = self._generate_data_units(
+            uploaded_files, self.params.get('creating_data_unit_batch_size')
+        )
         result['generated_data_units_count'] = len(generated_data_units)
 
         # Setup task with uploaded synapse-backend data units.
@@ -956,7 +960,9 @@ class UploadAction(Action):
 
         for organized_file in organized_files:
             try:
-                uploaded_data_file = client.upload_data_file(organized_file, collection_id)
+                # Determine if chunked upload should be used based on file size
+                use_chunked_upload = self._requires_chunked_upload(organized_file)
+                uploaded_data_file = client.upload_data_file(organized_file, collection_id, use_chunked_upload)
                 self.run.log_data_file(organized_file, UploadStatus.SUCCESS)
                 success_count += 1
                 upload_result.append(uploaded_data_file)
@@ -974,10 +980,10 @@ class UploadAction(Action):
 
         return upload_result
 
-    def _generate_data_units(self, uploaded_files: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    def _generate_data_units(self, uploaded_files: List[Dict[str, Any]], batch_size: int) -> List[Dict[str, Any]]:
         """Generate data units for the uploaded data.
 
-        TODO: make batch size configurable.
+        TODO: make dynamic batch size depend on uploaded file sizes
 
         Returns:
             Dict: The result of the generate data units process.
@@ -993,7 +999,7 @@ class UploadAction(Action):
         success_count = 0
         failed_count = 0
 
-        batches = get_batched_list(uploaded_files, 100)
+        batches = get_batched_list(uploaded_files, batch_size)
         batches_count = len(batches)
 
         # Initialize metrics
@@ -1137,6 +1143,32 @@ class UploadAction(Action):
 
         return organized_files
 
+    def _get_file_size_mb(self, file_path: Path) -> float:
+        """Get file size in MB.
+
+        Args:
+            file_path (Path): Path to the file.
+
+        Returns:
+            float: File size in MB.
+        """
+        return file_path.stat().st_size / (1024 * 1024)
+
+    def _requires_chunked_upload(self, organized_file: Dict[str, Any]) -> bool:
+        """Determine if chunked upload is required based on file size threshold.
+
+        Args:
+            organized_file (Dict[str, Any]): Organized file data with 'files' dict.
+
+        Returns:
+            bool: True if any file exceeds the threshold, False otherwise.
+        """
+        max_file_size_mb = self.params.get('max_file_size_mb', 50)
+        for file_path in organized_file.get('files', {}).values():
+            if isinstance(file_path, Path) and self._get_file_size_mb(file_path) > max_file_size_mb:
+                return True
+        return False
+
     def _cleanup_temp_directory(self, temp_path: Optional[Path] = None) -> None:
         """Clean up temporary directory.
 

{synapse_sdk-1.0.0b14.dist-info → synapse_sdk-1.0.0b16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0b14
+Version: 1.0.0b16
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0b14.dist-info → synapse_sdk-1.0.0b16.dist-info}/RECORD

@@ -1,6 +1,6 @@
 locale/en/LC_MESSAGES/messages.mo,sha256=fLIB0Lcriky4LLP8Qz1Ayq5Mr-spHNGrvEqzV2huweM,785
 locale/en/LC_MESSAGES/messages.po,sha256=YTtF-BPxoTfyw12m16zmcDb-wTv6DF8z2D5L_9VewPQ,1223
-locale/ko/LC_MESSAGES/messages.mo,sha256=n8EFy-d0AsNVolGtc2wktuyEmxh4HNdC9kKJmdi-_qQ,387
+locale/ko/LC_MESSAGES/messages.mo,sha256=7HJEJA0wKlN14xQ5VF4FCNet54tjw6mfWYj3IaBokgw,678
 locale/ko/LC_MESSAGES/messages.po,sha256=TFii_RbURDH-Du_9ZQf3wNh-2briGk1IqY33-9GKrMU,1126
 synapse_sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/i18n.py,sha256=VXMR-Zm_1hTAg9iPk3YZNNq-T1Bhx1J2fEtRT6kyYbg,766
@@ -34,7 +34,7 @@ synapse_sdk/clients/agent/service.py,sha256=s7KuPK_DB1nr2VHrigttV1WyFonaGHNrPvU8
 synapse_sdk/clients/backend/__init__.py,sha256=9FzjQn0ljRhtdaoG3n38Mdgte7GFwIh4OtEmoqVg2_E,2098
 synapse_sdk/clients/backend/annotation.py,sha256=t9KnSdCJ7NvDRkAOKn4lm5hgoawbLCwIHMeo45QiIlQ,1249
 synapse_sdk/clients/backend/core.py,sha256=oXAQwIz2QFuheaG79vXcYQFSlDDU1hxn2oRc6F5jyOc,2197
-synapse_sdk/clients/backend/data_collection.py,sha256=oLs7t_UzCLj3obcuw8K-zBW8UTDgDvmZ0RqHVHd3aPk,7451
+synapse_sdk/clients/backend/data_collection.py,sha256=C3_vWWkzdX0nfHmeTL1KDAejPkIffL9_Yz0JN7Xz1IM,7740
 synapse_sdk/clients/backend/hitl.py,sha256=1rKczQBKOeEVS0Ynu--mctbBF-zIkSz4_aklfbFY2CU,713
 synapse_sdk/clients/backend/integration.py,sha256=IdjPkllvHJ_vASHSmxsWO3CRlZz2L4eWkMOch7bHWok,2744
 synapse_sdk/clients/backend/ml.py,sha256=ynm1UQ-gJkJ-n1wU_Hjcxdhe8VC1LliNWOUJfurOrRE,1197
@@ -220,7 +220,7 @@ synapse_sdk/plugins/categories/smart_tool/templates/plugin/__init__.py,sha256=47
 synapse_sdk/plugins/categories/smart_tool/templates/plugin/auto_label.py,sha256=eevNg0nOcYFR4z_L_R-sCvVOYoLWSAH1jwDkAf3YCjY,320
 synapse_sdk/plugins/categories/upload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/upload/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/upload/actions/upload.py,sha256=oQRMnIFU-vYt2Ahfodxlc7IFvygpbhPtiu7M9UAabQ4,45663
+synapse_sdk/plugins/categories/upload/actions/upload.py,sha256=hWL42Uu3E7rghO8Ayebxs4G_WcUNxn_pEXH5pJOPj3E,46969
 synapse_sdk/plugins/categories/upload/templates/config.yaml,sha256=6_dRa0_J2aS8NSUfO4MKbPxZcdPS2FpJzzp51edYAZc,281
 synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/upload/templates/plugin/upload.py,sha256=IZU4sdSMSLKPCtlNqF7DP2howTdYR6hr74HCUZsGdPk,1559
@@ -277,9 +277,9 @@ synapse_sdk/utils/storage/providers/gcp.py,sha256=i2BQCu1Kej1If9SuNr2_lEyTcr5M_n
 synapse_sdk/utils/storage/providers/http.py,sha256=2DhIulND47JOnS5ZY7MZUex7Su3peAPksGo1Wwg07L4,5828
 synapse_sdk/utils/storage/providers/s3.py,sha256=ZmqekAvIgcQBdRU-QVJYv1Rlp6VHfXwtbtjTSphua94,2573
 synapse_sdk/utils/storage/providers/sftp.py,sha256=_8s9hf0JXIO21gvm-JVS00FbLsbtvly4c-ETLRax68A,1426
-synapse_sdk-1.0.0b14.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0b14.dist-info/METADATA,sha256=e4ZLvnHm57yBcrvRt6lcwipZt4nHqbntsg1v17ezbrk,3745
-synapse_sdk-1.0.0b14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-synapse_sdk-1.0.0b14.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-1.0.0b14.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0b14.dist-info/RECORD,,
+synapse_sdk-1.0.0b16.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0b16.dist-info/METADATA,sha256=FsjML_c4mqFuK23T5yo6xO4IYFFzQAnoFz4G2Mg8eIg,3745
+synapse_sdk-1.0.0b16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+synapse_sdk-1.0.0b16.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-1.0.0b16.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0b16.dist-info/RECORD,,