synapse-sdk 1.0.0b15__tar.gz → 1.0.0b16__tar.gz

{synapse_sdk-1.0.0b15 → synapse_sdk-1.0.0b16}/.github/workflows/pypi-publish.yml

@@ -34,28 +34,22 @@ jobs:
       - name: Build package
         run: python -m build
 
-      - name: Install MinIO Client
-        run: |
-          wget https://dl.min.io/client/mc/release/linux-amd64/mc
-          chmod +x mc
-          sudo mv mc /usr/local/bin/
+      - name: Install AWS CLI
+        run: pip install awscli
 
       - name: Store the distribution packages to MinIO ☁️
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.ARTIFACTS_S3_ACCESS_KEY }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.ARTIFACTS_S3_SECRET_KEY }}
         run: |
-          # Configure MinIO client
-          mc alias set minio https://datamaker-kr.cdn.datamaker.io $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
-
           # Create tarball of distribution packages
           tar -czf dist.tar.gz -C dist .
 
-          # Upload to MinIO
+          # Upload to MinIO using AWS CLI
           BUCKET_NAME="${{ vars.ARTIFACTS_S3_BUCKET_NAME }}"
           OBJECT_KEY="dist.tar.gz"
-
-          mc cp dist.tar.gz minio/$BUCKET_NAME/$OBJECT_KEY
+          
+          aws s3 cp dist.tar.gz s3://$BUCKET_NAME/$OBJECT_KEY --endpoint-url=https://datamaker-kr.cdn.datamaker.io
 
           echo "Artifacts uploaded to: s3://$BUCKET_NAME/$OBJECT_KEY"
           rm -f dist.tar.gz
@@ -70,25 +64,19 @@ jobs:
       # IMPORTANT: this permission is mandatory for trusted publishing
       id-token: write
     steps:
-      - name: Install MinIO Client
-        run: |
-          wget https://dl.min.io/client/mc/release/linux-amd64/mc
-          chmod +x mc
-          sudo mv mc /usr/local/bin/
+      - name: Install AWS CLI
+        run: pip install awscli
 
       - name: Retrieve distribution packages from MinIO ⬇️
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.ARTIFACTS_S3_ACCESS_KEY }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.ARTIFACTS_S3_SECRET_KEY }}
         run: |
-          # Configure MinIO client
-          mc alias set minio https://datamaker-kr.cdn.datamaker.io $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
-
-          # Download from MinIO
+          # Download from MinIO using AWS CLI
           BUCKET_NAME="${{ vars.ARTIFACTS_S3_BUCKET_NAME }}"
           OBJECT_KEY="dist.tar.gz"
-
-          mc cp minio/$BUCKET_NAME/$OBJECT_KEY dist.tar.gz
+          
+          aws s3 cp s3://$BUCKET_NAME/$OBJECT_KEY dist.tar.gz --endpoint-url=https://datamaker-kr.cdn.datamaker.io
 
           # Extract artifacts
           mkdir -p dist/
@@ -105,13 +93,10 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.ARTIFACTS_S3_ACCESS_KEY }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.ARTIFACTS_S3_SECRET_KEY }}
         run: |
-          # Configure MinIO client
-          mc alias set minio https://datamaker-kr.cdn.datamaker.io $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
-
-          # Delete from MinIO
+          # Delete from MinIO using AWS CLI
           BUCKET_NAME="${{ vars.ARTIFACTS_S3_BUCKET_NAME }}"
           OBJECT_KEY="dist.tar.gz"
-
-          mc rm minio/$BUCKET_NAME/$OBJECT_KEY
+          
+          aws s3 rm s3://$BUCKET_NAME/$OBJECT_KEY --endpoint-url=https://datamaker-kr.cdn.datamaker.io
 
           echo "Artifacts cleaned up from: s3://$BUCKET_NAME/$OBJECT_KEY"

{synapse_sdk-1.0.0b15/synapse_sdk.egg-info → synapse_sdk-1.0.0b16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0b15
+Version: 1.0.0b16
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0b15 → synapse_sdk-1.0.0b16}/synapse_sdk/clients/backend/data_collection.py

@@ -89,7 +89,7 @@ class DataCollectionClientMixin(BaseClient):
         path = 'data_files/'
         if use_chunked_upload:
             chunked_upload = self.create_chunked_upload(file_path)
-            data = {'chunked_upload': chunked_upload['id']}
+            data = {'chunked_upload': chunked_upload['id'], 'meta': {'filename': file_path.name}}
             return self._post(path, data=data)
         else:
             return self._post(path, files={'file': file_path})
@@ -164,7 +164,7 @@ class DataCollectionClientMixin(BaseClient):
 
                 self.create_tasks(tasks_data)
 
-    def upload_data_file(self, data: Dict, data_collection_id: int) -> Dict:
+    def upload_data_file(self, data: Dict, data_collection_id: int, use_chunked_upload: bool = False) -> Dict:
         """Upload files to synapse-backend.
 
         Args:
@@ -173,12 +173,14 @@ class DataCollectionClientMixin(BaseClient):
                     - files: The files to upload. (key: file name, value: file pathlib object)
                     - meta: The meta data to upload.
             data_collection_id: The data_collection id to upload the data to.
+            use_chunked_upload: Whether to use chunked upload for large files.(default False)
+                Automatically determined based on file size threshold in upload plugin (default 50MB).
 
         Returns:
             Dict: The result of the upload.
         """
         for name, path in data['files'].items():
-            data_file = self.create_data_file(path)
+            data_file = self.create_data_file(path, use_chunked_upload)
             data['data_collection'] = data_collection_id
             data['files'][name] = {'checksum': data_file['checksum'], 'path': str(path)}
         return data

{synapse_sdk-1.0.0b15 → synapse_sdk-1.0.0b16}/synapse_sdk/plugins/categories/upload/actions/upload.py

@@ -415,6 +415,8 @@ class UploadParams(BaseModel):
     project: int | None
     excel_metadata_path: str | None = None
     is_recursive: bool = False
+    max_file_size_mb: int = 50
+    creating_data_unit_batch_size: int = 100
 
     @field_validator('storage', mode='before')
     @classmethod
@@ -901,7 +903,9 @@ class UploadAction(Action):
         if not uploaded_files:
             self.run.log_message_with_code('NO_FILES_UPLOADED')
             raise ActionError('Upload is aborted due to no uploaded files.')
-        generated_data_units = self._generate_data_units(uploaded_files)
+        generated_data_units = self._generate_data_units(
+            uploaded_files, self.params.get('creating_data_unit_batch_size')
+        )
         result['generated_data_units_count'] = len(generated_data_units)
 
         # Setup task with uploaded synapse-backend data units.
@@ -956,7 +960,9 @@ class UploadAction(Action):
 
         for organized_file in organized_files:
             try:
-                uploaded_data_file = client.upload_data_file(organized_file, collection_id)
+                # Determine if chunked upload should be used based on file size
+                use_chunked_upload = self._requires_chunked_upload(organized_file)
+                uploaded_data_file = client.upload_data_file(organized_file, collection_id, use_chunked_upload)
                 self.run.log_data_file(organized_file, UploadStatus.SUCCESS)
                 success_count += 1
                 upload_result.append(uploaded_data_file)
@@ -974,10 +980,10 @@ class UploadAction(Action):
 
         return upload_result
 
-    def _generate_data_units(self, uploaded_files: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    def _generate_data_units(self, uploaded_files: List[Dict[str, Any]], batch_size: int) -> List[Dict[str, Any]]:
         """Generate data units for the uploaded data.
 
-        TODO: make batch size configurable.
+        TODO: make dynamic batch size depend on uploaded file sizes
 
         Returns:
             Dict: The result of the generate data units process.
@@ -993,7 +999,7 @@ class UploadAction(Action):
         success_count = 0
         failed_count = 0
 
-        batches = get_batched_list(uploaded_files, 100)
+        batches = get_batched_list(uploaded_files, batch_size)
         batches_count = len(batches)
 
         # Initialize metrics
@@ -1137,6 +1143,32 @@ class UploadAction(Action):
 
         return organized_files
 
+    def _get_file_size_mb(self, file_path: Path) -> float:
+        """Get file size in MB.
+
+        Args:
+            file_path (Path): Path to the file.
+
+        Returns:
+            float: File size in MB.
+        """
+        return file_path.stat().st_size / (1024 * 1024)
+
+    def _requires_chunked_upload(self, organized_file: Dict[str, Any]) -> bool:
+        """Determine if chunked upload is required based on file size threshold.
+
+        Args:
+            organized_file (Dict[str, Any]): Organized file data with 'files' dict.
+
+        Returns:
+            bool: True if any file exceeds the threshold, False otherwise.
+        """
+        max_file_size_mb = self.params.get('max_file_size_mb', 50)
+        for file_path in organized_file.get('files', {}).values():
+            if isinstance(file_path, Path) and self._get_file_size_mb(file_path) > max_file_size_mb:
+                return True
+        return False
+
     def _cleanup_temp_directory(self, temp_path: Optional[Path] = None) -> None:
         """Clean up temporary directory.
 

{synapse_sdk-1.0.0b15 → synapse_sdk-1.0.0b16/synapse_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0b15
+Version: 1.0.0b16
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT