synapse-sdk 1.0.0a93__py3-none-any.whl → 1.0.0a95__py3-none-any.whl

synapse_sdk/plugins/categories/upload/actions/upload.py

@@ -192,8 +192,7 @@ class UploadParams(BaseModel):
         storage (int): The storage of the action.
         collection (int): The collection of the action.
         project (int | None): The project of the action.
-        use_excel_metadata (bool): Whether to use excel file for additional metadata.
-        excel_metadata_path (str | None): Path to excel file containing metadata.
+        excel_metadata_path (str | None): Path to excel file containing metadata. Defaults to 'meta.xlsx' or 'meta.xls' in the path directory.
     """
 
     name: Annotated[str, AfterValidator(non_blank)]
@@ -202,7 +201,6 @@ class UploadParams(BaseModel):
     storage: int
     collection: int
     project: int | None
-    use_excel_metadata: bool = False
     excel_metadata_path: str | None = None
 
     @field_validator('storage', mode='before')
@@ -251,7 +249,7 @@ class UploadParams(BaseModel):
     @field_validator('excel_metadata_path', mode='before')
     @classmethod
     def check_excel_metadata_path(cls, value: str, info) -> str:
-        """Validate excel metadata file exists and is secure if use_excel_metadata is True.
+        """Validate excel metadata file exists and is secure if provided.
 
         This validator performs comprehensive security checks including:
         - File existence and format validation
@@ -271,48 +269,45 @@ class UploadParams(BaseModel):
         if not value:
             return value
 
-        # Check if use_excel_metadata is True
-        data = info.data
-        if data.get('use_excel_metadata', False):
-            excel_path = Path(value)
+        excel_path = Path(value)
 
-            # Check file existence
-            if not excel_path.exists():
-                raise PydanticCustomError('file_not_found', _('Excel metadata file not found.'))
-
-            # Check file extension
-            if excel_path.suffix.lower() not in ['.xlsx', '.xls']:
-                raise PydanticCustomError('invalid_file_type', _('Excel metadata file must be .xlsx or .xls format.'))
-
-            # Security check: file size limit
-            file_size = excel_path.stat().st_size
-            excel_config = ExcelSecurityConfig()
-            if file_size > excel_config.MAX_FILE_SIZE_BYTES:
-                raise PydanticCustomError(
-                    'file_too_large',
-                    _('Excel metadata file is too large. Maximum size is {}MB.').format(excel_config.MAX_FILE_SIZE_MB),
-                )
+        # Check file existence
+        if not excel_path.exists():
+            raise PydanticCustomError('file_not_found', _('Excel metadata file not found.'))
 
-            # Basic security check: ensure file is readable and not corrupted
-            try:
-                with open(excel_path, 'rb') as f:
-                    # Read first few bytes to check if it's a valid Excel file
-                    header = f.read(8)
-                    if not header:
-                        raise PydanticCustomError('invalid_file', _('Excel metadata file appears to be empty.'))
-
-                    # Check for valid Excel file signatures
-                    if excel_path.suffix.lower() == '.xlsx':
-                        # XLSX files start with PK (ZIP signature)
-                        if not header.startswith(b'PK'):
-                            raise PydanticCustomError('invalid_file', _('Excel metadata file appears to be corrupted.'))
-                    elif excel_path.suffix.lower() == '.xls':
-                        # XLS files have specific OLE signatures
-                        if not (header.startswith(b'\xd0\xcf\x11\xe0') or header.startswith(b'\x09\x08')):
-                            raise PydanticCustomError('invalid_file', _('Excel metadata file appears to be corrupted.'))
-
-            except (OSError, IOError):
-                raise PydanticCustomError('file_access_error', _('Cannot access Excel metadata file.'))
+        # Check file extension
+        if excel_path.suffix.lower() not in ['.xlsx', '.xls']:
+            raise PydanticCustomError('invalid_file_type', _('Excel metadata file must be .xlsx or .xls format.'))
+
+        # Security check: file size limit
+        file_size = excel_path.stat().st_size
+        excel_config = ExcelSecurityConfig()
+        if file_size > excel_config.MAX_FILE_SIZE_BYTES:
+            raise PydanticCustomError(
+                'file_too_large',
+                _('Excel metadata file is too large. Maximum size is {}MB.').format(excel_config.MAX_FILE_SIZE_MB),
+            )
+
+        # Basic security check: ensure file is readable and not corrupted
+        try:
+            with open(excel_path, 'rb') as f:
+                # Read first few bytes to check if it's a valid Excel file
+                header = f.read(8)
+                if not header:
+                    raise PydanticCustomError('invalid_file', _('Excel metadata file appears to be empty.'))
+
+                # Check for valid Excel file signatures
+                if excel_path.suffix.lower() == '.xlsx':
+                    # XLSX files start with PK (ZIP signature)
+                    if not header.startswith(b'PK'):
+                        raise PydanticCustomError('invalid_file', _('Excel metadata file appears to be corrupted.'))
+                elif excel_path.suffix.lower() == '.xls':
+                    # XLS files have specific OLE signatures
+                    if not (header.startswith(b'\xd0\xcf\x11\xe0') or header.startswith(b'\x09\x08')):
+                        raise PydanticCustomError('invalid_file', _('Excel metadata file appears to be corrupted.'))
+
+        except (OSError, IOError):
+            raise PydanticCustomError('file_access_error', _('Cannot access Excel metadata file.'))
 
         return value
 
@@ -354,12 +349,12 @@ class UploadAction(Action):
         },
     }
     metrics_categories = {
-        'data_file': {
+        'data_files': {
             'stand_by': 0,
             'failed': 0,
             'success': 0,
         },
-        'data_unit': {
+        'data_units': {
             'stand_by': 0,
             'failed': 0,
             'success': 0,
@@ -543,6 +538,24 @@ class UploadAction(Action):
         if row_count > self.excel_config.MAX_ROWS:
             raise ExcelParsingError(f'Too many rows: {row_count} (max: {self.excel_config.MAX_ROWS})')
 
+    def _find_excel_metadata_file(self, pathlib_cwd: Path) -> Optional[Path]:
+        """Find Excel metadata file in the directory.
+
+        Checks for meta.xlsx and meta.xls in the given directory.
+
+        Args:
+            pathlib_cwd (Path): The pathlib object representing the current working directory.
+
+        Returns:
+            Optional[Path]: Path to the Excel metadata file if found, None otherwise.
+        """
+        # Check for xlsx first, then xls
+        for extension in ['.xlsx', '.xls']:
+            excel_path = pathlib_cwd / f'meta{extension}'
+            if excel_path.exists() and excel_path.is_file():
+                return excel_path
+        return None
+
     def _read_excel_metadata(self, pathlib_cwd: Path) -> Dict[str, Dict[str, Any]]:
         """Read metadata from excel file with comprehensive security validation.
 
@@ -560,13 +573,20 @@ class UploadAction(Action):
             ExcelSecurityError: If security validation fails
             ExcelParsingError: If Excel content is invalid or exceeds limits
         """
-        if not self.params.get('use_excel_metadata', False) or not self.params.get('excel_metadata_path'):
-            return {}
+        excel_path = None
 
-        excel_path = pathlib_cwd / self.params['excel_metadata_path']
-        if not excel_path.exists():
-            self.run.log_message(f'Excel metadata file not found: {excel_path}', context=Context.WARNING.value)
-            return {}
+        # Check if user provided a specific excel_metadata_path
+        if self.params.get('excel_metadata_path'):
+            excel_path = pathlib_cwd / self.params['excel_metadata_path']
+            if not excel_path.exists():
+                self.run.log_message(f'Excel metadata file not found: {excel_path}', context=Context.WARNING.value)
+                return {}
+        else:
+            # Look for default meta.xlsx or meta.xls
+            excel_path = self._find_excel_metadata_file(pathlib_cwd)
+            if not excel_path:
+                # No Excel metadata file found, return empty dict (not an error)
+                return {}
 
         try:
             # Prepare Excel file with security validation
@@ -611,28 +631,26 @@ class UploadAction(Action):
         storage = self.client.get_storage(self.params['storage'])
         pathlib_cwd = get_pathlib(storage, self.params['path'])
 
-        # Read excel metadata if configured
+        # Read excel metadata if configured or default file exists
         excel_metadata: Dict[str, Dict[str, Any]] = {}
         try:
             excel_metadata = self._read_excel_metadata(pathlib_cwd)
             if excel_metadata:
                 self.run.log_message(f'Excel metadata loaded for {len(excel_metadata)} files')
-            elif self.params.get('use_excel_metadata', False):
-                self.run.log_message('Excel metadata enabled but no entries found')
-            # Don't log anything if Excel metadata is not being used
         except ExcelSecurityError as e:
             # Security violations should stop the process entirely
             self.run.log_message(f'Excel security validation failed: {str(e)}', context=Context.ERROR.value)
             self.run.log_message('Upload aborted due to Excel security concerns.', context=Context.ERROR.value)
             return result
         except ExcelParsingError as e:
-            # Parsing errors can be non-critical if Excel metadata is optional
-            if self.params.get('use_excel_metadata', False):
+            # Parsing errors can be non-critical if user didn't explicitly provide Excel file
+            if self.params.get('excel_metadata_path'):
+                # User explicitly provided Excel file, treat as error
                 self.run.log_message(f'Excel parsing failed: {str(e)}', context=Context.ERROR.value)
                 self.run.log_message('Upload aborted due to Excel parsing failure.', context=Context.ERROR.value)
                 return result
             else:
-                # If Excel metadata is not explicitly enabled, treat as warning and continue
+                # Default Excel file found but failed to parse, treat as warning and continue
                 self.run.log_message(f'Excel parsing failed (continuing): {str(e)}', context=Context.WARNING.value)
                 excel_metadata = {}
 
@@ -693,7 +711,6 @@ class UploadAction(Action):
 
         collection = self.run.client.get_data_collection(collection_id)
         self.run.set_progress(2, 2, category='analyze_collection')
-        self.run.log_message('Collection analysis completed.')
 
         return collection['file_specifications']
 
@@ -716,7 +733,7 @@ class UploadAction(Action):
         failed_count = 0
 
         # Initialize metrics
-        self._update_metrics(organized_files_count, success_count, failed_count, 'data_file')
+        self._update_metrics(organized_files_count, success_count, failed_count, 'data_files')
 
         for organized_file in organized_files:
             try:
@@ -730,12 +747,11 @@ class UploadAction(Action):
                 failed_count += 1
 
             current_progress += 1
-            self._update_metrics(organized_files_count, success_count, failed_count, 'data_file')
+            self._update_metrics(organized_files_count, success_count, failed_count, 'data_files')
             self.run.set_progress(current_progress, organized_files_count, category='upload_data_files')
 
         # Finish progress
         self.run.set_progress(organized_files_count, organized_files_count, category='upload_data_files')
-        self.run.log_message(f'Upload data files completed. Success: {success_count}, Failed: {failed_count}')
 
         return upload_result
 
@@ -762,7 +778,7 @@ class UploadAction(Action):
         batches_count = len(batches)
 
         # Initialize metrics
-        self._update_metrics(upload_result_count, success_count, failed_count, 'data_unit')
+        self._update_metrics(upload_result_count, success_count, failed_count, 'data_units')
 
         for batch in batches:
             try:
@@ -780,12 +796,11 @@ class UploadAction(Action):
                     self.run.log_data_unit(None, UploadStatus.FAILED, data_unit_meta=None)
 
             current_progress += 1
-            self._update_metrics(upload_result_count, success_count, failed_count, 'data_unit')
+            self._update_metrics(upload_result_count, success_count, failed_count, 'data_units')
             self.run.set_progress(current_progress, batches_count, category='generate_data_units')
 
         # Finish progress
         self.run.set_progress(upload_result_count, upload_result_count, category='generate_data_units')
-        self.run.log_message(f'Data units generation completed. Success: {success_count}, Failed: {failed_count}')
 
         return sum(generated_data_units, [])
 
@@ -899,7 +914,7 @@ class UploadAction(Action):
             else:
                 missing = [req for req in required_specs if req not in files_dict]
                 self.run.log_message(
-                    f'Dataset ID {file_name} missing required files: {", ".join(missing)}',
+                    f'{file_name} missing required files: {", ".join(missing)}',
                     context=Context.WARNING.value,
                 )
 

{synapse_sdk-1.0.0a93.dist-info → synapse_sdk-1.0.0a95.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a93
+Version: 1.0.0a95
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a93.dist-info → synapse_sdk-1.0.0a95.dist-info}/RECORD

@@ -165,7 +165,7 @@ synapse_sdk/plugins/categories/smart_tool/templates/plugin/__init__.py,sha256=47
 synapse_sdk/plugins/categories/smart_tool/templates/plugin/auto_label.py,sha256=eevNg0nOcYFR4z_L_R-sCvVOYoLWSAH1jwDkAf3YCjY,320
 synapse_sdk/plugins/categories/upload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/upload/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/upload/actions/upload.py,sha256=Nhlq1aIvnhybxKhm7KlJoTGP3NO-JsE2Ya7Dgi7HVOg,37427
+synapse_sdk/plugins/categories/upload/actions/upload.py,sha256=W2GYvzzV1ZPljUezwZiFYlihq1muIJVdinnsNW7Nyw8,37673
 synapse_sdk/plugins/categories/upload/templates/config.yaml,sha256=6_dRa0_J2aS8NSUfO4MKbPxZcdPS2FpJzzp51edYAZc,281
 synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/upload/templates/plugin/upload.py,sha256=IZU4sdSMSLKPCtlNqF7DP2howTdYR6hr74HCUZsGdPk,1559
@@ -221,9 +221,9 @@ synapse_sdk/utils/storage/providers/gcp.py,sha256=i2BQCu1Kej1If9SuNr2_lEyTcr5M_n
 synapse_sdk/utils/storage/providers/http.py,sha256=2DhIulND47JOnS5ZY7MZUex7Su3peAPksGo1Wwg07L4,5828
 synapse_sdk/utils/storage/providers/s3.py,sha256=ZmqekAvIgcQBdRU-QVJYv1Rlp6VHfXwtbtjTSphua94,2573
 synapse_sdk/utils/storage/providers/sftp.py,sha256=_8s9hf0JXIO21gvm-JVS00FbLsbtvly4c-ETLRax68A,1426
-synapse_sdk-1.0.0a93.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0a93.dist-info/METADATA,sha256=NQ5zn-ID4jYmBauhJUpI351Eyfos0i9pRmIqPFuOZUY,3837
-synapse_sdk-1.0.0a93.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-synapse_sdk-1.0.0a93.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-1.0.0a93.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0a93.dist-info/RECORD,,
+synapse_sdk-1.0.0a95.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0a95.dist-info/METADATA,sha256=xvnJ6ZPqDHNbb1yBCb0jmh456ZLZTJjoWjhmGGXlSgk,3837
+synapse_sdk-1.0.0a95.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+synapse_sdk-1.0.0a95.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-1.0.0a95.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0a95.dist-info/RECORD,,