synapse-sdk 1.0.0b12__py3-none-any.whl → 1.0.0b13__py3-none-any.whl

synapse_sdk/plugins/categories/export/actions/export.py

@@ -14,11 +14,20 @@ from synapse_sdk.plugins.categories.decorators import register_action
 from synapse_sdk.plugins.categories.export.enums import ExportStatus
 from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 from synapse_sdk.plugins.models import Run
+from synapse_sdk.shared.enums import Context
 from synapse_sdk.utils.pydantic.validators import non_blank
 from synapse_sdk.utils.storage import get_pathlib
 
 
 class ExportRun(Run):
+    class ExportEventLog(BaseModel):
+        """Export event log model."""
+
+        target_id: int
+        info: str | None = None
+        status: Context
+        created: str
+
     class DataFileLog(BaseModel):
         """Data file log model."""
 
@@ -35,6 +44,13 @@ class ExportRun(Run):
         failed: int
         success: int
 
+    LOG_MESSAGES = {
+        'NULL_DATA_DETECTED': {
+            'message': 'Data is null for export item',
+            'level': Context.WARNING,
+        },
+    }
+
     def log_file(
         self, log_type: str, target_id: int, data_file_info: dict, status: ExportStatus, error: str | None = None
     ):
@@ -59,6 +75,35 @@ class ExportRun(Run):
             ).model_dump(),
         )
 
+    def log_export_event(self, code: str, target_id: int, *args, level: Context | None = None):
+        """Log export event using predefined code.
+
+        Args:
+            code (str): The log message code.
+            target_id (int): The ID of the export target.
+            *args: Arguments to format the message.
+            level (Context | None): Optional context level override.
+        """
+        if code not in self.LOG_MESSAGES:
+            now = datetime.now().isoformat()
+            self.log(
+                'export_event',
+                self.ExportEventLog(
+                    target_id=target_id, info=f'Unknown log code: {code}', status=Context.DANGER, created=now
+                ).model_dump(),
+            )
+            return
+
+        log_config = self.LOG_MESSAGES[code]
+        message = log_config['message'].format(*args) if args else log_config['message']
+        log_level = level or log_config['level'] or Context.INFO
+
+        now = datetime.now().isoformat()
+        self.log(
+            'export_event',
+            self.ExportEventLog(info=message, status=log_level, target_id=target_id, created=now).model_dump(),
+        )
+
     def log_metrics(self, record: MetricsRecord, category: str):
         """Log export metrics.
 
@@ -67,7 +112,7 @@ class ExportRun(Run):
             category (str): The category of the metrics.
         """
         record = self.MetricsRecord.model_validate(record)
-        self.set_metrics(value=record.dict(), category=category)
+        self.set_metrics(value=record.model_dump(), category=category)
 
     def export_log_json_file(
         self,
@@ -185,7 +230,7 @@ class GroundTruthExportTargetHandler(ExportTargetHandler):
             yield {
                 'data': result['data'],
                 'files': result['data_unit']['files'][files_key],
-                'id': result['ground_truth'],
+                'id': result['id'],
             }
 
 

synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -1,4 +1,5 @@
 import json
+from itertools import tee
 from pathlib import Path
 
 import requests
@@ -49,12 +50,14 @@ def export(run, export_items, path_root, **params):
         origin_files_output_path = unique_export_path / 'origin_files'
         origin_files_output_path.mkdir(parents=True, exist_ok=True)
 
-    total = params['count']
+    export_items_count, export_items_process = tee(export_items)
+    total = sum(1 for _ in export_items_count)
+
     original_file_metrics_record = run.MetricsRecord(stand_by=total, success=0, failed=0)
     data_file_metrics_record = run.MetricsRecord(stand_by=total, success=0, failed=0)
     # progress init
     run.set_progress(0, total, category='dataset_conversion')
-    for no, export_item in enumerate(export_items, start=1):
+    for no, export_item in enumerate(export_items_process, start=1):
         run.set_progress(no, total, category='dataset_conversion')
         if no == 1:
             run.log_message('Converting dataset.')
@@ -170,6 +173,16 @@ def save_as_json(run, result, base_path, error_file_list):
     file_name = Path(get_original_file_name(result['files'])).stem
     json_data = result['data']
     file_info = {'file_name': f'{file_name}.json'}
+
+    if json_data is None:
+        error_msg = 'data is Null'
+        error_file_list.append([f'{file_name}.json', error_msg])
+        status = ExportStatus.FAILED
+        run.log_export_event('NULL_DATA_DETECTED', result['id'])
+        run.export_log_json_file(result['id'], file_info, status, error_msg)
+
+        return status
+
     error_msg = ''
     try:
         with (base_path / f'{file_name}.json').open('w', encoding='utf-8') as f:

synapse_sdk/plugins/categories/upload/actions/upload.py

@@ -404,6 +404,7 @@ class UploadParams(BaseModel):
         project (int | None): The project of the action.
         excel_metadata_path (str | None): Path to excel file containing metadata.
             Defaults to 'meta.xlsx' or 'meta.xls' in the path directory.
+        is_recursive (bool): Enable recursive file discovery in subdirectories. Defaults to False.
     """
 
     name: Annotated[str, AfterValidator(non_blank)]
@@ -413,6 +414,7 @@ class UploadParams(BaseModel):
     collection: int
     project: int | None
     excel_metadata_path: str | None = None
+    is_recursive: bool = False
 
     @field_validator('storage', mode='before')
     @classmethod
@@ -581,6 +583,14 @@ class UploadAction(Action):
         """Get uploader from entrypoint."""
         return self.entrypoint(self.run, path, file_specification, organized_files)
 
+    def _discover_files_recursive(self, dir_path: Path) -> List[Path]:
+        """Discover files recursively in a directory."""
+        return [file_path for file_path in dir_path.rglob('*') if file_path.is_file()]
+
+    def _discover_files_non_recursive(self, dir_path: Path) -> List[Path]:
+        """Discover files in a directory (non-recursive)."""
+        return [file_path for file_path in dir_path.glob('*') if file_path.is_file()]
+
     def _validate_excel_security(self, excel_path: Path) -> None:
         """Validate Excel file security constraints.
 
@@ -1028,15 +1038,6 @@ class UploadAction(Action):
     ) -> List[Dict[str, Any]]:
         """Organize files according to the file specification.
 
-        This method handles type-based directory structure where files are organized in
-        directories named after file types (e.g., 'image_1/' directory contains image files
-        like '1.jpg', '2.jpg'). For each dataset ID found in the primary directory, it attempts
-        to find corresponding files in all type directories.
-
-        TODO: Add Logic to handle file specific name patterns and extensions.
-        (e.g. pcd:S_DCH_230725_0156_LR_037.pcd, image_1:S_DCH_230725_0156_FC_037,
-        image_2:S_DCH_230725_0156_LF_037.jpg)
-
         Args:
             directory (Path): Root directory containing files to organize.
             file_specification (List[Dict[str, Any]]): File specification list with metadata.
@@ -1072,23 +1073,32 @@ class UploadAction(Action):
         dataset_files = {}
         required_specs = [spec['name'] for spec in file_specification if spec.get('is_required', False)]
 
+        # Get recursive setting from params
+        is_recursive = self.params.get('is_recursive', False)
+
         # Process all files from all type directories
         for spec_name, dir_path in type_dirs.items():
-            for file_path in dir_path.glob('*'):
-                if file_path.is_file():
-                    file_name = file_path.stem
-
-                    # Initialize dataset entry if it doesn't exist
-                    if file_name not in dataset_files:
-                        dataset_files[file_name] = {}
-
-                    # Map this file to its specification (handle duplicates)
-                    if spec_name not in dataset_files[file_name]:
+            # Use appropriate method based on recursive setting
+            if is_recursive:
+                files_list = self._discover_files_recursive(dir_path)
+            else:
+                files_list = self._discover_files_non_recursive(dir_path)
+
+            for file_path in files_list:
+                # Always use filename only for matching
+                file_name = file_path.stem
+
+                # Initialize dataset entry if it doesn't exist
+                if file_name not in dataset_files:
+                    dataset_files[file_name] = {}
+
+                # Map this file to its specification (handle duplicates)
+                if spec_name not in dataset_files[file_name]:
+                    dataset_files[file_name][spec_name] = file_path
+                else:
+                    existing_file = dataset_files[file_name][spec_name]
+                    if file_path.stat().st_mtime > existing_file.stat().st_mtime:
                         dataset_files[file_name][spec_name] = file_path
-                    else:
-                        existing_file = dataset_files[file_name][spec_name]
-                        if file_path.stat().st_mtime > existing_file.stat().st_mtime:
-                            dataset_files[file_name][spec_name] = file_path
 
         if not dataset_files:
             self.run.log_message_with_code('NO_FILES_FOUND_WARNING')

{synapse_sdk-1.0.0b12.dist-info → synapse_sdk-1.0.0b13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0b12
+Version: 1.0.0b13
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0b12.dist-info → synapse_sdk-1.0.0b13.dist-info}/RECORD

@@ -113,10 +113,10 @@ synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py,sh
 synapse_sdk/plugins/categories/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/export/enums.py,sha256=gtyngvQ1DKkos9iKGcbecwTVQQ6sDwbrBPSGPNb5Am0,127
 synapse_sdk/plugins/categories/export/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/export/actions/export.py,sha256=ieR9ZWOr59X2CAEjQ30PzsWmXaC93HKeFPhYqKIW2fM,11932
+synapse_sdk/plugins/categories/export/actions/export.py,sha256=5pmeexWbRpKlt4wJKGSaSzRibumenZY1u8uUncS_WXI,13477
 synapse_sdk/plugins/categories/export/templates/config.yaml,sha256=N7YmnFROb3s3M35SA9nmabyzoSb5O2t2TRPicwFNN2o,56
 synapse_sdk/plugins/categories/export/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/export/templates/plugin/export.py,sha256=GDb6Ucodsr5aBPMU4alr68-DyFoLR5TyhC_MCaJrkF0,6411
+synapse_sdk/plugins/categories/export/templates/plugin/export.py,sha256=DgoTtJF4Tq5nLUMpq0hF0XSbhXGjDb9XqoUUcrXBbnQ,6860
 synapse_sdk/plugins/categories/neural_net/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/actions/deployment.py,sha256=njhDp05KXbNB9VCa46VgWa8-ftc_f9HcAVpUr0836AM,1838
@@ -154,7 +154,7 @@ synapse_sdk/plugins/categories/smart_tool/templates/plugin/__init__.py,sha256=47
 synapse_sdk/plugins/categories/smart_tool/templates/plugin/auto_label.py,sha256=eevNg0nOcYFR4z_L_R-sCvVOYoLWSAH1jwDkAf3YCjY,320
 synapse_sdk/plugins/categories/upload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/upload/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/upload/actions/upload.py,sha256=XIKbXqzarSy6Rt41g03pkq0loAkKWBTX4yv47E25_rs,45203
+synapse_sdk/plugins/categories/upload/actions/upload.py,sha256=Ry83bR0_NKr7Ir3wj_aMkr7viype_FYJjGJU_0KLfbI,45554
 synapse_sdk/plugins/categories/upload/templates/config.yaml,sha256=6_dRa0_J2aS8NSUfO4MKbPxZcdPS2FpJzzp51edYAZc,281
 synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/upload/templates/plugin/upload.py,sha256=IZU4sdSMSLKPCtlNqF7DP2howTdYR6hr74HCUZsGdPk,1559
@@ -211,9 +211,9 @@ synapse_sdk/utils/storage/providers/gcp.py,sha256=i2BQCu1Kej1If9SuNr2_lEyTcr5M_n
 synapse_sdk/utils/storage/providers/http.py,sha256=2DhIulND47JOnS5ZY7MZUex7Su3peAPksGo1Wwg07L4,5828
 synapse_sdk/utils/storage/providers/s3.py,sha256=ZmqekAvIgcQBdRU-QVJYv1Rlp6VHfXwtbtjTSphua94,2573
 synapse_sdk/utils/storage/providers/sftp.py,sha256=_8s9hf0JXIO21gvm-JVS00FbLsbtvly4c-ETLRax68A,1426
-synapse_sdk-1.0.0b12.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0b12.dist-info/METADATA,sha256=pecWR-58KXEkIky8tZLrFknRIX3YKTDAVwYPzONzz38,3745
-synapse_sdk-1.0.0b12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-synapse_sdk-1.0.0b12.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-1.0.0b12.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0b12.dist-info/RECORD,,
+synapse_sdk-1.0.0b13.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0b13.dist-info/METADATA,sha256=a9HVEH9V67PGdiwKa5c-dPqE9y81AtvwHnk6A0O795Q,3745
+synapse_sdk-1.0.0b13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+synapse_sdk-1.0.0b13.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-1.0.0b13.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0b13.dist-info/RECORD,,