synapse-sdk 1.0.0a54__tar.gz → 1.0.0a55__tar.gz

{synapse_sdk-1.0.0a54 → synapse_sdk-1.0.0a55}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a54
+Version: 1.0.0a55
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a54 → synapse_sdk-1.0.0a55}/synapse_sdk/clients/backend/dataset.py

@@ -10,16 +10,16 @@ from synapse_sdk.clients.utils import get_batched_list
 
 class DatasetClientMixin(BaseClient):
     def list_dataset(self):
-        path = 'datasets/'
+        path = 'data_collections/'
         return self._list(path)
 
-    def get_dataset(self, dataset_id):
+    def get_dataset(self, data_collection_id):
         """Get dataset from synapse-backend.
 
         Args:
-            dataset_id: The dataset id to get.
+            data_collection_id: The data_collection id to get.
         """
-        path = f'datasets/{dataset_id}/?expand=file_specifications'
+        path = f'data_collections/{data_collection_id}/?expand=file_specifications'
         return self._get(path)
 
     def create_data_file(self, file_path: Path):
@@ -82,7 +82,7 @@ class DatasetClientMixin(BaseClient):
 
                 self.create_tasks(tasks_data)
 
-    def upload_data_file(self, data: Dict, dataset_id: int) -> Dict:
+    def upload_data_file(self, data: Dict, data_collection_id: int) -> Dict:
         """Upload files to synapse-backend.
 
         Args:
@@ -90,13 +90,13 @@ class DatasetClientMixin(BaseClient):
                 * structure:
                     - files: The files to upload. (key: file name, value: file pathlib object)
                     - meta: The meta data to upload.
-            dataset_id: The dataset id to upload the data to.
+            data_collection_id: The dataset id to upload the data to.
 
         Returns:
             Dict: The result of the upload.
         """
         for name, path in data['files'].items():
             data_file = self.create_data_file(path)
-            data['dataset'] = dataset_id
+            data['data_collection'] = data_collection_id
             data['files'][name] = {'checksum': data_file['checksum'], 'path': str(path)}
         return data

{synapse_sdk-1.0.0a54 → synapse_sdk-1.0.0a55}/synapse_sdk/plugins/categories/upload/actions/upload.py

@@ -1,6 +1,8 @@
 import json
+import re
 from datetime import datetime
 from enum import Enum
+from pathlib import Path
 from typing import Annotated, Dict, List
 
 from pydantic import AfterValidator, BaseModel, field_validator
@@ -201,7 +203,7 @@ class UploadAction(Action):
             'generate_tasks': (5, 45, 25, 25, 0),
             'generate_ground_truths': (5, 35, 30, 15, 15),
         }
-        options = kwargs['plugin_config']['actions']['upload']['options']
+        options = self.config.get('options', {})
         progress_categories = self.progress_categories
         if options['allow_generate_tasks'] and not kwargs['params']['allow_generate_ground_truths']:
             ratio_name = 'generate_tasks'
@@ -217,9 +219,9 @@ class UploadAction(Action):
             progress_categories[category]['proportion'] = progress_ratios[ratio_name][i]
         self.progress_categories = progress_categories
 
-    def get_uploader(self, path):
+    def get_uploader(self, path, file_specification, organized_files):
         """Get uploader from entrypoint."""
-        return self.entrypoint(self.run, path)
+        return self.entrypoint(self.run, path, file_specification, organized_files)
 
     def start(self) -> Dict:
         """Start upload process.
@@ -231,19 +233,23 @@ class UploadAction(Action):
         storage = self.client.get_storage(self.params['storage'])
         pathlib_cwd = get_pathlib(storage, self.params['path'])
 
-        # Initialize uploader.
-        uploader = self.get_uploader(pathlib_cwd)
-
         # Analyze Collection file specifications to determine the data structure for upload.
         file_specification_template = self._analyze_collection()
+        organized_files = self._organize_files(pathlib_cwd, file_specification_template)
+
+        # Initialize uploader.
+        uploader = self.get_uploader(pathlib_cwd, file_specification_template, organized_files)
 
         # Setup result dict.
         result = {}
 
-        # Organize data according to Collection file specification structure.
+        # Get organized files from the uploader (plugin developer's custom implementation)
+        # or use the default organization method if uploader doesn't provide valid files
         organized_files = uploader.handle_upload_files()
-        if not self._validate_organized_files(file_specification_template, organized_files):
-            self.run.log_message('Validate organized files failed.')
+
+        # Validate the organized files
+        if not self._validate_organized_files(organized_files, file_specification_template):
+            self.run.log_message('Validation failed.', context=Context.ERROR.value)
             self.run.end_log()
             return result
 
@@ -307,11 +313,6 @@ class UploadAction(Action):
 
         return collection['file_specifications']
 
-    def _validate_organized_files(self, file_specification_template: Dict, organized_files: List) -> bool:
-        """Validate organized files from Uploader."""
-        validator = FileSpecificationValidator(file_specification_template, organized_files)
-        return validator.validate()
-
     def _upload_files(self, organized_files, organized_files_count: int) -> List:
         """Upload files to synapse-backend.
 
@@ -419,3 +420,174 @@ class UploadAction(Action):
         # Finish progress
         self.run.log_message('Generating ground truths completed')
         self.run.set_progress(1, 1, category='generate_ground_truths')
+
+    def _validate_organized_files(self, organized_files: List, file_specification_template: Dict) -> bool:
+        """Validate organized files from Uploader."""
+        validator = FileSpecificationValidator(file_specification_template, organized_files)
+        return validator.validate()
+
+    def _organize_files(self, directory: Path, file_specification: List) -> List:
+        """Organize files according to the file specification.
+        This method handles type-based directory structure where files are organized in
+        directories named after file types (e.g., 'image_1/' directory contains image files
+        like '1.jpg', '2.jpg'). For each dataset ID found in the primary directory, it attempts
+        to find corresponding files in all type directories.
+
+        TODO : Add Logic to handle file specific name patterns and extensions.
+        (e.g. pcd:S_DCH_230725_0156_LR_037.pcd, image_1:S_DCH_230725_0156_FC_037, image_2:S_DCH_230725_0156_LF_037.jpg)
+        Args:
+            directory (Path): Root directory containing files to organize.
+            file_specification (List): File specification list.
+        Returns:
+            List: List of dictionaries containing organized files.
+        """
+        organized_files = []
+        self.run.log_message(f'Looking for files in {directory}...')
+
+        # Check for type-based directory structure (e.g., image_1/, pcd_1/)
+        type_dirs = {}
+        type_extensions = {}  # Store common extensions for each type directory
+
+        for spec in file_specification:
+            spec_name = spec['name']
+
+            spec_dir = directory / spec_name
+            if spec_dir.exists() and spec_dir.is_dir():
+                type_dirs[spec_name] = spec_dir
+
+                # Analyze file extensions in this directory
+                extensions = {}
+                for file_path in spec_dir.glob('*'):
+                    if file_path.is_file():
+                        ext = file_path.suffix.lower()
+                        extensions[ext] = extensions.get(ext, 0) + 1
+
+                # Find the most common extension
+                if extensions:
+                    common_ext = max(extensions.items(), key=lambda x: x[1])[0]
+                    type_extensions[spec_name] = common_ext
+                    self.run.log_message(f'Found type directory: {spec_name} (common extension: {common_ext})')
+
+        # If type-based directories don't exist, exit early
+        if not type_dirs:
+            self.run.log_message('No type-based directory structure found.', context=Context.INFO.value)
+            return organized_files
+
+        self.run.log_message('Detected type-based directory structure')
+
+        # Build a comprehensive map of all dataset IDs across all type directories
+        dataset_files = {}  # Dictionary: file_name -> {spec_name -> file_path}
+
+        # First pass: collect all dataset IDs from all type directories
+        for spec_name, dir_path in type_dirs.items():
+            for file_path in dir_path.glob('*'):
+                if file_path.is_file():
+                    file_name = file_path.stem
+
+                    # Initialize dataset entry if it doesn't exist
+                    if file_name not in dataset_files:
+                        dataset_files[file_name] = {}
+
+                    # Map this file to its specification
+                    if spec_name not in dataset_files[file_name]:
+                        dataset_files[file_name][spec_name] = file_path
+                    else:
+                        # If multiple files with same file_name for same spec, use most recent
+                        existing_file = dataset_files[file_name][spec_name]
+                        if file_path.stat().st_mtime > existing_file.stat().st_mtime:
+                            dataset_files[file_name][spec_name] = file_path
+                            self.run.log_message(
+                                f"Found newer file for name of {file_name}, spec '{spec_name}': "
+                                f'{file_path.name} (replacing {existing_file.name})'
+                            )
+
+        if not dataset_files:
+            self.run.log_message('No dataset files found.', context=Context.WARNING.value)
+            return organized_files
+
+        self.run.log_message(f'Found {len(dataset_files)} potential datasets by ID')
+
+        # Second pass: organize valid datasets
+        for file_name, files_dict in sorted(dataset_files.items()):
+            self.run.log_message(f'Processing file name: {file_name}')
+
+            # Add file spec details for logging
+            for spec_name, file_path in files_dict.items():
+                self.run.log_message(f"Mapped '{spec_name}' to: {file_path.name}")
+
+            # Check if all required files are present
+            required_specs = [spec['name'] for spec in file_specification if spec.get('is_required', False)]
+            if all(req in files_dict for req in required_specs):
+                # Create metadata for this dataset
+                meta_data = {
+                    'origin_file_stem': file_name,
+                    'created_at': datetime.now().isoformat(),
+                }
+
+                # Add the organized dataset
+                organized_files.append({'files': files_dict, 'meta': meta_data})
+                self.run.log_message(f'Successfully organized dataset for ID {file_name}')
+            else:
+                # Missing required files warning
+                missing = [req for req in required_specs if req not in files_dict]
+                self.run.log_message(
+                    f'Dataset ID {file_name} is missing required files: {", ".join(missing)}',
+                    context=Context.WARNING.value,
+                )
+
+        self.run.log_message(f'Total datasets organized: {len(organized_files)}')
+        return organized_files
+
+    def _map_files_to_specification(self, directory: Path, file_specification: List) -> Dict[str, Path]:
+        """Map files in a directory to the file specification.
+
+        Args:
+            directory (Path): Directory containing files to map.
+            file_specification (List): File specification list.
+
+        Returns:
+            Dict[str, Path]: Dictionary mapping file specification names to file paths.
+        """
+        files_dict = {}
+
+        # Get all files in the directory once
+        all_files = [f for f in directory.iterdir() if f.is_file()]
+
+        # Process each file specification
+        for file_spec in file_specification:
+            file_name = file_spec['name']
+            is_required = file_spec.get('is_required', False)
+
+            # Generate name pattern based on the specification
+            name_parts = re.split(r'_(\d+)$', file_name)
+
+            # Find files matching the pattern
+            matching_files = []
+            if len(name_parts) > 1:
+                base_name = name_parts[0]
+                index = name_parts[1]
+                # Match patterns like "pcd_1.ext", "point_cloud_1.ext", etc.
+                for file in all_files:
+                    if base_name in file.stem and f'_{index}' in file.stem:
+                        matching_files.append(file)
+            else:
+                # Simple match - just find files containing the pattern
+                for file in all_files:
+                    if file_name in file.stem:
+                        matching_files.append(file)
+
+            # Process matching files
+            if matching_files:
+                # Sort by modification time (newest first) if multiple files match
+                if len(matching_files) > 1:
+                    matching_files.sort(key=lambda f: f.stat().st_mtime, reverse=True)
+                    self.run.log_message(
+                        f"Multiple files match '{file_name}'. Using most recent: {matching_files[0].name}"
+                    )
+
+                files_dict[file_name] = matching_files[0]
+                self.run.log_message(f"Mapped '{file_name}' to: {matching_files[0].name}")
+            elif is_required:
+                self.run.log_message(f"Required file '{file_name}' not found.", context=Context.WARNING.value)
+
+        return files_dict

synapse_sdk-1.0.0a55/synapse_sdk/plugins/categories/upload/templates/plugin/upload.py

@@ -0,0 +1,40 @@
+from pathlib import Path
+from typing import List
+
+
+class Uploader:
+    """Plugin upload action interface for organizing files.
+
+    This class provides a minimal interface for plugin developers to implement
+    their own file organization logic.
+    """
+
+    def __init__(self, run, path: Path, file_specification: List = None, organized_files: List = None):
+        """Initialize the plugin upload action class.
+
+        Args:
+            run: Plugin run object with logging capabilities.
+            path: Path object pointing to the upload target directory.
+            file_specification: List of specifications that define the structure of files to be uploaded.
+                Each specification contains details like file name, type, and requirements.
+        """
+        self.run = run
+        self.path = path
+        self.file_specification = file_specification
+        self.organized_files = organized_files
+
+    def handle_upload_files(self) -> List:
+        """Customize the organization of files for upload.
+
+        This method provides a hook for plugin developers to modify the default file organization.
+        You can override this method to filter files, transform data, or add custom metadata
+        based on your specific requirements.
+
+        Args:
+            organized_files (List): The default organized files structure.
+                Each item is a dictionary with 'files' and 'meta' keys.
+
+        Returns:
+            List: The modified list of organized files to be uploaded.
+        """
+        return self.organized_files

{synapse_sdk-1.0.0a54 → synapse_sdk-1.0.0a55}/synapse_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a54
+Version: 1.0.0a55
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

synapse_sdk-1.0.0a54/synapse_sdk/plugins/categories/upload/templates/plugin/upload.py

@@ -1,44 +0,0 @@
-from pathlib import Path
-from typing import List
-
-
-class Uploader:
-    """Plugin upload action class.
-
-    * Organize, upload, setup task, generate ground truths for the uploaded data.
-    """
-
-    def __init__(self, run, path: Path, *args, **kwargs):
-        """Initialize the plugin upload action class.
-
-        Args:
-            run: Plugin run object.
-            path: pathlib object by upload target destination path.
-        """
-        self.run = run
-        self.path = path
-
-    def handle_upload_files(self) -> List:
-        """Handle upload files.
-
-        * Organize data according to collection file specification structure.
-        * Structure files according to the file specification of the target collection.
-
-        Returns:
-            List: List of dictionaries containing 'files' and 'meta'.
-
-        Examples:
-            [
-              {
-          "files": {
-              'image_1': image_1_pathlib_object,
-              'image_2': image_2_pathlib_object,
-              'meta_1': meta_1_pathlib_object,
-          },
-          "meta": {
-            "key": "value"
-          }
-              }
-            ]
-        """
-        return []