synapse-sdk 1.0.0a35__tar.gz → 1.0.0a37__tar.gz

{synapse_sdk-1.0.0a35/synapse_sdk.egg-info → synapse_sdk-1.0.0a37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a35
+Version: 1.0.0a37
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a35 → synapse_sdk-1.0.0a37}/synapse_sdk/clients/agent/core.py

@@ -5,3 +5,7 @@ class CoreClientMixin(BaseClient):
     def health_check(self):
         path = 'health/'
         return self._get(path)
+
+    def get_metrics(self, panel):
+        path = f'metrics/{panel}/'
+        return self._get(path)

{synapse_sdk-1.0.0a35 → synapse_sdk-1.0.0a37}/synapse_sdk/plugins/categories/export/actions/export.py

@@ -1,17 +1,77 @@
+import json
 from abc import ABC, abstractmethod
-from typing import Any, Literal
+from datetime import datetime
+from typing import Annotated, Any, Literal
 
-from pydantic import BaseModel, field_validator
+from pydantic import AfterValidator, BaseModel, field_validator
 from pydantic_core import PydanticCustomError
 
 from synapse_sdk.clients.exceptions import ClientError
 from synapse_sdk.i18n import gettext as _
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.categories.export.enums import ExportStatus
 from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.utils.pydantic.validators import non_blank
 from synapse_sdk.utils.storage import get_pathlib
 
 
+class ExportRun(Run):
+    class DataFileLog(BaseModel):
+        """Data file log model."""
+
+        target_id: int
+        data_file_info: str | None
+        status: ExportStatus
+        error: str | None = None
+        created: str
+
+    def log_file(
+        self, log_type: str, target_id: int, data_file_info: dict, status: ExportStatus, error: str | None = None
+    ):
+        """Log export file information.
+
+        Args:
+            log_type (str): The type of log ('export_data_file' or 'export_origin_file').
+            target_id (int): The ID of the data file.
+            data_file_info (dict): The JSON info of the data file.
+            status (ExportStatus): The status of the data file.
+            error (str | None): The error message, if any.
+        """
+        now = datetime.now().isoformat()
+        self.log(
+            log_type,
+            self.DataFileLog(
+                target_id=target_id,
+                data_file_info=json.dumps(data_file_info),
+                status=status.value,
+                error=error,
+                created=now,
+            ).model_dump(),
+        )
+
+    def export_log_json_file(
+        self,
+        target_id: int,
+        data_file_info: dict,
+        status: ExportStatus = ExportStatus.STAND_BY,
+        error: str | None = None,
+    ):
+        """Log export json data file."""
+        self.log_file('export_data_file', target_id, data_file_info, status, error)
+
+    def export_log_original_file(
+        self,
+        target_id: int,
+        data_file_info: dict,
+        status: ExportStatus = ExportStatus.STAND_BY,
+        error: str | None = None,
+    ):
+        """Log export origin data file."""
+        self.log_file('export_origin_file', target_id, data_file_info, status, error)
+
+
 class ExportTargetHandler(ABC):
     """
     Abstract base class for handling export targets.
@@ -153,6 +213,8 @@ class ExportParams(BaseModel):
     Parameters for the export action.
 
     Attributes:
+        name (str): The name of the action.
+        description (str | None): The description of the action.
         storage (int): The storage ID to save the exported data.
         save_original_file (bool): Whether to save the original file.
         path (str): The path to save the exported data.
@@ -160,6 +222,8 @@ class ExportParams(BaseModel):
         filter (dict): The filter criteria to apply.
     """
 
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str | None = None
     storage: int
     save_original_file: bool = True
     path: str
@@ -193,6 +257,7 @@ class ExportAction(Action):
     category = PluginCategory.EXPORT
     method = RunMethod.JOB
     params_model = ExportParams
+    run_class = ExportRun
     progress_categories = {
         'dataset_conversion': {
             'proportion': 100,

synapse_sdk-1.0.0a37/synapse_sdk/plugins/categories/export/enums.py

@@ -0,0 +1,7 @@
+from enum import Enum
+
+
+class ExportStatus(str, Enum):
+    SUCCESS = 'success'
+    FAILED = 'failed'
+    STAND_BY = 'stand_by'

{synapse_sdk-1.0.0a35 → synapse_sdk-1.0.0a37}/synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -3,6 +3,8 @@ from pathlib import Path
 
 import requests
 
+from synapse_sdk.plugins.categories.export.enums import ExportStatus
+
 
 def export(run, export_items, path_root, **params):
     """Executes the export task.
@@ -54,12 +56,12 @@ def export(run, export_items, path_root, **params):
         if save_original_file_flag:
             if no == 1:
                 run.log_message('Saving original file.')
-            save_original_file(final_data, origin_files_output_path, errors_original_file_list)
+            save_original_file(run, final_data, origin_files_output_path, errors_original_file_list)
 
         # Extract data as JSON files
         if no == 1:
             run.log_message('Saving json file.')
-        save_as_json(final_data, json_output_path, errors_json_file_list)
+        save_as_json(run, final_data, json_output_path, errors_json_file_list)
 
     run.end_log()
 
@@ -100,10 +102,11 @@ def get_original_file_pathlib(files):
     return Path(files['meta']['path_original'])
 
 
-def save_original_file(result, base_path, error_file_list):
+def save_original_file(run, result, base_path, error_file_list):
     """Saves the original file.
 
     Args:
+        run : Execution object
         result (dict): API response data containing file information.
         base_path (Path): The directory where the file will be saved.
         error_file_list (list): A list to store error files.
@@ -111,17 +114,25 @@ def save_original_file(result, base_path, error_file_list):
     file_url = result['files']['url']
     file_name = get_original_file_pathlib(result['files']).name
     response = requests.get(file_url)
+    file_info = {'file_name': file_name}
+    error_msg = ''
     try:
         with (base_path / file_name).open('wb') as file:
             file.write(response.content)
+        status = ExportStatus.SUCCESS
     except Exception as e:
-        error_file_list.append([file_name, str(e)])
+        error_msg = str(e)
+        error_file_list.append([file_name, error_msg])
+        status = ExportStatus.FAILED
+
+    run.export_log_original_file(result['id'], file_info, status, error_msg)
 
 
-def save_as_json(result, base_path, error_file_list):
+def save_as_json(run, result, base_path, error_file_list):
     """Saves the data as a JSON file.
 
     Args:
+        run : Execution object
         result (dict): API response data containing file information.
         base_path (Path): The directory where the file will be saved.
         error_file_list (list): A list to store error files.
@@ -129,8 +140,15 @@ def save_as_json(result, base_path, error_file_list):
     # Default save file name: original file name
     file_name = get_original_file_pathlib(result['files']).stem
     json_data = result['data']
+    file_info = {'file_name': file_name}
+    error_msg = ''
     try:
         with (base_path / f'{file_name}.json').open('w', encoding='utf-8') as f:
             json.dump(json_data, f, indent=4, ensure_ascii=False)
+        status = ExportStatus.SUCCESS
     except Exception as e:
+        error_msg = str(e)
         error_file_list.append([f'{file_name}.json', str(e)])
+        status = ExportStatus.FAILED
+
+    run.export_log_json_file(result['id'], file_info, status, error_msg)

{synapse_sdk-1.0.0a35 → synapse_sdk-1.0.0a37}/synapse_sdk/plugins/categories/neural_net/templates/config.yaml

@@ -1,5 +1,13 @@
-task: object_detection
+name: plugin_name
+code: plugin_code
+version: plugin_version
+readme: README.md
+description: This is plugin_name plugin
+category: neural_net
+tasks:
+  - object_detection
 data_type: image
+package_manager: uv
 actions:
   train:
     dataset: dataset
@@ -18,4 +26,4 @@ actions:
     endpoints:
       - method: get
   test:
-    entrypoint: plugin.test.test
+    entrypoint: plugin.test.test

{synapse_sdk-1.0.0a35 → synapse_sdk-1.0.0a37}/synapse_sdk/plugins/categories/upload/actions/upload.py

@@ -1,3 +1,6 @@
+import json
+from datetime import datetime
+from enum import Enum
 from typing import Annotated, Dict, List
 
 from pydantic import AfterValidator, BaseModel, field_validator
@@ -16,8 +19,69 @@ from synapse_sdk.utils.pydantic.validators import non_blank
 from synapse_sdk.utils.storage import get_pathlib
 
 
+class UploadStatus(str, Enum):
+    SUCCESS = 'success'
+    FAILED = 'failed'
+
+
 class UploadRun(Run):
-    pass
+    class DataFileLog(BaseModel):
+        """Data file log model."""
+
+        data_file_info: str | None
+        status: UploadStatus
+        created: str
+
+    class DataUnitLog(BaseModel):
+        """Data unit log model."""
+
+        data_unit_id: int | None
+        status: UploadStatus
+        created: str
+
+    class TaskLog(BaseModel):
+        """Task log model."""
+
+        task_id: int | None
+        status: UploadStatus
+        created: str
+
+    def log_data_file(self, data_file_info: dict, status: UploadStatus):
+        """Upload data_file log.
+
+        Args:
+            data_file_info (dict): The json info of the data file.
+            checksum (str): The checksum of the data file.
+            status (DataUnitStatus): The status of the data unit.
+        """
+        now = datetime.now().isoformat()
+        self.log(
+            'upload_data_file',
+            self.DataFileLog(data_file_info=json.dumps(data_file_info), status=status.value, created=now).model_dump(),
+        )
+
+    def log_data_unit(self, data_unit_id: int, status: UploadStatus):
+        """Upload data_unit log.
+
+        Args:
+            data_unit_id (int): The ID of the data unit.
+            status (DataUnitStatus): The status of the data unit.
+        """
+        now = datetime.now().isoformat()
+        self.log(
+            'upload_data_unit',
+            self.DataUnitLog(data_unit_id=data_unit_id, status=status.value, created=now).model_dump(),
+        )
+
+    def log_task(self, task_id: int, status: UploadStatus):
+        """Upload task log.
+
+        Args:
+            task_id (int): The ID of the task.
+            status (UploadStatus): The status of the task.
+        """
+        now = datetime.now().isoformat()
+        self.log('upload_task', self.TaskLog(task_id=task_id, status=status.value, created=now).model_dump())
 
 
 class UploadParams(BaseModel):
@@ -108,6 +172,7 @@ class UploadAction(Action):
     name = 'upload'
     category = PluginCategory.UPLOAD
     method = RunMethod.JOB
+    run_class = UploadRun
     progress_categories = {
         'analyze_collection': {
             'proportion': 5,
@@ -144,9 +209,7 @@ class UploadAction(Action):
         uploader = self.get_uploader(pathlib_cwd)
 
         # Analyze Collection file specifications to determine the data structure for upload.
-        self.run.set_progress(0, 1, category='analyze_collection')
         file_specification_template = self._analyze_collection()
-        self.run.set_progress(1, 1, category='analyze_collection')
 
         # Setup result dict.
         result = {}
@@ -162,49 +225,35 @@ class UploadAction(Action):
         if not organized_files_count:
             self.run.log_message('Files not found on the path.', context=Context.WARNING.value)
             return result
-
-        self.run.set_progress(0, organized_files_count, category='upload_data_files')
-        self.run.log_message('Uploading data files...')
-        result['uploaded_files'] = self._upload_files(organized_files)
-        self.run.set_progress(organized_files_count, organized_files_count, category='upload_data_files')
-        self.run.log_message('Upload data files completed.')
+        uploaded_files = self._upload_files(organized_files, organized_files_count)
+        result['uploaded_files_count'] = len(uploaded_files)
 
         # Generate data units for the uploaded data.
-        upload_result_count = len(result['uploaded_files'])
+        upload_result_count = len(uploaded_files)
         if not upload_result_count:
             self.run.log_message('No files were uploaded.', context=Context.WARNING.value)
             return result
-
-        self.run.set_progress(0, upload_result_count, category='generate_data_units')
-        generated_data_units = self._generate_data_units(result['uploaded_files'])
-        result['generated_data_units'] = generated_data_units
-        self.run.set_progress(upload_result_count, upload_result_count, category='generate_data_units')
+        generated_data_units = self._generate_data_units(uploaded_files, upload_result_count)
+        result['generated_data_units_count'] = len(generated_data_units)
 
         # Setup task with uploaded synapse-backend data units.
         if not len(generated_data_units):
             self.run.log_message('No data units were generated.', context=Context.WARNING.value)
             return result
 
-        self.run.set_progress(0, 1, category='generate_tasks')
         if self.config['options']['allow_generate_tasks'] and self.params['is_generate_tasks']:
-            self.run.log_message('Generating tasks with data files...')
-            self._generate_tasks(generated_data_units)
-            self.run.log_message('Generating tasks completed')
+            generated_tasks = self._generate_tasks(generated_data_units)
+            result['generated_tasks_count'] = len(generated_tasks)
         else:
             self.run.log_message('Generating tasks process has passed.')
 
-        self.run.set_progress(1, 1, category='generate_tasks')
-
         # Generate ground truths for the uploaded data.
         # TODO: Need to add ground truths generation logic later.
-        self.run.set_progress(0, 1, category='generate_ground_truths')
         if self.config['options']['allow_generate_ground_truths'] and self.params['is_generate_ground_truths']:
-            self.run.log_message('Generating ground truths...')
-            self._generate_ground_truths()
-            self.run.log_message('Generating ground truths completed')
+            generated_ground_truths = self._generate_ground_truths()
+            result['generated_ground_truths_count'] = len(generated_ground_truths)
         else:
             self.run.log_message('Generating ground truths process has passed.')
-        self.run.set_progress(1, 1, category='generate_ground_truths')
 
         return result
 
@@ -214,9 +263,17 @@ class UploadAction(Action):
         Returns:
             Dict: The file specifications of the collection.
         """
+
+        # Initialize progress
+        self.run.set_progress(0, 1, category='analyze_collection')
+
         client = self.run.client
         collection_id = self.params['collection']
         collection = client.get_dataset(collection_id)
+
+        # Finish progress
+        self.run.set_progress(1, 1, category='analyze_collection')
+
         return collection['file_specifications']
 
     def _validate_organized_files(self, file_specification_template: Dict, organized_files: List) -> bool:
@@ -224,24 +281,35 @@ class UploadAction(Action):
         validator = FileSpecificationValidator(file_specification_template, organized_files)
         return validator.validate()
 
-    def _upload_files(self, organized_files) -> List:
+    def _upload_files(self, organized_files, organized_files_count: int) -> List:
         """Upload files to synapse-backend.
 
         Returns:
             Dict: The result of the upload.
         """
+        # Initialize progress
+        self.run.set_progress(0, organized_files_count, category='upload_data_files')
+        self.run.log_message('Uploading data files...')
+
         client = self.run.client
         collection_id = self.params['collection']
         upload_result = []
         organized_files_count = len(organized_files)
         current_progress = 0
         for organized_file in organized_files:
-            upload_result.append(client.upload_data_file(organized_file, collection_id))
+            uploaded_data_file = client.upload_data_file(organized_file, collection_id)
+            self.run.log_data_file(organized_file, UploadStatus.SUCCESS)
+            upload_result.append(uploaded_data_file)
             self.run.set_progress(current_progress, organized_files_count, category='upload_data_files')
             current_progress += 1
+
+        # Finish progress
+        self.run.set_progress(organized_files_count, organized_files_count, category='upload_data_files')
+        self.run.log_message('Upload data files completed.')
+
         return upload_result
 
-    def _generate_data_units(self, uploaded_files: List) -> List:
+    def _generate_data_units(self, uploaded_files: List, upload_result_count: int) -> List:
         """Generate data units for the uploaded data.
 
         TODO: make batch size configurable.
@@ -249,23 +317,36 @@ class UploadAction(Action):
         Returns:
             Dict: The result of the generate data units process.
         """
+        # Initialize progress
+        self.run.set_progress(0, upload_result_count, category='generate_data_units')
+
         client = self.run.client
 
-        generation_result = []
+        generated_data_units = []
         current_progress = 0
         batches = get_batched_list(uploaded_files, 100)
         batches_count = len(batches)
         for batch in batches:
-            generation_result.append(client.create_data_units(batch))
+            created_data_units = client.create_data_units(batch)
+            generated_data_units.append(created_data_units)
             self.run.set_progress(current_progress, batches_count, category='generate_data_units')
             current_progress += 1
-        return generation_result
+            for created_data_unit in created_data_units:
+                self.run.log_data_unit(created_data_unit['id'], UploadStatus.SUCCESS)
+
+        # Finish progress
+        self.run.set_progress(upload_result_count, upload_result_count, category='generate_data_units')
 
-    def _generate_tasks(self, generated_data_units: List):
+        return sum(generated_data_units, [])
+
+    def _generate_tasks(self, generated_data_units: List) -> List:
         """Setup task with uploaded synapse-backend data units.
 
         TODO: make batch size configurable.
         """
+        # Initialize progress
+        self.run.set_progress(0, 1, category='generate_tasks')
+        self.run.log_message('Generating tasks with data files...')
 
         # Prepare batches for processing
         client = self.run.client
@@ -273,21 +354,37 @@ class UploadAction(Action):
         current_progress = 0
 
         # Generate tasks
+        generated_tasks = []
         generated_data_units_count = len(generated_data_units)
-        for data_units in generated_data_units:
+        for data_unit in generated_data_units:
             tasks_data = []
-            for data_unit in data_units:
-                task_data = {'project': project_id, 'data_unit': data_unit['id']}
-                tasks_data.append(task_data)
-
+            task_data = {'project': project_id, 'data_unit': data_unit['id']}
+            tasks_data.append(task_data)
             if tasks_data:
-                client.create_tasks(tasks_data)
+                created_tasks = client.create_tasks(tasks_data)
+                created_task_ids = [created_task['id'] for created_task in created_tasks]
+                generated_tasks.append(created_task_ids)
+                for created_task_id in created_task_ids:
+                    self.run.log_task(created_task_id, UploadStatus.SUCCESS)
 
             self.run.set_progress(current_progress, generated_data_units_count, category='generate_tasks')
             current_progress += 1
 
+        # Finish progress
+        self.run.log_message('Generating tasks completed')
+        self.run.set_progress(1, 1, category='generate_tasks')
+
+        return sum(generated_tasks, [])
+
     def _generate_ground_truths(self):
         """Generate ground truths for the uploaded data.
 
         TODO: Need to add ground truths generation logic later.
         """
+        # Initialize progress
+        self.run.set_progress(0, 1, category='generate_ground_truths')
+        self.run.log_message('Generating ground truths...')
+
+        # Finish progress
+        self.run.log_message('Generating ground truths completed')
+        self.run.set_progress(1, 1, category='generate_ground_truths')

synapse_sdk-1.0.0a37/synapse_sdk/utils/dataset.py

@@ -0,0 +1,46 @@
+import logging
+
+
+def remap_class_labels(dataset, data_type, class_pattern, new_class):
+    """
+    Remaps class labels in a dataset by replacing specific classification patterns.
+
+    This function finds items of a specified data type (e.g., 'pcd', 'image', 'video')
+    where the classification matches a given pattern (e.g., 'truck', 'car'), and
+    remaps them to a new class label (e.g., 'vehicle'). Useful for consolidating
+    or standardizing class labels in ground truth data.
+
+    Args:
+        dataset (dict): The dataset containing ground truth labels
+        data_type (str): Type of data to process (e.g., 'pcd', 'image', 'video')
+        class_pattern (dict): Pattern to identify target classifications (e.g., {'name': 'truck'})
+        new_class (dict): New classification to apply (e.g., {'name': 'vehicle'})
+
+    Returns:
+        dict: Dataset with remapped class labels
+    """
+    updated_dataset = dataset.copy()
+
+    if data_type not in updated_dataset:
+        logging.log(logging.WARNING, f"Data type '{data_type}' not found in dataset.")
+        return updated_dataset
+
+    def matches_pattern(classification, pattern):
+        """Check if a classification matches the specified pattern."""
+        for key, pattern_value in pattern.items():
+            if key not in classification:
+                return False
+
+            if isinstance(pattern_value, dict) and isinstance(classification[key], dict):
+                if not matches_pattern(classification[key], pattern_value):
+                    return False
+            elif classification[key] != pattern_value:
+                return False
+
+        return True
+
+    for item in updated_dataset[data_type]:
+        if 'classification' in item and matches_pattern(item['classification'], class_pattern):
+            item['classification'] = new_class.copy()
+
+    return updated_dataset

{synapse_sdk-1.0.0a35 → synapse_sdk-1.0.0a37}/synapse_sdk/utils/storage/__init__.py

@@ -36,3 +36,33 @@ def get_pathlib(storage_config: str | dict, path_root: str) -> Path:
     """
     storage_class = get_storage(storage_config)
     return storage_class.get_pathlib(path_root)
+
+
+def get_path_file_count(storage_config: str | dict, path_root: str) -> int:
+    """Get the file count in the path.
+
+    Args:
+        storage_config (str | dict): The storage config by synapse-backend storage api.
+        path (str): The path.
+
+    Returns:
+        int: The file count in the path.
+    """
+    storage_class = get_storage(storage_config)
+    pathlib_obj = storage_class.get_pathlib(path_root)
+    return storage_class.get_path_file_count(pathlib_obj)
+
+
+def get_path_total_size(storage_config: str | dict, path_root: str) -> int:
+    """Get total size of the files in the path.
+
+    Args:
+        storage_config (str | dict): The storage config by synapse-backend storage api.
+        path (str): The path.
+
+    Returns:
+        int: The total size of the files in the path.
+    """
+    storage_class = get_storage(storage_config)
+    pathlib_obj = storage_class.get_pathlib(path_root)
+    return storage_class.get_path_total_size(pathlib_obj)

{synapse_sdk-1.0.0a35 → synapse_sdk-1.0.0a37}/synapse_sdk/utils/storage/providers/__init__.py

@@ -39,4 +39,34 @@ class BaseStorage:
         raise NotImplementedError
 
     def get_pathlib(self, path):
+        """Get the path as a pathlib object.
+
+        Args:
+            path (str): The path to convert.
+
+        Returns:
+            pathlib.Path: The converted path.
+        """
+        raise NotImplementedError
+
+    def get_path_file_count(self, pathlib_obj):
+        """Get the file count in the path.
+
+        Args:
+            pathlib_obj (UPath): The path to get file count.
+
+        Returns:
+            int: The file count in the path.
+        """
+        raise NotImplementedError
+
+    def get_path_total_size(self, pathlib_obj):
+        """Get the total size of the path.
+
+        Args:
+            pathlib_obj (UPath): The path to get total size.
+
+        Returns:
+            int: The total size of the path.
+        """
         raise NotImplementedError

{synapse_sdk-1.0.0a35 → synapse_sdk-1.0.0a37}/synapse_sdk/utils/storage/providers/s3.py

@@ -41,3 +41,38 @@ class S3Storage(BaseStorage):
 
     def get_pathlib(self, path):
         return self.upath.joinuri(path)
+
+    def get_path_file_count(self, pathlib_obj: UPath):
+        """Get file count in the path from S3 provider.
+
+        TODO: Need to find a method to get file count using S3 API
+
+        Args:
+            pathlib_obj (UPath): The path to get file count.
+
+        Returns:
+            int: The file count in the path.
+        """
+        count = 0
+        files = list(pathlib_obj.glob('**/*'))
+        for file in files:
+            if file.is_file():
+                count += 1
+        return count
+
+    def get_path_total_size(self, pathlib_obj: UPath):
+        """Get total size of the files in the path from S3 provider.
+
+        TODO: Need to find a method to get total file size using S3 API
+
+        Args:
+            pathlib_obj (UPath): The path to get total size.
+
+        Returns:
+            int: The total size of the files in the path.
+        """
+        total_size = 0
+        for file in pathlib_obj.glob('**/*'):
+            if file.is_file():
+                total_size += file.stat().st_size
+        return total_size