synapse-sdk 1.0.0a55__tar.gz → 1.0.0a56__tar.gz

{synapse_sdk-1.0.0a55/synapse_sdk/plugins/templates/synapse-{{cookiecutter.plugin_code}}-plugin → synapse_sdk-1.0.0a56}/.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.1
+    rev: v0.11.10
     hooks:
       - id: ruff
         args: [ --fix ]

{synapse_sdk-1.0.0a55 → synapse_sdk-1.0.0a56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a55
+Version: 1.0.0a56
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a55 → synapse_sdk-1.0.0a56}/synapse_sdk/clients/backend/__init__.py

@@ -1,6 +1,6 @@
 from synapse_sdk.clients.backend.annotation import AnnotationClientMixin
 from synapse_sdk.clients.backend.core import CoreClientMixin
-from synapse_sdk.clients.backend.dataset import DatasetClientMixin
+from synapse_sdk.clients.backend.data_collection import DataCollectionClientMixin
 from synapse_sdk.clients.backend.hitl import HITLClientMixin
 from synapse_sdk.clients.backend.integration import IntegrationClientMixin
 from synapse_sdk.clients.backend.ml import MLClientMixin
@@ -9,7 +9,7 @@ from synapse_sdk.clients.backend.ml import MLClientMixin
 class BackendClient(
     AnnotationClientMixin,
     CoreClientMixin,
-    DatasetClientMixin,
+    DataCollectionClientMixin,
     IntegrationClientMixin,
     MLClientMixin,
     HITLClientMixin,

synapse_sdk-1.0.0a55/synapse_sdk/clients/backend/dataset.py → synapse_sdk-1.0.0a56/synapse_sdk/clients/backend/data_collection.py

@@ -8,13 +8,13 @@ from synapse_sdk.clients.base import BaseClient
 from synapse_sdk.clients.utils import get_batched_list
 
 
-class DatasetClientMixin(BaseClient):
-    def list_dataset(self):
+class DataCollectionClientMixin(BaseClient):
+    def list_data_collection(self):
         path = 'data_collections/'
         return self._list(path)
 
-    def get_dataset(self, data_collection_id):
-        """Get dataset from synapse-backend.
+    def get_data_collection(self, data_collection_id):
+        """Get data_collection from synapse-backend.
 
         Args:
             data_collection_id: The data_collection id to get.
@@ -40,19 +40,19 @@ class DatasetClientMixin(BaseClient):
         path = 'data_units/'
         return self._post(path, data=data)
 
-    def upload_dataset(
+    def upload_data_collection(
         self,
-        dataset_id: int,
-        dataset: Dict,
+        data_collection_id: int,
+        data_collection: Dict,
         project_id: Optional[int] = None,
         batch_size: int = 1000,
         process_pool: int = 10,
     ):
-        """Upload dataset to synapse-backend.
+        """Upload data_collection to synapse-backend.
 
         Args:
-            dataset_id: The dataset id to upload the data to.
-            dataset: The dataset to upload.
+            data_collection_id: The data_collection id to upload the data to.
+            data_collection: The data_collection to upload.
                 * structure:
                     - files: The files to upload. (key: file name, value: file pathlib object)
                     - meta: The meta data to upload.
@@ -60,14 +60,14 @@ class DatasetClientMixin(BaseClient):
             batch_size: The batch size to upload the data.
             process_pool: The process pool to upload the data.
         """
-        # TODO validate dataset with schema
+        # TODO validate data_collection with schema
 
-        params = [(data, dataset_id) for data in dataset]
+        params = [(data, data_collection_id) for data in data_collection]
 
         with Pool(processes=process_pool) as pool:
-            dataset = pool.starmap(self.upload_data_file, tqdm(params))
+            data_collection = pool.starmap(self.upload_data_file, tqdm(params))
 
-        batches = get_batched_list(dataset, batch_size)
+        batches = get_batched_list(data_collection, batch_size)
 
         for batch in tqdm(batches):
             data_units = self.create_data_units(batch)
@@ -90,7 +90,7 @@ class DatasetClientMixin(BaseClient):
                 * structure:
                     - files: The files to upload. (key: file name, value: file pathlib object)
                     - meta: The meta data to upload.
-            data_collection_id: The dataset id to upload the data to.
+            data_collection_id: The data_collection id to upload the data to.
 
         Returns:
             Dict: The result of the upload.

{synapse_sdk-1.0.0a55 → synapse_sdk-1.0.0a56}/synapse_sdk/plugins/categories/export/actions/export.py

@@ -27,6 +27,13 @@ class ExportRun(Run):
         error: str | None = None
         created: str
 
+    class MetricsRecord(BaseModel):
+        """Metrics record model."""
+
+        stand_by: int
+        failed: int
+        success: int
+
     def log_file(
         self, log_type: str, target_id: int, data_file_info: dict, status: ExportStatus, error: str | None = None
     ):
@@ -51,6 +58,16 @@ class ExportRun(Run):
             ).model_dump(),
         )
 
+    def log_metrics(self, record: MetricsRecord, category: str):
+        """Log export metrics.
+
+        Args:
+            record (MetricsRecord): The metrics record to log.
+            category (str): The category of the metrics.
+        """
+        record = self.MetricsRecord.model_validate(record)
+        self.set_metrics(value=record.dict(), category=category)
+
     def export_log_json_file(
         self,
         target_id: int,
@@ -263,6 +280,7 @@ class ExportAction(Action):
             'proportion': 100,
         }
     }
+    metrics_categories = {'data_file', 'original_file'}
 
     def get_filtered_results(self, filters, handler):
         """Get filtered target results."""

{synapse_sdk-1.0.0a55 → synapse_sdk-1.0.0a56}/synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -42,6 +42,8 @@ def export(run, export_items, path_root, **params):
     origin_files_output_path.mkdir(parents=True, exist_ok=True)
 
     total = params['count']
+    original_file_metrics_record = run.MetricsRecord(stand_by=total, success=0, failed=0)
+    data_file_metrics_record = run.MetricsRecord(stand_by=total, success=0, failed=0)
     # progress init
     run.set_progress(0, total, category='dataset_conversion')
     for no, export_item in enumerate(export_items, start=1):
@@ -56,12 +58,30 @@ def export(run, export_items, path_root, **params):
         if save_original_file_flag:
             if no == 1:
                 run.log_message('Saving original file.')
-            save_original_file(run, final_data, origin_files_output_path, errors_original_file_list)
+            original_status = save_original_file(run, final_data, origin_files_output_path, errors_original_file_list)
+
+            original_file_metrics_record.stand_by -= 1
+            if original_status == ExportStatus.FAILED:
+                original_file_metrics_record.failed += 1
+                continue
+            else:
+                original_file_metrics_record.success += 1
+
+        run.log_metrics(record=original_file_metrics_record, category='original_file')
 
         # Extract data as JSON files
         if no == 1:
             run.log_message('Saving json file.')
-        save_as_json(run, final_data, json_output_path, errors_json_file_list)
+        data_status = save_as_json(run, final_data, json_output_path, errors_json_file_list)
+
+        data_file_metrics_record.stand_by -= 1
+        if data_status == ExportStatus.FAILED:
+            data_file_metrics_record.failed += 1
+            continue
+        else:
+            data_file_metrics_record.success += 1
+
+        run.log_metrics(record=data_file_metrics_record, category='data_file')
 
     run.end_log()
 
@@ -126,6 +146,7 @@ def save_original_file(run, result, base_path, error_file_list):
         status = ExportStatus.FAILED
 
     run.export_log_original_file(result['id'], file_info, status, error_msg)
+    return status
 
 
 def save_as_json(run, result, base_path, error_file_list):
@@ -152,3 +173,4 @@ def save_as_json(run, result, base_path, error_file_list):
         status = ExportStatus.FAILED
 
     run.export_log_json_file(result['id'], file_info, status, error_msg)
+    return status

synapse_sdk-1.0.0a56/synapse_sdk/plugins/categories/upload/actions/task_pre_annotation.py

@@ -0,0 +1,101 @@
+from enum import Enum
+from typing import Annotated
+
+from pydantic import AfterValidator, BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.utils.pydantic.validators import non_blank
+
+
+class TaskDataAnnotationType(str, Enum):
+    FILE = 'file'
+    INFERENCE = 'inference'
+
+
+class TaskPreAnnotationRun(Run):
+    pass
+
+
+class TaskPreAnnotationParams(BaseModel):
+    """TaskPreAnnotation action parameters.
+
+    Args:
+        name (str): The name of the action.
+        description (str | None): The description of the action.
+        project (int): The project ID.
+        data_collection (int): The data collection ID.
+        task_data_annotation_type (TaskDataAnnotationType): The type of task data annotation.
+    """
+
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str | None
+    project: int
+    data_collection: int
+    task_data_annotation_type: TaskDataAnnotationType
+
+    @field_validator('data_collection', mode='before')
+    @classmethod
+    def check_data_collection_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend collection exists."""
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_data_collection(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', 'Error occurred while checking data collection exists.')
+        return value
+
+    @field_validator('project', mode='before')
+    @classmethod
+    def check_project_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend project exists."""
+        if not value:
+            return value
+
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_project(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', 'Error occurred while checking project exists.')
+        return value
+
+
+@register_action
+class TaskPreAnnotationAction(Action):
+    """TaskPreAnnotation action class.
+
+    * Annotate data to tasks.
+    """
+
+    name = 'task_pre_annotation'
+    category = PluginCategory.UPLOAD
+    method = RunMethod.JOB
+    run_class = TaskPreAnnotationRun
+    progress_categories = {
+        'generate_tasks': {
+            'proportion': 10,
+        },
+        'annotate_task_data': {
+            'proportion': 90,
+        },
+    }
+
+    def start(self):
+        """Start task_pre_annotation action.
+
+        * Generate tasks.
+        * Annotate data to tasks.
+        """
+        task_pre_annotation = self.get_task_pre_annotation()
+        task_pre_annotation.handle_annotate_data_from_files()
+        return {}
+
+    def get_task_pre_annotation(self):
+        """Get task pre annotation entrypoint."""
+        return self.entrypoint()

synapse_sdk-1.0.0a56/synapse_sdk/plugins/categories/upload/actions/upload.py

@@ -0,0 +1,313 @@
+import json
+from datetime import datetime
+from enum import Enum
+from typing import Annotated, Dict, List
+
+from pydantic import AfterValidator, BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.clients.utils import get_batched_list
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.shared.enums import Context
+from synapse_sdk.utils.pydantic.validators import non_blank
+from synapse_sdk.utils.storage import get_pathlib
+
+
+class UploadStatus(str, Enum):
+    SUCCESS = 'success'
+    FAILED = 'failed'
+
+
+class UploadRun(Run):
+    class DataFileLog(BaseModel):
+        """Data file log model."""
+
+        data_file_info: str | None
+        status: UploadStatus
+        created: str
+
+    class DataUnitLog(BaseModel):
+        """Data unit log model."""
+
+        data_unit_id: int | None
+        status: UploadStatus
+        created: str
+
+    class TaskLog(BaseModel):
+        """Task log model."""
+
+        task_id: int | None
+        status: UploadStatus
+        created: str
+
+    def log_data_file(self, data_file_info: dict, status: UploadStatus):
+        """Upload data_file log.
+
+        Args:
+            data_file_info (dict): The json info of the data file.
+            checksum (str): The checksum of the data file.
+            status (DataUnitStatus): The status of the data unit.
+        """
+        now = datetime.now().isoformat()
+        self.log(
+            'upload_data_file',
+            self.DataFileLog(data_file_info=json.dumps(data_file_info), status=status.value, created=now).model_dump(),
+        )
+
+    def log_data_unit(self, data_unit_id: int, status: UploadStatus):
+        """Upload data_unit log.
+
+        Args:
+            data_unit_id (int): The ID of the data unit.
+            status (DataUnitStatus): The status of the data unit.
+        """
+        now = datetime.now().isoformat()
+        self.log(
+            'upload_data_unit',
+            self.DataUnitLog(data_unit_id=data_unit_id, status=status.value, created=now).model_dump(),
+        )
+
+    def log_task(self, task_id: int, status: UploadStatus):
+        """Upload task log.
+
+        Args:
+            task_id (int): The ID of the task.
+            status (UploadStatus): The status of the task.
+        """
+        now = datetime.now().isoformat()
+        self.log('upload_task', self.TaskLog(task_id=task_id, status=status.value, created=now).model_dump())
+
+
+class UploadParams(BaseModel):
+    """Upload action parameters.
+
+    Args:
+        name (str): The name of the action.
+        description (str | None): The description of the action.
+        checkpoint (int | None): The checkpoint of the action.
+        path (str): The path of the action.
+        storage (int): The storage of the action.
+        collection (int): The collection of the action.
+        project (int | None): The project of the action.
+        is_generate_tasks (bool): The flag to generate tasks.
+        is_generate_ground_truths (bool): The flag to generate ground truths
+    """
+
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str | None
+    path: str
+    storage: int
+    collection: int
+    project: int | None
+
+    @field_validator('storage', mode='before')
+    @classmethod
+    def check_storage_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend storage exists.
+
+        TODO: Need to define validation method naming convention.
+        TODO: Need to make validation method reusable.
+        """
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_storage(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Error occurred while checking storage exists.'))
+        return value
+
+    @field_validator('collection', mode='before')
+    @classmethod
+    def check_collection_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend collection exists."""
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_data_collection(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Error occurred while checking collection exists.'))
+        return value
+
+    @field_validator('project', mode='before')
+    @classmethod
+    def check_project_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend project exists."""
+        if not value:
+            return value
+
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_project(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Error occurred while checking project exists.'))
+        return value
+
+
+@register_action
+class UploadAction(Action):
+    """Upload action class.
+
+    Attrs:
+        name (str): The name of the action.
+        category (PluginCategory): The category of the action.
+        method (RunMethod): The method to run of the action.
+
+    Progress Categories:
+        analyze_collection: The progress category for the analyze collection process.
+        data_file_upload: The progress category for the upload process.
+        generate_data_units: The progress category for the generate data units process.
+    """
+
+    name = 'upload'
+    category = PluginCategory.UPLOAD
+    method = RunMethod.JOB
+    run_class = UploadRun
+    progress_categories = {
+        'analyze_collection': {
+            'proportion': 10,
+        },
+        'upload_data_files': {
+            'proportion': 50,
+        },
+        'generate_data_units': {
+            'proportion': 40,
+        },
+    }
+
+    def get_uploader(self, path, file_specification, organized_files):
+        """Get uploader from entrypoint."""
+        return self.entrypoint(self.run, path, file_specification, organized_files)
+
+    def start(self) -> Dict:
+        """Start upload process.
+
+        Returns:
+            Dict: The result of the upload process.
+        """
+        # Setup path object with path and storage.
+        storage = self.client.get_storage(self.params['storage'])
+        pathlib_cwd = get_pathlib(storage, self.params['path'])
+
+        # Analyze Collection file specifications to determine the data structure for upload.
+        file_specification_template = self._analyze_collection()
+        organized_files = self._organize_files(pathlib_cwd, file_specification_template)
+
+        # Initialize uploader.
+        uploader = self.get_uploader(pathlib_cwd, file_specification_template, organized_files)
+
+        # Setup result dict.
+        result = {}
+
+        # Get organized files from the uploader (plugin developer's custom implementation)
+        # or use the default organization method if uploader doesn't provide valid files
+        organized_files = uploader.handle_upload_files()
+
+        # Validate the organized files
+        if not self._validate_organized_files(organized_files, file_specification_template):
+            self.run.log_message('Validation failed.', context=Context.ERROR.value)
+            self.run.end_log()
+            return result
+
+        # Upload files to synapse-backend.
+        organized_files_count = len(organized_files)
+        if not organized_files_count:
+            self.run.log_message('Files not found on the path.', context=Context.WARNING.value)
+            self.run.end_log()
+            return result
+        uploaded_files = self._upload_files(organized_files, organized_files_count)
+        result['uploaded_files_count'] = len(uploaded_files)
+
+        # Generate data units for the uploaded data.
+        upload_result_count = len(uploaded_files)
+        if not upload_result_count:
+            self.run.log_message('No files were uploaded.', context=Context.WARNING.value)
+            self.run.end_log()
+            return result
+        generated_data_units = self._generate_data_units(uploaded_files, upload_result_count)
+        result['generated_data_units_count'] = len(generated_data_units)
+
+        self.run.end_log()
+        return result
+
+    def _analyze_collection(self) -> Dict:
+        """Analyze Synapse Collection Specifications.
+
+        Returns:
+            Dict: The file specifications of the collection.
+        """
+
+        # Initialize progress
+        self.run.set_progress(0, 1, category='analyze_collection')
+
+        client = self.run.client
+        collection_id = self.params['collection']
+        collection = client.get_data_collection(collection_id)
+
+        # Finish progress
+        self.run.set_progress(1, 1, category='analyze_collection')
+
+        return collection['file_specifications']
+
+    def _upload_files(self, organized_files, organized_files_count: int) -> List:
+        """Upload files to synapse-backend.
+
+        Returns:
+            Dict: The result of the upload.
+        """
+        # Initialize progress
+        self.run.set_progress(0, organized_files_count, category='upload_data_files')
+        self.run.log_message('Uploading data files...')
+
+        client = self.run.client
+        collection_id = self.params['collection']
+        upload_result = []
+        organized_files_count = len(organized_files)
+        current_progress = 0
+        for organized_file in organized_files:
+            uploaded_data_file = client.upload_data_file(organized_file, collection_id)
+            self.run.log_data_file(organized_file, UploadStatus.SUCCESS)
+            upload_result.append(uploaded_data_file)
+            self.run.set_progress(current_progress, organized_files_count, category='upload_data_files')
+            current_progress += 1
+
+        # Finish progress
+        self.run.set_progress(organized_files_count, organized_files_count, category='upload_data_files')
+        self.run.log_message('Upload data files completed.')
+
+        return upload_result
+
+    def _generate_data_units(self, uploaded_files: List, upload_result_count: int) -> List:
+        """Generate data units for the uploaded data.
+
+        TODO: make batch size configurable.
+
+        Returns:
+            Dict: The result of the generate data units process.
+        """
+        # Initialize progress
+        self.run.set_progress(0, upload_result_count, category='generate_data_units')
+
+        client = self.run.client
+
+        generated_data_units = []
+        current_progress = 0
+        batches = get_batched_list(uploaded_files, 100)
+        batches_count = len(batches)
+        for batch in batches:
+            created_data_units = client.create_data_units(batch)
+            generated_data_units.append(created_data_units)
+            self.run.set_progress(current_progress, batches_count, category='generate_data_units')
+            current_progress += 1
+            for created_data_unit in created_data_units:
+                self.run.log_data_unit(created_data_unit['id'], UploadStatus.SUCCESS)
+
+        # Finish progress
+        self.run.set_progress(upload_result_count, upload_result_count, category='generate_data_units')
+
+        return sum(generated_data_units, [])

{synapse_sdk-1.0.0a55 → synapse_sdk-1.0.0a56}/synapse_sdk/plugins/categories/upload/templates/config.yaml

@@ -1,9 +1,10 @@
 actions:
   upload:
     entrypoint: plugin.upload.Uploader
-    options:
-      allow_generate_tasks: false # Allow the plugin to generate tasks for the uploaded data
-      allow_generate_ground_truths: false # Allow the plugin to generate ground truths for the uploaded data
     supported_data_type: image # A primary data type of synapse backend collection. (e.g. 'image', 'text', 'video', 'pcd', 'audio')
     ui_schema: |
       Dumped FormKit Schema for upload plugin custom options
+  task_pre_annotation:
+    entrypoint: plugin.upload.TaskPreAnnotation
+    ui_schema: |
+      Dumped FormKit Schema for upload plugin custom options

synapse_sdk-1.0.0a56/synapse_sdk/plugins/categories/upload/templates/plugin/task_pre_annotation.py

@@ -0,0 +1,14 @@
+class TaskPreAnnotation:
+    def __init__(self, run, *args, **kwargs):
+        """Initialize the plugin task pre annotation action class.
+
+        Args:
+            run: Plugin run object.
+        """
+        self.run = run
+
+    def handle_annotate_data_from_files(self):
+        pass
+
+    def handle_annotate_data_with_inference(self):
+        pass

{synapse_sdk-1.0.0a55 → synapse_sdk-1.0.0a56}/synapse_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a55
+Version: 1.0.0a56
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a55 → synapse_sdk-1.0.0a56}/synapse_sdk.egg-info/SOURCES.txt

@@ -48,7 +48,7 @@ synapse_sdk/clients/agent/service.py
 synapse_sdk/clients/backend/__init__.py
 synapse_sdk/clients/backend/annotation.py
 synapse_sdk/clients/backend/core.py
-synapse_sdk/clients/backend/dataset.py
+synapse_sdk/clients/backend/data_collection.py
 synapse_sdk/clients/backend/hitl.py
 synapse_sdk/clients/backend/integration.py
 synapse_sdk/clients/backend/ml.py
@@ -117,9 +117,11 @@ synapse_sdk/plugins/categories/smart_tool/templates/plugin/__init__.py
 synapse_sdk/plugins/categories/smart_tool/templates/plugin/auto_label.py
 synapse_sdk/plugins/categories/upload/__init__.py
 synapse_sdk/plugins/categories/upload/actions/__init__.py
+synapse_sdk/plugins/categories/upload/actions/task_pre_annotation.py
 synapse_sdk/plugins/categories/upload/actions/upload.py
 synapse_sdk/plugins/categories/upload/templates/config.yaml
 synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py
+synapse_sdk/plugins/categories/upload/templates/plugin/task_pre_annotation.py
 synapse_sdk/plugins/categories/upload/templates/plugin/upload.py
 synapse_sdk/plugins/templates/cookiecutter.json
 synapse_sdk/plugins/templates/hooks/post_gen_project.py