synapse-sdk 1.0.0a57__tar.gz → 1.0.0a59__tar.gz

{synapse_sdk-1.0.0a57 → synapse_sdk-1.0.0a59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a57
+Version: 1.0.0a59
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a57 → synapse_sdk-1.0.0a59}/synapse_sdk/clients/backend/annotation.py

@@ -7,6 +7,14 @@ class AnnotationClientMixin(BaseClient):
         path = f'projects/{pk}/'
         return self._get(path)
 
+    def get_task(self, pk, params):
+        path = f'tasks/{pk}/'
+        return self._get(path, params=params)
+
+    def patch_task(self, pk, data):
+        path = f'tasks/{pk}/'
+        return self._patch(path, data=data)
+
     def get_task_tag(self, pk):
         path = f'task_tags/{pk}/'
         return self._get(path)

{synapse_sdk-1.0.0a57 → synapse_sdk-1.0.0a59}/synapse_sdk/clients/backend/data_collection.py

@@ -5,7 +5,7 @@ from typing import Dict, Optional
 from tqdm import tqdm
 
 from synapse_sdk.clients.base import BaseClient
-from synapse_sdk.clients.utils import get_batched_list
+from synapse_sdk.clients.utils import get_batched_list, get_default_url_conversion
 
 
 class DataCollectionClientMixin(BaseClient):
@@ -31,6 +31,10 @@ class DataCollectionClientMixin(BaseClient):
         path = 'data_files/'
         return self._post(path, files={'file': file_path})
 
+    def get_data_unit(self, data_unit_id: int, params=None):
+        path = f'data_units/{data_unit_id}/'
+        return self._get(path, params=params)
+
     def create_data_units(self, data):
         """Create data units to synapse-backend.
 
@@ -40,6 +44,11 @@ class DataCollectionClientMixin(BaseClient):
         path = 'data_units/'
         return self._post(path, data=data)
 
+    def list_data_units(self, params=None, url_conversion=None, list_all=False):
+        path = 'data_units/'
+        url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
+        return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
+
     def upload_data_collection(
         self,
         data_collection_id: int,

synapse_sdk-1.0.0a59/synapse_sdk/plugins/categories/pre_annotation/actions/to_task.py

@@ -0,0 +1,210 @@
+import json
+from datetime import datetime
+from enum import Enum
+from typing import Annotated
+
+import requests
+from pydantic import AfterValidator, BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.utils.pydantic.validators import non_blank
+
+
+class AnnotationMethod(str, Enum):
+    FILE = 'file'
+    INFERENCE = 'inference'
+
+
+class AnnotateTaskDataStatus(str, Enum):
+    SUCCESS = 'success'
+    FAILED = 'failed'
+
+
+class ToTaskRun(Run):
+    class AnnotateTaskDataLog(BaseModel):
+        """Log model for annotate task data."""
+
+        task_info: str | None
+        status: AnnotateTaskDataStatus
+        created: str
+
+    class MetricsRecord(BaseModel):
+        """Metrics record model."""
+
+        stand_by: int
+        failed: int
+        success: int
+
+    def log_annotate_task_data(self, task_info: dict, status: AnnotateTaskDataStatus):
+        """Log annotate task data."""
+        now = datetime.now().isoformat()
+        self.log(
+            'annotate_task_data',
+            self.AnnotateTaskDataLog(task_info=json.dumps(task_info), status=status, created=now).model_dump(),
+        )
+
+    def log_metrics(self, record: MetricsRecord, category: str):
+        """Log FileToTask metrics.
+
+        Args:
+            record (MetricsRecord): The metrics record to log.
+            category (str): The category of the metrics.
+        """
+        record = self.MetricsRecord.model_validate(record)
+        self.set_metrics(value=record.dict(), category=category)
+
+
+class ToTaskParams(BaseModel):
+    """ToTask action parameters.
+
+    Args:
+        name (str): The name of the action.
+        description (str | None): The description of the action.
+        project (int): The project ID.
+        task_filter (dict): The filter of tasks.
+        method (AnnotationMethod): The method of annotation.
+        target_specification_name (str | None): The name of the target specification.
+        pre_processor (int | None): The pre processor ID.
+        pre_processor_params (dict): The params of the pre processor.
+    """
+
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str | None = None
+    project: int
+    task_filter: dict
+    method: AnnotationMethod | None = None
+    target_specification_name: str | None = None
+    pre_processor: int | None = None
+    pre_processor_params: dict
+
+    @field_validator('project', mode='before')
+    @classmethod
+    def check_project_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend project exists."""
+        if not value:
+            return value
+
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_project(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', 'Error occurred while checking project exists.')
+        return value
+
+
+@register_action
+class ToTaskAction(Action):
+    """ToTask action class.
+
+    * Annotate data to tasks.
+    """
+
+    name = 'to_task'
+    category = PluginCategory.PRE_ANNOTATION
+    method = RunMethod.JOB
+    run_class = ToTaskRun
+    progress_categories = {
+        'annotate_task_data': {
+            'proportion': 100,
+        },
+    }
+    metrics_categories = {'annotate_task_data'}
+
+    def start(self):
+        """Start file_to_task action.
+
+        * Generate tasks.
+        * Annotate data to tasks.
+        """
+
+        # entrypoint = self.entrypoint(self.run)
+        client = self.run.client
+        # project_id = self.params['project']
+        # project = client.get_project(project_id)
+        # data_collection_id = project['dataset']
+        # data_collection = client.get_dataset(data_collection_id)
+
+        # Generate tasks if provided project is empty.
+        task_ids_query_params = {
+            'project': self.params['project'],
+            'fields': 'id',
+        }
+        if self.params.get('task_filter'):
+            task_ids_query_params.update(self.params['task_filter'])
+        task_ids_generator, task_ids_count = client.list_tasks(params=task_ids_query_params, list_all=True)
+        task_ids = [item['id'] for item in task_ids_generator]
+
+        # If no tasks found, break the job.
+        if not task_ids_count:
+            self.run.log_message('Tasks to annotate not found.')
+            self.run.end_log()
+
+        # Annotate data to tasks.
+        task_data_annotation_type = self.params['task_data_annotation_type']
+        if task_data_annotation_type == AnnotationMethod.FILE:
+            self._handle_annotate_data_from_files(task_ids)
+        elif task_data_annotation_type == AnnotationMethod.INFERENCE:
+            self._handle_annotate_data_with_inference(task_ids)
+
+        return {}
+
+    def _handle_annotate_data_from_files(self, task_ids: list[int]):
+        """Handle annotate data from files to tasks.
+
+        Args:
+            task_ids (list[int]): List of task IDs to annotate data to.
+        """
+        client = self.run.client
+        if not (target_task_data_specification_code := self.params.get('target_task_data_specification_code')):
+            self.run.log_message('Target task data specification code not found.')
+            self.run.end_log()
+        task_params = {
+            'fields': 'id,data,data_unit',
+            'expand': 'data_unit',
+        }
+        for task_id in task_ids:
+            task = client.get_task(task_id, params=task_params)
+            data_file = task['data_unit']['files'].get(target_task_data_specification_code)
+            if not data_file:
+                self.run.log_message(f'File specification not found for task {task_id}')
+                self.run.log_annotate_task_data(
+                    {'task_id': task_id, 'error': 'File specification not found'}, AnnotateTaskDataStatus.FAILED
+                )
+                continue
+            url = data_file.get('url')
+            if not url:
+                self.run.log_message(f'URL not found for task {task_id}')
+                self.run.log_annotate_task_data(
+                    {'task_id': task_id, 'error': 'URL not found'}, AnnotateTaskDataStatus.FAILED
+                )
+                continue
+
+            try:
+                response = requests.get(url)
+                response.raise_for_status()  # Raise an exception for 4XX/5XX responses
+                data = json.loads(response.content)
+                client.patch_task(task_id, data={'data': data})
+
+                # Log success
+                self.run.log_annotate_task_data({'task_id': task_id, 'url': url}, AnnotateTaskDataStatus.SUCCESS)
+            except Exception as e:
+                self.run.log_message(f'Failed to get content from URL for task {task_id}: {str(e)}')
+                self.run.log_annotate_task_data(
+                    {'task_id': task_id, 'url': url, 'error': str(e)}, AnnotateTaskDataStatus.FAILED
+                )
+                continue
+
+    def _handle_annotate_data_with_inference(self, task_ids: list[int]):
+        """Handle annotate data with inference to tasks.
+
+        Args:
+            task_ids (list[int]): List of task IDs to annotate data to.
+        """
+        self.run.log_message('Pre annotation with inference is not supported.')
+        self.run.end_log()

synapse_sdk-1.0.0a59/synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml

@@ -0,0 +1,7 @@
+actions:
+  pre_annotation:
+    entrypoint: plugin.pre_annotation.pre_annotate
+  to_task:
+    entrypoint: plugin.to_task.AnnotationToTask
+    ui_schema: |
+      Dumped FormKit Schema for pre_annotation plugin to_task action's custom options

synapse_sdk-1.0.0a57/synapse_sdk/plugins/categories/upload/templates/plugin/task_pre_annotation.py → synapse_sdk-1.0.0a59/synapse_sdk/plugins/categories/pre_annotation/templates/plugin/to_task.py

@@ -1,4 +1,4 @@
-class TaskPreAnnotation:
+class AnnotationToTask:
     def __init__(self, run, *args, **kwargs):
         """Initialize the plugin task pre annotation action class.
 
@@ -6,9 +6,3 @@ class TaskPreAnnotation:
             run: Plugin run object.
         """
         self.run = run
-
-    def handle_annotate_data_from_files(self):
-        pass
-
-    def handle_annotate_data_with_inference(self):
-        pass

{synapse_sdk-1.0.0a57 → synapse_sdk-1.0.0a59}/synapse_sdk/plugins/categories/upload/actions/upload.py

@@ -1,6 +1,7 @@
 import json
 from datetime import datetime
 from enum import Enum
+from pathlib import Path
 from typing import Annotated, Dict, List
 
 from pydantic import AfterValidator, BaseModel, field_validator
@@ -8,6 +9,7 @@ from pydantic_core import PydanticCustomError
 
 from synapse_sdk.clients.exceptions import ClientError
 from synapse_sdk.clients.utils import get_batched_list
+from synapse_sdk.clients.validators.collections import FileSpecificationValidator
 from synapse_sdk.i18n import gettext as _
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
@@ -45,6 +47,13 @@ class UploadRun(Run):
         status: UploadStatus
         created: str
 
+    class MetricsRecord(BaseModel):
+        """Metrics record model."""
+
+        stand_by: int
+        failed: int
+        success: int
+
     def log_data_file(self, data_file_info: dict, status: UploadStatus):
         """Upload data_file log.
 
@@ -82,6 +91,15 @@ class UploadRun(Run):
         now = datetime.now().isoformat()
         self.log('upload_task', self.TaskLog(task_id=task_id, status=status.value, created=now).model_dump())
 
+    def log_metrics(self, record: MetricsRecord, category: str):
+        """Log upload metrics.
+        Args:
+            record (MetricsRecord): The metrics record to log.
+            category (str): The category of the metrics.
+        """
+        record = self.MetricsRecord.model_validate(record)
+        self.set_metrics(value=record.model_dump(), category=category)
+
 
 class UploadParams(BaseModel):
     """Upload action parameters.
@@ -94,8 +112,6 @@ class UploadParams(BaseModel):
         storage (int): The storage of the action.
         collection (int): The collection of the action.
         project (int | None): The project of the action.
-        is_generate_tasks (bool): The flag to generate tasks.
-        is_generate_ground_truths (bool): The flag to generate ground truths
     """
 
     name: Annotated[str, AfterValidator(non_blank)]
@@ -104,6 +120,8 @@ class UploadParams(BaseModel):
     storage: int
     collection: int
     project: int | None
+    is_generate_tasks: bool = False
+    is_generate_ground_truths: bool = False
 
     @field_validator('storage', mode='before')
     @classmethod
@@ -162,6 +180,12 @@ class UploadAction(Action):
         analyze_collection: The progress category for the analyze collection process.
         data_file_upload: The progress category for the upload process.
         generate_data_units: The progress category for the generate data units process.
+        generate_tasks: The progress category for the generate tasks process.
+        generate_ground_truths: The progress category for the generate ground truths process.
+
+    Metrics Categories:
+        data_file: The metrics category for the data file.
+        data_unit: The metrics category for the data unit.
     """
 
     name = 'upload'
@@ -170,15 +194,16 @@ class UploadAction(Action):
     run_class = UploadRun
     progress_categories = {
         'analyze_collection': {
-            'proportion': 10,
+            'proportion': 0,
         },
         'upload_data_files': {
-            'proportion': 50,
+            'proportion': 0,
         },
         'generate_data_units': {
-            'proportion': 40,
+            'proportion': 0,
         },
     }
+    metrics_categories = {'data_file', 'data_unit'}
 
     def get_uploader(self, path, file_specification, organized_files):
         """Get uploader from entrypoint."""
@@ -232,6 +257,12 @@ class UploadAction(Action):
         generated_data_units = self._generate_data_units(uploaded_files, upload_result_count)
         result['generated_data_units_count'] = len(generated_data_units)
 
+        # Setup task with uploaded synapse-backend data units.
+        if not len(generated_data_units):
+            self.run.log_message('No data units were generated.', context=Context.WARNING.value)
+            self.run.end_log()
+            return result
+
         self.run.end_log()
         return result
 
@@ -263,6 +294,7 @@ class UploadAction(Action):
         # Initialize progress
         self.run.set_progress(0, organized_files_count, category='upload_data_files')
         self.run.log_message('Uploading data files...')
+        data_file_metrics_record = self.run.MetricsRecord(stand_by=organized_files_count, success=0, failed=0)
 
         client = self.run.client
         collection_id = self.params['data_collection']
@@ -272,6 +304,9 @@ class UploadAction(Action):
         for organized_file in organized_files:
             uploaded_data_file = client.upload_data_file(organized_file, collection_id)
             self.run.log_data_file(organized_file, UploadStatus.SUCCESS)
+            data_file_metrics_record.stand_by -= 1
+            data_file_metrics_record.success += 1
+            self.run.log_metrics(record=data_file_metrics_record, category='data_file')
             upload_result.append(uploaded_data_file)
             self.run.set_progress(current_progress, organized_files_count, category='upload_data_files')
             current_progress += 1
@@ -292,6 +327,7 @@ class UploadAction(Action):
         """
         # Initialize progress
         self.run.set_progress(0, upload_result_count, category='generate_data_units')
+        data_unit_metrics_record = self.run.MetricsRecord(stand_by=upload_result_count, success=0, failed=0)
 
         client = self.run.client
 
@@ -301,6 +337,9 @@ class UploadAction(Action):
         batches_count = len(batches)
         for batch in batches:
             created_data_units = client.create_data_units(batch)
+            data_unit_metrics_record.stand_by -= len(created_data_units)
+            data_unit_metrics_record.success += len(created_data_units)
+            self.run.log_metrics(record=data_unit_metrics_record, category='data_unit')
             generated_data_units.append(created_data_units)
             self.run.set_progress(current_progress, batches_count, category='generate_data_units')
             current_progress += 1
@@ -311,3 +350,120 @@ class UploadAction(Action):
         self.run.set_progress(upload_result_count, upload_result_count, category='generate_data_units')
 
         return sum(generated_data_units, [])
+
+    def _validate_organized_files(self, organized_files: List, file_specification_template: Dict) -> bool:
+        """Validate organized files from Uploader."""
+        validator = FileSpecificationValidator(file_specification_template, organized_files)
+        return validator.validate()
+
+    def _organize_files(self, directory: Path, file_specification: List) -> List:
+        """Organize files according to the file specification.
+        This method handles type-based directory structure where files are organized in
+        directories named after file types (e.g., 'image_1/' directory contains image files
+        like '1.jpg', '2.jpg'). For each dataset ID found in the primary directory, it attempts
+        to find corresponding files in all type directories.
+
+        TODO : Add Logic to handle file specific name patterns and extensions.
+        (e.g. pcd:S_DCH_230725_0156_LR_037.pcd, image_1:S_DCH_230725_0156_FC_037, image_2:S_DCH_230725_0156_LF_037.jpg)
+        Args:
+            directory (Path): Root directory containing files to organize.
+            file_specification (List): File specification list.
+        Returns:
+            List: List of dictionaries containing organized files.
+        """
+        organized_files = []
+        self.run.log_message(f'Looking for files in {directory}...')
+
+        # Check for type-based directory structure (e.g., image_1/, pcd_1/)
+        type_dirs = {}
+        type_extensions = {}  # Store common extensions for each type directory
+
+        for spec in file_specification:
+            spec_name = spec['name']
+
+            spec_dir = directory / spec_name
+            if spec_dir.exists() and spec_dir.is_dir():
+                type_dirs[spec_name] = spec_dir
+
+                # Analyze file extensions in this directory
+                extensions = {}
+                for file_path in spec_dir.glob('*'):
+                    if file_path.is_file():
+                        ext = file_path.suffix.lower()
+                        extensions[ext] = extensions.get(ext, 0) + 1
+
+                # Find the most common extension
+                if extensions:
+                    common_ext = max(extensions.items(), key=lambda x: x[1])[0]
+                    type_extensions[spec_name] = common_ext
+                    self.run.log_message(f'Found type directory: {spec_name} (common extension: {common_ext})')
+
+        # If type-based directories don't exist, exit early
+        if not type_dirs:
+            self.run.log_message('No type-based directory structure found.', context=Context.INFO.value)
+            return organized_files
+
+        self.run.log_message('Detected type-based directory structure')
+
+        # Build a comprehensive map of all dataset IDs across all type directories
+        dataset_files = {}  # Dictionary: file_name -> {spec_name -> file_path}
+
+        # First pass: collect all dataset IDs from all type directories
+        for spec_name, dir_path in type_dirs.items():
+            for file_path in dir_path.glob('*'):
+                if file_path.is_file():
+                    file_name = file_path.stem
+
+                    # Initialize dataset entry if it doesn't exist
+                    if file_name not in dataset_files:
+                        dataset_files[file_name] = {}
+
+                    # Map this file to its specification
+                    if spec_name not in dataset_files[file_name]:
+                        dataset_files[file_name][spec_name] = file_path
+                    else:
+                        # If multiple files with same file_name for same spec, use most recent
+                        existing_file = dataset_files[file_name][spec_name]
+                        if file_path.stat().st_mtime > existing_file.stat().st_mtime:
+                            dataset_files[file_name][spec_name] = file_path
+                            self.run.log_message(
+                                f"Found newer file for name of {file_name}, spec '{spec_name}': "
+                                f'{file_path.name} (replacing {existing_file.name})'
+                            )
+
+        if not dataset_files:
+            self.run.log_message('No dataset files found.', context=Context.WARNING.value)
+            return organized_files
+
+        self.run.log_message(f'Found {len(dataset_files)} potential datasets by ID')
+
+        # Second pass: organize valid datasets
+        for file_name, files_dict in sorted(dataset_files.items()):
+            self.run.log_message(f'Processing file name: {file_name}')
+
+            # Add file spec details for logging
+            for spec_name, file_path in files_dict.items():
+                self.run.log_message(f"Mapped '{spec_name}' to: {file_path.name}")
+
+            # Check if all required files are present
+            required_specs = [spec['name'] for spec in file_specification if spec.get('is_required', False)]
+            if all(req in files_dict for req in required_specs):
+                # Create metadata for this dataset
+                meta_data = {
+                    'origin_file_stem': file_name,
+                    'created_at': datetime.now().isoformat(),
+                }
+
+                # Add the organized dataset
+                organized_files.append({'files': files_dict, 'meta': meta_data})
+                self.run.log_message(f'Successfully organized dataset for ID {file_name}')
+            else:
+                # Missing required files warning
+                missing = [req for req in required_specs if req not in files_dict]
+                self.run.log_message(
+                    f'Dataset ID {file_name} is missing required files: {", ".join(missing)}',
+                    context=Context.WARNING.value,
+                )
+
+        self.run.log_message(f'Total datasets organized: {len(organized_files)}')
+        return organized_files

{synapse_sdk-1.0.0a57 → synapse_sdk-1.0.0a59}/synapse_sdk/plugins/categories/upload/templates/config.yaml

@@ -1,10 +1,9 @@
 actions:
   upload:
     entrypoint: plugin.upload.Uploader
+    options:
+      allow_generate_tasks: false # Allow the plugin to generate tasks for the uploaded data
+      allow_generate_ground_truths: false # Allow the plugin to generate ground truths for the uploaded data
     supported_data_type: image # A primary data type of synapse backend collection. (e.g. 'image', 'text', 'video', 'pcd', 'audio')
     ui_schema: |
       Dumped FormKit Schema for upload plugin custom options
-  task_pre_annotation:
-    entrypoint: plugin.upload.TaskPreAnnotation
-    ui_schema: |
-      Dumped FormKit Schema for upload plugin custom options

{synapse_sdk-1.0.0a57 → synapse_sdk-1.0.0a59}/synapse_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a57
+Version: 1.0.0a59
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a57 → synapse_sdk-1.0.0a59}/synapse_sdk.egg-info/SOURCES.txt

@@ -106,9 +106,11 @@ synapse_sdk/plugins/categories/post_annotation/templates/plugin/post_annotation.
 synapse_sdk/plugins/categories/pre_annotation/__init__.py
 synapse_sdk/plugins/categories/pre_annotation/actions/__init__.py
 synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation.py
+synapse_sdk/plugins/categories/pre_annotation/actions/to_task.py
 synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml
 synapse_sdk/plugins/categories/pre_annotation/templates/plugin/__init__.py
 synapse_sdk/plugins/categories/pre_annotation/templates/plugin/pre_annotation.py
+synapse_sdk/plugins/categories/pre_annotation/templates/plugin/to_task.py
 synapse_sdk/plugins/categories/smart_tool/__init__.py
 synapse_sdk/plugins/categories/smart_tool/actions/__init__.py
 synapse_sdk/plugins/categories/smart_tool/actions/auto_label.py
@@ -117,11 +119,9 @@ synapse_sdk/plugins/categories/smart_tool/templates/plugin/__init__.py
 synapse_sdk/plugins/categories/smart_tool/templates/plugin/auto_label.py
 synapse_sdk/plugins/categories/upload/__init__.py
 synapse_sdk/plugins/categories/upload/actions/__init__.py
-synapse_sdk/plugins/categories/upload/actions/task_pre_annotation.py
 synapse_sdk/plugins/categories/upload/actions/upload.py
 synapse_sdk/plugins/categories/upload/templates/config.yaml
 synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py
-synapse_sdk/plugins/categories/upload/templates/plugin/task_pre_annotation.py
 synapse_sdk/plugins/categories/upload/templates/plugin/upload.py
 synapse_sdk/plugins/templates/cookiecutter.json
 synapse_sdk/plugins/templates/hooks/post_gen_project.py

synapse_sdk-1.0.0a57/synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml

@@ -1,3 +0,0 @@
-actions:
-  pre_annotation:
-    entrypoint: plugin.pre_annotation.pre_annotate