synapse-sdk 1.0.0a52__py3-none-any.whl → 1.0.0a54__py3-none-any.whl

synapse_sdk/clients/ray/serve.py

@@ -8,6 +8,7 @@ class ServeClientMixin(BaseClient):
         response = self._get(path, params=params)
         for key, item in response['applications'].items():
             response['applications'][key]['deployments'] = list(item['deployments'].values())
+            response['applications'][key]['route_prefix'] = item['route_prefix']
         return list(response['applications'].values())
 
     def get_serve_application(self, pk, params=None):
@@ -15,6 +16,7 @@ class ServeClientMixin(BaseClient):
         response = self._get(path, params=params)
         try:
             response['applications'][pk]['deployments'] = list(response['applications'][pk]['deployments'].values())
+            response['applications'][pk]['route_prefix'] = response['applications'][pk]['route_prefix']
             return response['applications'][pk]
         except KeyError:
             raise ClientError(404, 'Serve Application Not Found')

synapse_sdk/plugins/categories/export/actions/export.py

@@ -27,13 +27,6 @@ class ExportRun(Run):
         error: str | None = None
         created: str
 
-    class MetricsRecord(BaseModel):
-        """Metrics record model."""
-
-        stand_by: int
-        failed: int
-        success: int
-
     def log_file(
         self, log_type: str, target_id: int, data_file_info: dict, status: ExportStatus, error: str | None = None
     ):
@@ -58,16 +51,6 @@ class ExportRun(Run):
             ).model_dump(),
         )
 
-    def log_metrics(self, record: MetricsRecord, category: str):
-        """Log export metrics.
-
-        Args:
-            record (MetricsRecord): The metrics record to log.
-            category (str): The category of the metrics.
-        """
-        record = self.MetricsRecord.model_validate(record)
-        self.set_metrics(value=record.dict(), category=category)
-
     def export_log_json_file(
         self,
         target_id: int,
@@ -280,7 +263,6 @@ class ExportAction(Action):
             'proportion': 100,
         }
     }
-    metrics_categories = {'data_file', 'original_file'}
 
     def get_filtered_results(self, filters, handler):
         """Get filtered target results."""

synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -42,8 +42,6 @@ def export(run, export_items, path_root, **params):
     origin_files_output_path.mkdir(parents=True, exist_ok=True)
 
     total = params['count']
-    original_file_metrics_record = run.MetricsRecord(stand_by=total, success=0, failed=0)
-    data_file_metrics_record = run.MetricsRecord(stand_by=total, success=0, failed=0)
     # progress init
     run.set_progress(0, total, category='dataset_conversion')
     for no, export_item in enumerate(export_items, start=1):
@@ -58,30 +56,12 @@ def export(run, export_items, path_root, **params):
         if save_original_file_flag:
             if no == 1:
                 run.log_message('Saving original file.')
-            original_status = save_original_file(run, final_data, origin_files_output_path, errors_original_file_list)
-
-            original_file_metrics_record.stand_by -= 1
-            if original_status == ExportStatus.FAILED:
-                original_file_metrics_record.failed += 1
-                continue
-            else:
-                original_file_metrics_record.success += 1
-
-        run.log_metrics(record=original_file_metrics_record, category='original_file')
+            save_original_file(run, final_data, origin_files_output_path, errors_original_file_list)
 
         # Extract data as JSON files
         if no == 1:
             run.log_message('Saving json file.')
-        data_status = save_as_json(run, final_data, json_output_path, errors_json_file_list)
-
-        data_file_metrics_record.stand_by -= 1
-        if data_status == ExportStatus.FAILED:
-            data_file_metrics_record.failed += 1
-            continue
-        else:
-            data_file_metrics_record.success += 1
-
-        run.log_metrics(record=data_file_metrics_record, category='data_file')
+        save_as_json(run, final_data, json_output_path, errors_json_file_list)
 
     run.end_log()
 
@@ -146,7 +126,6 @@ def save_original_file(run, result, base_path, error_file_list):
         status = ExportStatus.FAILED
 
     run.export_log_original_file(result['id'], file_info, status, error_msg)
-    return status
 
 
 def save_as_json(run, result, base_path, error_file_list):
@@ -173,4 +152,3 @@ def save_as_json(run, result, base_path, error_file_list):
         status = ExportStatus.FAILED
 
     run.export_log_json_file(result['id'], file_info, status, error_msg)
-    return status

synapse_sdk/plugins/categories/upload/actions/upload.py

@@ -105,6 +105,8 @@ class UploadParams(BaseModel):
     storage: int
     collection: int
     project: int | None
+    is_generate_tasks: bool = False
+    is_generate_ground_truths: bool = False
 
     @field_validator('storage', mode='before')
     @classmethod
@@ -163,6 +165,8 @@ class UploadAction(Action):
         analyze_collection: The progress category for the analyze collection process.
         data_file_upload: The progress category for the upload process.
         generate_data_units: The progress category for the generate data units process.
+        generate_tasks: The progress category for the generate tasks process.
+        generate_ground_truths: The progress category for the generate ground truths process.
     """
 
     name = 'upload'
@@ -171,16 +175,48 @@ class UploadAction(Action):
     run_class = UploadRun
     progress_categories = {
         'analyze_collection': {
-            'proportion': 10,
+            'proportion': 0,
         },
         'upload_data_files': {
-            'proportion': 50,
+            'proportion': 0,
         },
         'generate_data_units': {
-            'proportion': 40,
+            'proportion': 0,
+        },
+        'generate_tasks': {
+            'proportion': 0,
+        },
+        'generate_ground_truths': {
+            'proportion': 0,
         },
     }
 
+    def __init__(self, *args, **kwargs):
+        """Initialize UploadAction."""
+        super().__init__(*args, **kwargs)
+
+        # Setup progress categories ratio by options.
+        progress_ratios = {
+            'upload_only': (5, 60, 35, 0, 0),
+            'generate_tasks': (5, 45, 25, 25, 0),
+            'generate_ground_truths': (5, 35, 30, 15, 15),
+        }
+        options = kwargs['plugin_config']['actions']['upload']['options']
+        progress_categories = self.progress_categories
+        if options['allow_generate_tasks'] and not kwargs['params']['allow_generate_ground_truths']:
+            ratio_name = 'generate_tasks'
+        elif options['allow_generate_ground_truths'] and kwargs['params']['allow_generate_tasks']:
+            ratio_name = 'generate_ground_truths'
+        else:
+            ratio_name = 'upload_only'
+
+        assert len(progress_categories) == len(progress_ratios[ratio_name]), (
+            'Progress categories and ratios length mismatch.'
+        )
+        for i, category in enumerate(progress_categories):
+            progress_categories[category]['proportion'] = progress_ratios[ratio_name][i]
+        self.progress_categories = progress_categories
+
     def get_uploader(self, path):
         """Get uploader from entrypoint."""
         return self.entrypoint(self.run, path)
@@ -229,6 +265,26 @@ class UploadAction(Action):
         generated_data_units = self._generate_data_units(uploaded_files, upload_result_count)
         result['generated_data_units_count'] = len(generated_data_units)
 
+        # Setup task with uploaded synapse-backend data units.
+        if not len(generated_data_units):
+            self.run.log_message('No data units were generated.', context=Context.WARNING.value)
+            self.run.end_log()
+            return result
+
+        if self.config['options']['allow_generate_tasks'] and self.params['is_generate_tasks']:
+            generated_tasks = self._generate_tasks(generated_data_units)
+            result['generated_tasks_count'] = len(generated_tasks)
+        else:
+            self.run.log_message('Generating tasks process has passed.')
+
+        # Generate ground truths for the uploaded data.
+        # TODO: Need to add ground truths generation logic later.
+        if self.config['options']['allow_generate_ground_truths'] and self.params['is_generate_ground_truths']:
+            generated_ground_truths = self._generate_ground_truths()
+            result['generated_ground_truths_count'] = len(generated_ground_truths)
+        else:
+            self.run.log_message('Generating ground truths process has passed.')
+
         self.run.end_log()
         return result
 
@@ -313,3 +369,53 @@ class UploadAction(Action):
         self.run.set_progress(upload_result_count, upload_result_count, category='generate_data_units')
 
         return sum(generated_data_units, [])
+
+    def _generate_tasks(self, generated_data_units: List) -> List:
+        """Setup task with uploaded synapse-backend data units.
+
+        TODO: make batch size configurable.
+        """
+        # Initialize progress
+        self.run.set_progress(0, 1, category='generate_tasks')
+        self.run.log_message('Generating tasks with data files...')
+
+        # Prepare batches for processing
+        client = self.run.client
+        project_id = self.params['project']
+        current_progress = 0
+
+        # Generate tasks
+        generated_tasks = []
+        generated_data_units_count = len(generated_data_units)
+        for data_unit in generated_data_units:
+            tasks_data = []
+            task_data = {'project': project_id, 'data_unit': data_unit['id']}
+            tasks_data.append(task_data)
+            if tasks_data:
+                created_tasks = client.create_tasks(tasks_data)
+                created_task_ids = [created_task['id'] for created_task in created_tasks]
+                generated_tasks.append(created_task_ids)
+                for created_task_id in created_task_ids:
+                    self.run.log_task(created_task_id, UploadStatus.SUCCESS)
+
+            self.run.set_progress(current_progress, generated_data_units_count, category='generate_tasks')
+            current_progress += 1
+
+        # Finish progress
+        self.run.log_message('Generating tasks completed')
+        self.run.set_progress(1, 1, category='generate_tasks')
+
+        return sum(generated_tasks, [])
+
+    def _generate_ground_truths(self):
+        """Generate ground truths for the uploaded data.
+
+        TODO: Need to add ground truths generation logic later.
+        """
+        # Initialize progress
+        self.run.set_progress(0, 1, category='generate_ground_truths')
+        self.run.log_message('Generating ground truths...')
+
+        # Finish progress
+        self.run.log_message('Generating ground truths completed')
+        self.run.set_progress(1, 1, category='generate_ground_truths')

synapse_sdk/plugins/categories/upload/templates/config.yaml

@@ -1,10 +1,9 @@
 actions:
   upload:
     entrypoint: plugin.upload.Uploader
+    options:
+      allow_generate_tasks: false # Allow the plugin to generate tasks for the uploaded data
+      allow_generate_ground_truths: false # Allow the plugin to generate ground truths for the uploaded data
     supported_data_type: image # A primary data type of synapse backend collection. (e.g. 'image', 'text', 'video', 'pcd', 'audio')
     ui_schema: |
       Dumped FormKit Schema for upload plugin custom options
-  task_pre_annotation:
-    entrypoint: plugin.upload.TaskPreAnnotation
-    ui_schema: |
-      Dumped FormKit Schema for upload plugin custom options

{synapse_sdk-1.0.0a52.dist-info → synapse_sdk-1.0.0a54.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a52
+Version: 1.0.0a54
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a52.dist-info → synapse_sdk-1.0.0a54.dist-info}/RECORD

@@ -38,7 +38,7 @@ synapse_sdk/clients/backend/ml.py,sha256=JoPH9Ly2E3HJ7S5mdGLtcGq7ruQVVrYfWArogwZ
 synapse_sdk/clients/backend/models.py,sha256=s5d9sGGQ0Elj0HOGC1TuwE-eBkY1aTfJPl6ls11bNCk,1961
 synapse_sdk/clients/ray/__init__.py,sha256=9ZSPXVVxlJ8Wp8ku7l021ENtPjVrGgQDgqifkkVAXgM,187
 synapse_sdk/clients/ray/core.py,sha256=a4wyCocAma2HAm-BHlbZnoVbpfdR-Aad2FM0z6vPFvw,731
-synapse_sdk/clients/ray/serve.py,sha256=rbCpXZYWf0oP8XJ9faa9QFNPYU7h8dltIG8xn9ZconY,907
+synapse_sdk/clients/ray/serve.py,sha256=eFhCYIv_irc_2RyuV3bzeWIVyz_1NlqwoNVh5KSWilY,1092
 synapse_sdk/clients/validators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/clients/validators/collections.py,sha256=LtnwvutsScubOUcZ2reGHLCzseXxtNIdnH2nv098aUU,1195
 synapse_sdk/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -61,10 +61,10 @@ synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py,sh
 synapse_sdk/plugins/categories/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/export/enums.py,sha256=gtyngvQ1DKkos9iKGcbecwTVQQ6sDwbrBPSGPNb5Am0,127
 synapse_sdk/plugins/categories/export/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/export/actions/export.py,sha256=2lIjur8EiwTB9sc16FV8ZaPXFxUtGRPx9hreG_DKLQA,10483
+synapse_sdk/plugins/categories/export/actions/export.py,sha256=xqPB_MufeMP3riaKCbGVFGukV8RdXcg6-zUrkw4t1-A,9922
 synapse_sdk/plugins/categories/export/templates/config.yaml,sha256=N7YmnFROb3s3M35SA9nmabyzoSb5O2t2TRPicwFNN2o,56
 synapse_sdk/plugins/categories/export/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/export/templates/plugin/export.py,sha256=zG8mSn7ZGIj8cttWmb7GEPcGgQRbZ97brJCzkuK7RP8,6106
+synapse_sdk/plugins/categories/export/templates/plugin/export.py,sha256=JA2Y_A30QyJekSqDq8PeRuFR9k0yjQjOG-Xy6C8zPew,5196
 synapse_sdk/plugins/categories/neural_net/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/actions/deployment.py,sha256=y2LrS-pwazqRI5O0q1NUy45NQYsBj6ykbrXnDMs_fqE,1987
@@ -100,11 +100,9 @@ synapse_sdk/plugins/categories/smart_tool/templates/plugin/__init__.py,sha256=47
 synapse_sdk/plugins/categories/smart_tool/templates/plugin/auto_label.py,sha256=eevNg0nOcYFR4z_L_R-sCvVOYoLWSAH1jwDkAf3YCjY,320
 synapse_sdk/plugins/categories/upload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/upload/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/upload/actions/task_pre_annotation.py,sha256=SehWN55Turhyv_GEBpetykvB4-mZnoDL0h5XlWszPe8,3058
-synapse_sdk/plugins/categories/upload/actions/upload.py,sha256=3qjuvH28BfMdBK2bOTo-GlqoF24eKPX10hFhha0-GEk,11278
-synapse_sdk/plugins/categories/upload/templates/config.yaml,sha256=1O0kMfkFMGYwnpBcttrlC9bu4xzU9docw2MBOq_Elmo,417
+synapse_sdk/plugins/categories/upload/actions/upload.py,sha256=8FIRkEtSPTToSZBYn7B2qaTj9778RDjr84BHnn6ajSM,15951
+synapse_sdk/plugins/categories/upload/templates/config.yaml,sha256=kwHNWHFYbzDi1mEh40KozatPZbZGH44dlP0t0J7ejJw,483
 synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/upload/templates/plugin/task_pre_annotation.py,sha256=9XkUZu7USjVjDPufM0NlYmkdKfV7Hf_9v5GN1RgZzS0,350
 synapse_sdk/plugins/categories/upload/templates/plugin/upload.py,sha256=dnK8gy33GjG5ettayawDJv1gM3xCm1K6lM-PfeeTjQw,1163
 synapse_sdk/plugins/templates/cookiecutter.json,sha256=NxOWk9A_v1pO0Ny4IYT9Cj5iiJ16--cIQrGC67QdR0I,396
 synapse_sdk/plugins/templates/hooks/post_gen_project.py,sha256=jqlYkY1O2TxIR-Vh3gnwILYy8k-D39Xx66d2KNQVMCs,147
@@ -136,9 +134,9 @@ synapse_sdk/utils/storage/providers/__init__.py,sha256=x7RGwZryT2FpVxS7fGWryRVpq
 synapse_sdk/utils/storage/providers/gcp.py,sha256=i2BQCu1Kej1If9SuNr2_lEyTcr5M_ncGITZrL0u5wEA,363
 synapse_sdk/utils/storage/providers/s3.py,sha256=W94rQvhGRXti3R4mYP7gmU5pcyCQpGFIBLvxxqLVdRM,2231
 synapse_sdk/utils/storage/providers/sftp.py,sha256=_8s9hf0JXIO21gvm-JVS00FbLsbtvly4c-ETLRax68A,1426
-synapse_sdk-1.0.0a52.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0a52.dist-info/METADATA,sha256=HXeyrqhs7vO8PtUOUf4hy7XREJYywB-Mq55s_Oa4CjQ,1303
-synapse_sdk-1.0.0a52.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-synapse_sdk-1.0.0a52.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-1.0.0a52.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0a52.dist-info/RECORD,,
+synapse_sdk-1.0.0a54.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0a54.dist-info/METADATA,sha256=JHOOafiDzcZBZjHOAL8AUzlEjhWs1YyLz6VH4jCGK4I,1303
+synapse_sdk-1.0.0a54.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+synapse_sdk-1.0.0a54.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-1.0.0a54.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0a54.dist-info/RECORD,,

{synapse_sdk-1.0.0a52.dist-info → synapse_sdk-1.0.0a54.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

synapse_sdk/plugins/categories/upload/actions/task_pre_annotation.py

@@ -1,101 +0,0 @@
-from enum import Enum
-from typing import Annotated
-
-from pydantic import AfterValidator, BaseModel, field_validator
-from pydantic_core import PydanticCustomError
-
-from synapse_sdk.clients.exceptions import ClientError
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import PluginCategory, RunMethod
-from synapse_sdk.plugins.models import Run
-from synapse_sdk.utils.pydantic.validators import non_blank
-
-
-class TaskDataAnnotationType(str, Enum):
-    FILE = 'file'
-    INFERENCE = 'inference'
-
-
-class TaskPreAnnotationRun(Run):
-    pass
-
-
-class TaskPreAnnotationParams(BaseModel):
-    """TaskPreAnnotation action parameters.
-
-    Args:
-        name (str): The name of the action.
-        description (str | None): The description of the action.
-        project (int): The project ID.
-        data_collection (int): The data collection ID.
-        task_data_annotation_type (TaskDataAnnotationType): The type of task data annotation.
-    """
-
-    name: Annotated[str, AfterValidator(non_blank)]
-    description: str | None
-    project: int
-    data_collection: int
-    task_data_annotation_type: TaskDataAnnotationType
-
-    @field_validator('data_collection', mode='before')
-    @classmethod
-    def check_data_collection_exists(cls, value: str, info) -> str:
-        """Validate synapse-backend collection exists."""
-        action = info.context['action']
-        client = action.client
-        try:
-            client.get_dataset(value)
-        except ClientError:
-            raise PydanticCustomError('client_error', 'Error occurred while checking data collection exists.')
-        return value
-
-    @field_validator('project', mode='before')
-    @classmethod
-    def check_project_exists(cls, value: str, info) -> str:
-        """Validate synapse-backend project exists."""
-        if not value:
-            return value
-
-        action = info.context['action']
-        client = action.client
-        try:
-            client.get_project(value)
-        except ClientError:
-            raise PydanticCustomError('client_error', 'Error occurred while checking project exists.')
-        return value
-
-
-@register_action
-class TaskPreAnnotationAction(Action):
-    """TaskPreAnnotation action class.
-
-    * Annotate data to tasks.
-    """
-
-    name = 'task_pre_annotation'
-    category = PluginCategory.UPLOAD
-    method = RunMethod.JOB
-    run_class = TaskPreAnnotationRun
-    progress_categories = {
-        'generate_tasks': {
-            'proportion': 10,
-        },
-        'annotate_task_data': {
-            'proportion': 90,
-        },
-    }
-
-    def start(self):
-        """Start task_pre_annotation action.
-
-        * Generate tasks.
-        * Annotate data to tasks.
-        """
-        task_pre_annotation = self.get_task_pre_annotation()
-        task_pre_annotation.handle_annotate_data_from_files()
-        return {}
-
-    def get_task_pre_annotation(self):
-        """Get task pre annotation entrypoint."""
-        return self.entrypoint()

synapse_sdk/plugins/categories/upload/templates/plugin/task_pre_annotation.py

@@ -1,14 +0,0 @@
-class TaskPreAnnotation:
-    def __init__(self, run, *args, **kwargs):
-        """Initialize the plugin task pre annotation action class.
-
-        Args:
-            run: Plugin run object.
-        """
-        self.run = run
-
-    def handle_annotate_data_from_files(self):
-        pass
-
-    def handle_annotate_data_with_inference(self):
-        pass