synapse-sdk 1.0.0a58__py3-none-any.whl → 1.0.0a59__py3-none-any.whl

synapse_sdk/clients/backend/annotation.py

@@ -7,6 +7,14 @@ class AnnotationClientMixin(BaseClient):
         path = f'projects/{pk}/'
         return self._get(path)
 
+    def get_task(self, pk, params):
+        path = f'tasks/{pk}/'
+        return self._get(path, params=params)
+
+    def patch_task(self, pk, data):
+        path = f'tasks/{pk}/'
+        return self._patch(path, data=data)
+
     def get_task_tag(self, pk):
         path = f'task_tags/{pk}/'
         return self._get(path)

synapse_sdk/clients/backend/data_collection.py

@@ -5,7 +5,7 @@ from typing import Dict, Optional
 from tqdm import tqdm
 
 from synapse_sdk.clients.base import BaseClient
-from synapse_sdk.clients.utils import get_batched_list
+from synapse_sdk.clients.utils import get_batched_list, get_default_url_conversion
 
 
 class DataCollectionClientMixin(BaseClient):
@@ -31,6 +31,10 @@ class DataCollectionClientMixin(BaseClient):
         path = 'data_files/'
         return self._post(path, files={'file': file_path})
 
+    def get_data_unit(self, data_unit_id: int, params=None):
+        path = f'data_units/{data_unit_id}/'
+        return self._get(path, params=params)
+
     def create_data_units(self, data):
         """Create data units to synapse-backend.
 
@@ -40,6 +44,11 @@ class DataCollectionClientMixin(BaseClient):
         path = 'data_units/'
         return self._post(path, data=data)
 
+    def list_data_units(self, params=None, url_conversion=None, list_all=False):
+        path = 'data_units/'
+        url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
+        return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
+
     def upload_data_collection(
         self,
         data_collection_id: int,

synapse_sdk/plugins/categories/pre_annotation/actions/to_task.py

@@ -0,0 +1,210 @@
+import json
+from datetime import datetime
+from enum import Enum
+from typing import Annotated
+
+import requests
+from pydantic import AfterValidator, BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.utils.pydantic.validators import non_blank
+
+
+class AnnotationMethod(str, Enum):
+    FILE = 'file'
+    INFERENCE = 'inference'
+
+
+class AnnotateTaskDataStatus(str, Enum):
+    SUCCESS = 'success'
+    FAILED = 'failed'
+
+
+class ToTaskRun(Run):
+    class AnnotateTaskDataLog(BaseModel):
+        """Log model for annotate task data."""
+
+        task_info: str | None
+        status: AnnotateTaskDataStatus
+        created: str
+
+    class MetricsRecord(BaseModel):
+        """Metrics record model."""
+
+        stand_by: int
+        failed: int
+        success: int
+
+    def log_annotate_task_data(self, task_info: dict, status: AnnotateTaskDataStatus):
+        """Log annotate task data."""
+        now = datetime.now().isoformat()
+        self.log(
+            'annotate_task_data',
+            self.AnnotateTaskDataLog(task_info=json.dumps(task_info), status=status, created=now).model_dump(),
+        )
+
+    def log_metrics(self, record: MetricsRecord, category: str):
+        """Log FileToTask metrics.
+
+        Args:
+            record (MetricsRecord): The metrics record to log.
+            category (str): The category of the metrics.
+        """
+        record = self.MetricsRecord.model_validate(record)
+        self.set_metrics(value=record.dict(), category=category)
+
+
+class ToTaskParams(BaseModel):
+    """ToTask action parameters.
+
+    Args:
+        name (str): The name of the action.
+        description (str | None): The description of the action.
+        project (int): The project ID.
+        task_filter (dict): The filter of tasks.
+        method (AnnotationMethod): The method of annotation.
+        target_specification_name (str | None): The name of the target specification.
+        pre_processor (int | None): The pre processor ID.
+        pre_processor_params (dict): The params of the pre processor.
+    """
+
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str | None = None
+    project: int
+    task_filter: dict
+    method: AnnotationMethod | None = None
+    target_specification_name: str | None = None
+    pre_processor: int | None = None
+    pre_processor_params: dict
+
+    @field_validator('project', mode='before')
+    @classmethod
+    def check_project_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend project exists."""
+        if not value:
+            return value
+
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_project(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', 'Error occurred while checking project exists.')
+        return value
+
+
+@register_action
+class ToTaskAction(Action):
+    """ToTask action class.
+
+    * Annotate data to tasks.
+    """
+
+    name = 'to_task'
+    category = PluginCategory.PRE_ANNOTATION
+    method = RunMethod.JOB
+    run_class = ToTaskRun
+    progress_categories = {
+        'annotate_task_data': {
+            'proportion': 100,
+        },
+    }
+    metrics_categories = {'annotate_task_data'}
+
+    def start(self):
+        """Start file_to_task action.
+
+        * Generate tasks.
+        * Annotate data to tasks.
+        """
+
+        # entrypoint = self.entrypoint(self.run)
+        client = self.run.client
+        # project_id = self.params['project']
+        # project = client.get_project(project_id)
+        # data_collection_id = project['dataset']
+        # data_collection = client.get_dataset(data_collection_id)
+
+        # Generate tasks if provided project is empty.
+        task_ids_query_params = {
+            'project': self.params['project'],
+            'fields': 'id',
+        }
+        if self.params.get('task_filter'):
+            task_ids_query_params.update(self.params['task_filter'])
+        task_ids_generator, task_ids_count = client.list_tasks(params=task_ids_query_params, list_all=True)
+        task_ids = [item['id'] for item in task_ids_generator]
+
+        # If no tasks found, break the job.
+        if not task_ids_count:
+            self.run.log_message('Tasks to annotate not found.')
+            self.run.end_log()
+
+        # Annotate data to tasks.
+        task_data_annotation_type = self.params['task_data_annotation_type']
+        if task_data_annotation_type == AnnotationMethod.FILE:
+            self._handle_annotate_data_from_files(task_ids)
+        elif task_data_annotation_type == AnnotationMethod.INFERENCE:
+            self._handle_annotate_data_with_inference(task_ids)
+
+        return {}
+
+    def _handle_annotate_data_from_files(self, task_ids: list[int]):
+        """Handle annotate data from files to tasks.
+
+        Args:
+            task_ids (list[int]): List of task IDs to annotate data to.
+        """
+        client = self.run.client
+        if not (target_task_data_specification_code := self.params.get('target_task_data_specification_code')):
+            self.run.log_message('Target task data specification code not found.')
+            self.run.end_log()
+        task_params = {
+            'fields': 'id,data,data_unit',
+            'expand': 'data_unit',
+        }
+        for task_id in task_ids:
+            task = client.get_task(task_id, params=task_params)
+            data_file = task['data_unit']['files'].get(target_task_data_specification_code)
+            if not data_file:
+                self.run.log_message(f'File specification not found for task {task_id}')
+                self.run.log_annotate_task_data(
+                    {'task_id': task_id, 'error': 'File specification not found'}, AnnotateTaskDataStatus.FAILED
+                )
+                continue
+            url = data_file.get('url')
+            if not url:
+                self.run.log_message(f'URL not found for task {task_id}')
+                self.run.log_annotate_task_data(
+                    {'task_id': task_id, 'error': 'URL not found'}, AnnotateTaskDataStatus.FAILED
+                )
+                continue
+
+            try:
+                response = requests.get(url)
+                response.raise_for_status()  # Raise an exception for 4XX/5XX responses
+                data = json.loads(response.content)
+                client.patch_task(task_id, data={'data': data})
+
+                # Log success
+                self.run.log_annotate_task_data({'task_id': task_id, 'url': url}, AnnotateTaskDataStatus.SUCCESS)
+            except Exception as e:
+                self.run.log_message(f'Failed to get content from URL for task {task_id}: {str(e)}')
+                self.run.log_annotate_task_data(
+                    {'task_id': task_id, 'url': url, 'error': str(e)}, AnnotateTaskDataStatus.FAILED
+                )
+                continue
+
+    def _handle_annotate_data_with_inference(self, task_ids: list[int]):
+        """Handle annotate data with inference to tasks.
+
+        Args:
+            task_ids (list[int]): List of task IDs to annotate data to.
+        """
+        self.run.log_message('Pre annotation with inference is not supported.')
+        self.run.end_log()

synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml

@@ -1,3 +1,7 @@
 actions:
   pre_annotation:
     entrypoint: plugin.pre_annotation.pre_annotate
+  to_task:
+    entrypoint: plugin.to_task.AnnotationToTask
+    ui_schema: |
+      Dumped FormKit Schema for pre_annotation plugin to_task action's custom options

synapse_sdk/plugins/categories/{upload/templates/plugin/task_pre_annotation.py → pre_annotation/templates/plugin/to_task.py}

@@ -1,4 +1,4 @@
-class TaskPreAnnotation:
+class AnnotationToTask:
     def __init__(self, run, *args, **kwargs):
         """Initialize the plugin task pre annotation action class.
 
@@ -6,9 +6,3 @@ class TaskPreAnnotation:
             run: Plugin run object.
         """
         self.run = run
-
-    def handle_annotate_data_from_files(self):
-        pass
-
-    def handle_annotate_data_with_inference(self):
-        pass

synapse_sdk/plugins/categories/upload/templates/config.yaml

@@ -7,7 +7,3 @@ actions:
     supported_data_type: image # A primary data type of synapse backend collection. (e.g. 'image', 'text', 'video', 'pcd', 'audio')
     ui_schema: |
       Dumped FormKit Schema for upload plugin custom options
-  task_pre_annotation:
-    entrypoint: plugin.upload.TaskPreAnnotation
-    ui_schema: |
-      Dumped FormKit Schema for upload plugin custom options

{synapse_sdk-1.0.0a58.dist-info → synapse_sdk-1.0.0a59.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a58
+Version: 1.0.0a59
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a58.dist-info → synapse_sdk-1.0.0a59.dist-info}/RECORD

@@ -29,9 +29,9 @@ synapse_sdk/clients/agent/core.py,sha256=x2jgORTjT7pJY67SLuc-5lMG6CD5OWpy8UgGeTf
 synapse_sdk/clients/agent/ray.py,sha256=JrwLyVOUDG2yYsbPrxyUtWbM-FWp9B6Bl_GdDby0rt8,1559
 synapse_sdk/clients/agent/service.py,sha256=s7KuPK_DB1nr2VHrigttV1WyFonaGHNrPvU8loRxHcE,478
 synapse_sdk/clients/backend/__init__.py,sha256=MC3pndBk-SPyW9L6WnrTozoub9-EK7auXFvPHCaxeFU,1209
-synapse_sdk/clients/backend/annotation.py,sha256=f4jS4qlXH7M7mQ3EuCq-NrjJ_hJNDz8pEFAYqf-e008,996
+synapse_sdk/clients/backend/annotation.py,sha256=rtof8YVQ6M1wNm66EGbg6GRGDMAyY1YzkTnzN3GRT5o,1220
 synapse_sdk/clients/backend/core.py,sha256=5XAOdo6JZ0drfk-FMPJ96SeTd9oja-VnTwzGXdvK7Bg,1027
-synapse_sdk/clients/backend/data_collection.py,sha256=kj9TurBAljK_mFF75oaazlqnL0bd6PHbgRfR3KyTUmI,3623
+synapse_sdk/clients/backend/data_collection.py,sha256=uI-_ByLh-Xez4VIIVRBO8FCNUpDcxhBcLxCVFb_aG7o,4104
 synapse_sdk/clients/backend/hitl.py,sha256=na2mSXFud92p4zUEuagcDWk2klxO7xn-e86cm0VZEvs,709
 synapse_sdk/clients/backend/integration.py,sha256=9LjkYcBpi7aog-MODSDS4RlmYahypu65qxBj-AcY7xc,2683
 synapse_sdk/clients/backend/ml.py,sha256=JoPH9Ly2E3HJ7S5mdGLtcGq7ruQVVrYfWArogwZLlms,1193
@@ -89,9 +89,11 @@ synapse_sdk/plugins/categories/post_annotation/templates/plugin/post_annotation.
 synapse_sdk/plugins/categories/pre_annotation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/pre_annotation/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/pre_annotation/actions/pre_annotation.py,sha256=6ib3RmnGrjpsQ0e_G-mRH1lfFunQ3gh2M831vuDn7HU,344
-synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml,sha256=bUaibKbb-kai3QAHWbCYbO8nT-DQpV-KhuBszB7i-hI,78
+synapse_sdk/plugins/categories/pre_annotation/actions/to_task.py,sha256=J_buXjizGGjme1U3Kn8QtvXIEbfJN_Iud6uX8t0AWq8,7490
+synapse_sdk/plugins/categories/pre_annotation/templates/config.yaml,sha256=A_yhaLm505FrOkNuy_mhqa6UUN3GSZphzJUiAN7rEoQ,239
 synapse_sdk/plugins/categories/pre_annotation/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/pre_annotation/templates/plugin/pre_annotation.py,sha256=HBHxHuv2gMBzDB2alFfrzI_SZ1Ztk6mo7eFbR5GqHKw,106
+synapse_sdk/plugins/categories/pre_annotation/templates/plugin/to_task.py,sha256=fgmTNaPgk8MnElEds4A6LiSmktNTVTNR7jO7dM7FwbI,223
 synapse_sdk/plugins/categories/smart_tool/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/smart_tool/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/smart_tool/actions/auto_label.py,sha256=fHiqA8ntmzjs2GMVMuByR7Clh2zhLie8OPF9B8OmwxM,1279
@@ -100,11 +102,9 @@ synapse_sdk/plugins/categories/smart_tool/templates/plugin/__init__.py,sha256=47
 synapse_sdk/plugins/categories/smart_tool/templates/plugin/auto_label.py,sha256=eevNg0nOcYFR4z_L_R-sCvVOYoLWSAH1jwDkAf3YCjY,320
 synapse_sdk/plugins/categories/upload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/upload/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/upload/actions/task_pre_annotation.py,sha256=YkQZ7QECu6-PnSEv2lAbbL3smxeIHxUiu9ruBdA0_0k,3066
 synapse_sdk/plugins/categories/upload/actions/upload.py,sha256=zZx14F3789IJUH7BmZdXI43HXy0RdLknqT3aZk0WIHQ,18965
-synapse_sdk/plugins/categories/upload/templates/config.yaml,sha256=1ru7RFSXIUXeaIE1v7GD5K5YkGebwjzDQfzpTDxgsEg,631
+synapse_sdk/plugins/categories/upload/templates/config.yaml,sha256=kwHNWHFYbzDi1mEh40KozatPZbZGH44dlP0t0J7ejJw,483
 synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/upload/templates/plugin/task_pre_annotation.py,sha256=9XkUZu7USjVjDPufM0NlYmkdKfV7Hf_9v5GN1RgZzS0,350
 synapse_sdk/plugins/categories/upload/templates/plugin/upload.py,sha256=IZU4sdSMSLKPCtlNqF7DP2howTdYR6hr74HCUZsGdPk,1559
 synapse_sdk/plugins/templates/cookiecutter.json,sha256=NxOWk9A_v1pO0Ny4IYT9Cj5iiJ16--cIQrGC67QdR0I,396
 synapse_sdk/plugins/templates/hooks/post_gen_project.py,sha256=jqlYkY1O2TxIR-Vh3gnwILYy8k-D39Xx66d2KNQVMCs,147
@@ -136,9 +136,9 @@ synapse_sdk/utils/storage/providers/__init__.py,sha256=x7RGwZryT2FpVxS7fGWryRVpq
 synapse_sdk/utils/storage/providers/gcp.py,sha256=i2BQCu1Kej1If9SuNr2_lEyTcr5M_ncGITZrL0u5wEA,363
 synapse_sdk/utils/storage/providers/s3.py,sha256=W94rQvhGRXti3R4mYP7gmU5pcyCQpGFIBLvxxqLVdRM,2231
 synapse_sdk/utils/storage/providers/sftp.py,sha256=_8s9hf0JXIO21gvm-JVS00FbLsbtvly4c-ETLRax68A,1426
-synapse_sdk-1.0.0a58.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0a58.dist-info/METADATA,sha256=jGVaRNWWU0MKtiyUl3XuU8uOufFcE4B3M11jrLtQYzI,1303
-synapse_sdk-1.0.0a58.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-synapse_sdk-1.0.0a58.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-1.0.0a58.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0a58.dist-info/RECORD,,
+synapse_sdk-1.0.0a59.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0a59.dist-info/METADATA,sha256=o12nDh_7n--A4zsPrm14q8c1ccDYZULnD6SgLIr8q3g,1303
+synapse_sdk-1.0.0a59.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+synapse_sdk-1.0.0a59.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-1.0.0a59.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0a59.dist-info/RECORD,,

synapse_sdk/plugins/categories/upload/actions/task_pre_annotation.py

@@ -1,101 +0,0 @@
-from enum import Enum
-from typing import Annotated
-
-from pydantic import AfterValidator, BaseModel, field_validator
-from pydantic_core import PydanticCustomError
-
-from synapse_sdk.clients.exceptions import ClientError
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import PluginCategory, RunMethod
-from synapse_sdk.plugins.models import Run
-from synapse_sdk.utils.pydantic.validators import non_blank
-
-
-class TaskDataAnnotationType(str, Enum):
-    FILE = 'file'
-    INFERENCE = 'inference'
-
-
-class TaskPreAnnotationRun(Run):
-    pass
-
-
-class TaskPreAnnotationParams(BaseModel):
-    """TaskPreAnnotation action parameters.
-
-    Args:
-        name (str): The name of the action.
-        description (str | None): The description of the action.
-        project (int): The project ID.
-        data_collection (int): The data collection ID.
-        task_data_annotation_type (TaskDataAnnotationType): The type of task data annotation.
-    """
-
-    name: Annotated[str, AfterValidator(non_blank)]
-    description: str | None
-    project: int
-    data_collection: int
-    task_data_annotation_type: TaskDataAnnotationType
-
-    @field_validator('data_collection', mode='before')
-    @classmethod
-    def check_data_collection_exists(cls, value: str, info) -> str:
-        """Validate synapse-backend collection exists."""
-        action = info.context['action']
-        client = action.client
-        try:
-            client.get_data_collection(value)
-        except ClientError:
-            raise PydanticCustomError('client_error', 'Error occurred while checking data collection exists.')
-        return value
-
-    @field_validator('project', mode='before')
-    @classmethod
-    def check_project_exists(cls, value: str, info) -> str:
-        """Validate synapse-backend project exists."""
-        if not value:
-            return value
-
-        action = info.context['action']
-        client = action.client
-        try:
-            client.get_project(value)
-        except ClientError:
-            raise PydanticCustomError('client_error', 'Error occurred while checking project exists.')
-        return value
-
-
-@register_action
-class TaskPreAnnotationAction(Action):
-    """TaskPreAnnotation action class.
-
-    * Annotate data to tasks.
-    """
-
-    name = 'task_pre_annotation'
-    category = PluginCategory.UPLOAD
-    method = RunMethod.JOB
-    run_class = TaskPreAnnotationRun
-    progress_categories = {
-        'generate_tasks': {
-            'proportion': 10,
-        },
-        'annotate_task_data': {
-            'proportion': 90,
-        },
-    }
-
-    def start(self):
-        """Start task_pre_annotation action.
-
-        * Generate tasks.
-        * Annotate data to tasks.
-        """
-        task_pre_annotation = self.get_task_pre_annotation()
-        task_pre_annotation.handle_annotate_data_from_files()
-        return {}
-
-    def get_task_pre_annotation(self):
-        """Get task pre annotation entrypoint."""
-        return self.entrypoint()