synapse-sdk 1.0.0a36__py3-none-any.whl → 1.0.0a38__py3-none-any.whl

synapse_sdk/clients/base.py

@@ -11,6 +11,7 @@ from synapse_sdk.utils.file import files_url_to_path_from_objs
 class BaseClient:
     name = None
     base_url = None
+    page_size = 100
 
     def __init__(self, base_url):
         self.base_url = base_url
@@ -146,7 +147,8 @@ class BaseClient:
         else:
             return response
 
-    def _list_all(self, path, url_conversion=None, params=None, **kwargs):
+    def _list_all(self, path, url_conversion=None, params={}, **kwargs):
+        params['page_size'] = self.page_size
         response = self._get(path, url_conversion, params=params, **kwargs)
         yield from response['results']
         if response['next']:

synapse_sdk/plugins/categories/base.py

@@ -133,6 +133,11 @@ class Action:
     def plugin_package_manager(self):
         return self.plugin_config.get('package_manager', 'pip')
 
+    @property
+    def package_manager_options(self):
+        options = {'pip': {}, 'uv': {'uv_pip_install_options': []}}
+        return options[self.plugin_package_manager]
+
     def get_run(self):
         context = {
             'plugin_release': self.plugin_release,
@@ -147,13 +152,16 @@ class Action:
         return {env: os.environ[env] for env in self.REQUIRED_ENVS if env in os.environ}
 
     def get_runtime_env(self):
-        runtime_env = {self.plugin_package_manager: [], 'working_dir': self.plugin_url}
+        runtime_env = {self.plugin_package_manager: {'packages': []}, 'working_dir': self.plugin_url}
 
         if self.requirements:
-            runtime_env[self.plugin_package_manager] += self.requirements
+            runtime_env[self.plugin_package_manager]['packages'] += self.requirements
 
         if self.debug:
-            runtime_env[self.plugin_package_manager] += self.debug_modules
+            runtime_env[self.plugin_package_manager]['packages'] += self.debug_modules
+
+        for key, value in self.package_manager_options.items():
+            runtime_env[self.plugin_package_manager][key] = value
 
         # 맨 마지막에 진행되어야 함
         runtime_env['env_vars'] = self.envs

synapse_sdk/plugins/categories/export/actions/export.py

@@ -1,17 +1,77 @@
+import json
 from abc import ABC, abstractmethod
-from typing import Any, Literal
+from datetime import datetime
+from typing import Annotated, Any, Literal
 
-from pydantic import BaseModel, field_validator
+from pydantic import AfterValidator, BaseModel, field_validator
 from pydantic_core import PydanticCustomError
 
 from synapse_sdk.clients.exceptions import ClientError
 from synapse_sdk.i18n import gettext as _
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.categories.export.enums import ExportStatus
 from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.utils.pydantic.validators import non_blank
 from synapse_sdk.utils.storage import get_pathlib
 
 
+class ExportRun(Run):
+    class DataFileLog(BaseModel):
+        """Data file log model."""
+
+        target_id: int
+        data_file_info: str | None
+        status: ExportStatus
+        error: str | None = None
+        created: str
+
+    def log_file(
+        self, log_type: str, target_id: int, data_file_info: dict, status: ExportStatus, error: str | None = None
+    ):
+        """Log export file information.
+
+        Args:
+            log_type (str): The type of log ('export_data_file' or 'export_origin_file').
+            target_id (int): The ID of the data file.
+            data_file_info (dict): The JSON info of the data file.
+            status (ExportStatus): The status of the data file.
+            error (str | None): The error message, if any.
+        """
+        now = datetime.now().isoformat()
+        self.log(
+            log_type,
+            self.DataFileLog(
+                target_id=target_id,
+                data_file_info=json.dumps(data_file_info),
+                status=status.value,
+                error=error,
+                created=now,
+            ).model_dump(),
+        )
+
+    def export_log_json_file(
+        self,
+        target_id: int,
+        data_file_info: dict,
+        status: ExportStatus = ExportStatus.STAND_BY,
+        error: str | None = None,
+    ):
+        """Log export json data file."""
+        self.log_file('export_data_file', target_id, data_file_info, status, error)
+
+    def export_log_original_file(
+        self,
+        target_id: int,
+        data_file_info: dict,
+        status: ExportStatus = ExportStatus.STAND_BY,
+        error: str | None = None,
+    ):
+        """Log export origin data file."""
+        self.log_file('export_origin_file', target_id, data_file_info, status, error)
+
+
 class ExportTargetHandler(ABC):
     """
     Abstract base class for handling export targets.
@@ -153,6 +213,8 @@ class ExportParams(BaseModel):
     Parameters for the export action.
 
     Attributes:
+        name (str): The name of the action.
+        description (str | None): The description of the action.
         storage (int): The storage ID to save the exported data.
         save_original_file (bool): Whether to save the original file.
         path (str): The path to save the exported data.
@@ -160,6 +222,8 @@ class ExportParams(BaseModel):
         filter (dict): The filter criteria to apply.
     """
 
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str | None = None
     storage: int
     save_original_file: bool = True
     path: str
@@ -193,6 +257,7 @@ class ExportAction(Action):
     category = PluginCategory.EXPORT
     method = RunMethod.JOB
     params_model = ExportParams
+    run_class = ExportRun
     progress_categories = {
         'dataset_conversion': {
             'proportion': 100,

synapse_sdk/plugins/categories/export/enums.py

@@ -0,0 +1,7 @@
+from enum import Enum
+
+
+class ExportStatus(str, Enum):
+    SUCCESS = 'success'
+    FAILED = 'failed'
+    STAND_BY = 'stand_by'

synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -3,6 +3,8 @@ from pathlib import Path
 
 import requests
 
+from synapse_sdk.plugins.categories.export.enums import ExportStatus
+
 
 def export(run, export_items, path_root, **params):
     """Executes the export task.
@@ -54,12 +56,12 @@ def export(run, export_items, path_root, **params):
         if save_original_file_flag:
             if no == 1:
                 run.log_message('Saving original file.')
-            save_original_file(final_data, origin_files_output_path, errors_original_file_list)
+            save_original_file(run, final_data, origin_files_output_path, errors_original_file_list)
 
         # Extract data as JSON files
         if no == 1:
             run.log_message('Saving json file.')
-        save_as_json(final_data, json_output_path, errors_json_file_list)
+        save_as_json(run, final_data, json_output_path, errors_json_file_list)
 
     run.end_log()
 
@@ -100,10 +102,11 @@ def get_original_file_pathlib(files):
     return Path(files['meta']['path_original'])
 
 
-def save_original_file(result, base_path, error_file_list):
+def save_original_file(run, result, base_path, error_file_list):
     """Saves the original file.
 
     Args:
+        run : Execution object
         result (dict): API response data containing file information.
         base_path (Path): The directory where the file will be saved.
         error_file_list (list): A list to store error files.
@@ -111,17 +114,25 @@ def save_original_file(result, base_path, error_file_list):
     file_url = result['files']['url']
     file_name = get_original_file_pathlib(result['files']).name
     response = requests.get(file_url)
+    file_info = {'file_name': file_name}
+    error_msg = ''
     try:
         with (base_path / file_name).open('wb') as file:
             file.write(response.content)
+        status = ExportStatus.SUCCESS
     except Exception as e:
-        error_file_list.append([file_name, str(e)])
+        error_msg = str(e)
+        error_file_list.append([file_name, error_msg])
+        status = ExportStatus.FAILED
+
+    run.export_log_original_file(result['id'], file_info, status, error_msg)
 
 
-def save_as_json(result, base_path, error_file_list):
+def save_as_json(run, result, base_path, error_file_list):
     """Saves the data as a JSON file.
 
     Args:
+        run : Execution object
         result (dict): API response data containing file information.
         base_path (Path): The directory where the file will be saved.
         error_file_list (list): A list to store error files.
@@ -129,8 +140,15 @@ def save_as_json(result, base_path, error_file_list):
     # Default save file name: original file name
     file_name = get_original_file_pathlib(result['files']).stem
     json_data = result['data']
+    file_info = {'file_name': file_name}
+    error_msg = ''
     try:
         with (base_path / f'{file_name}.json').open('w', encoding='utf-8') as f:
             json.dump(json_data, f, indent=4, ensure_ascii=False)
+        status = ExportStatus.SUCCESS
     except Exception as e:
+        error_msg = str(e)
         error_file_list.append([f'{file_name}.json', str(e)])
+        status = ExportStatus.FAILED
+
+    run.export_log_json_file(result['id'], file_info, status, error_msg)

synapse_sdk/plugins/categories/neural_net/actions/tune.py

@@ -0,0 +1,268 @@
+import copy
+import tempfile
+from pathlib import Path
+from typing import Annotated, Optional
+
+from pydantic import AfterValidator, BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.categories.neural_net.actions.train import TrainAction, TrainRun
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.utils.file import archive
+from synapse_sdk.utils.module_loading import import_string
+from synapse_sdk.utils.pydantic.validators import non_blank
+
+
+class TuneRun(TrainRun):
+    is_tune = True
+    completed_samples = 0
+    num_samples = 0
+    checkpoint_output = None
+
+
+class TuneConfig(BaseModel):
+    mode: Optional[str] = None
+    metric: Optional[str] = None
+    num_samples: int = 1
+    max_concurrent_trials: Optional[int] = None
+
+
+class TuneParams(BaseModel):
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str
+    checkpoint: int | None
+    dataset: int
+    tune_config: TuneConfig
+
+    @field_validator('name')
+    @staticmethod
+    def unique_name(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            job_exists = client.exists(
+                'list_jobs',
+                params={
+                    'ids_ex': action.job_id,
+                    'category': 'neural_net',
+                    'job__action': 'tune',
+                    'is_active': True,
+                    'params': f'name:{value}',
+                },
+            )
+            assert not job_exists, '존재하는 튜닝 작업 이름입니다.'
+        except ClientError:
+            raise PydanticCustomError('client_error', '')
+        return value
+
+
+@register_action
+class TuneAction(TrainAction):
+    """
+    **Must read** Important notes before using Tune:
+
+    1. Path to the model output (which is the return value of your train function)
+       should be set to the checkpoint_output attribute of the run object **before**
+       starting the training.
+    2. Before exiting the training function, report the results to Tune.
+    3. When using own tune.py, take note of the difference in the order of parameters.
+       tune() function starts with hyperparameter, run, dataset, checkpoint, **kwargs
+       whereas the train() function starts with run, dataset, hyperparameter, checkpoint, **kwargs.
+    ----
+    1)
+    Set the output path for the checkpoint to export best model
+
+    output_path = Path('path/to/your/weights')
+    run.checkpoint_output = str(output_path)
+
+    2)
+    Before exiting the training function, report the results to Tune.
+    The results_dict should contain the metrics you want to report.
+
+    Example: (In train function)
+    results_dict = {
+        "accuracy": accuracy,
+        "loss": loss,
+        # Add other metrics as needed
+    }
+    if hasattr(self.dm_run, 'is_tune') and self.dm_run.is_tune:
+        tune.report(results_dict, checkpoint=tune.Checkpoint.from_directory(self.dm_run.checkpoint_output))
+
+
+    3)
+    tune() function takes hyperparameter, run, dataset, checkpoint, **kwargs in that order
+    whereas train() function takes run, dataset, hyperparameter, checkpoint, **kwargs in that order.
+
+    --------------------------------------------------------------------------------------------------------
+
+    **중요** Tune 사용 전 반드시 읽어야 할 사항들
+
+    1. 본 플러그인의 train 함수에서, 학습을 진행하기 코드 전에
+       결과 모델 파일의 경로(train함수의 리턴 값)을 checkpoint_output 속성에 설정해야 합니다.
+    2. 학습이 종료되기 전에, 결과를 Tune에 보고해야 합니다.
+    3. 플러그인에서 tune.py를 직접 생성해서 사용할 시, 매개변수의 순서가 다릅니다.
+
+    ----
+    1)
+    체크포인트를 설정할 경로를 지정합니다.
+    output_path = Path('path/to/your/weights')
+    run.checkpoint_output = str(output_path)
+
+    2)
+    학습이 종료되기 전에, 결과를 Tune에 보고합니다.
+    results_dict = {
+        "accuracy": accuracy,
+        "loss": loss,
+        # 필요한 다른 메트릭 추가
+    }
+    if hasattr(self.dm_run, 'is_tune') and self.dm_run.is_tune:
+        tune.report(results_dict, checkpoint=tune.Checkpoint.from_directory(self.dm_run.checkpoint_output))
+
+    3)
+    tune() 함수는 hyperparameter, run, dataset, checkpoint, **kwargs 순서이고
+    train() 함수는 run, dataset, hyperparameter, checkpoint, **kwargs 순서입니다.
+    """
+
+    name = 'tune'
+    category = PluginCategory.NEURAL_NET
+    method = RunMethod.JOB
+    run_class = TuneRun
+    params_model = TuneParams
+    progress_categories = {
+        'dataset': {
+            'proportion': 5,
+        },
+        'trials': {
+            'proportion': 90,
+        },
+        'model_upload': {
+            'proportion': 5,
+        },
+    }
+
+    def start(self):
+        from ray import tune
+
+        # download dataset
+        self.run.log_message('Preparing dataset for hyperparameter tuning.')
+        input_dataset = self.get_dataset()
+
+        # retrieve checkpoint
+        checkpoint = None
+        if self.params['checkpoint']:
+            self.run.log_message('Retrieving checkpoint.')
+            checkpoint = self.get_model(self.params['checkpoint'])
+
+        # train dataset
+        self.run.log_message('Starting training for hyperparameter tuning.')
+
+        # Save num_samples to TuneRun for logging
+        self.run.num_samples = self.params['tune_config']['num_samples']
+
+        entrypoint = self.entrypoint
+        if not self._tune_override_exists():
+            # entrypoint must be train entrypoint
+            def _tune(param_space, run, dataset, checkpoint=None, **kwargs):
+                return entrypoint(run, dataset, param_space, checkpoint, **kwargs)
+
+            entrypoint = _tune
+
+        trainable = tune.with_parameters(entrypoint, run=self.run, dataset=input_dataset, checkpoint=checkpoint)
+        tune_config = self.params['tune_config']
+
+        hyperparameter = self.params['hyperparameter']
+        param_space = self.convert_tune_params(hyperparameter)
+        temp_path = tempfile.TemporaryDirectory()
+
+        tuner = tune.Tuner(
+            tune.with_resources(trainable, resources=self.tune_resources),
+            tune_config=tune.TuneConfig(**tune_config),
+            run_config=tune.RunConfig(
+                name=f'synapse_tune_hpo_{self.job_id}',
+                log_to_file=('stdout.log', 'stderr.log'),
+                storage_path=temp_path.name,
+            ),
+            param_space=param_space,
+        )
+        result = tuner.fit()
+
+        best_result = result.get_best_result()
+
+        # upload model_data
+        self.run.log_message('Registering best model data.')
+        self.run.set_progress(0, 1, category='model_upload')
+        self.create_model_from_result(best_result)
+        self.run.set_progress(1, 1, category='model_upload')
+
+        self.run.end_log()
+
+        return {'best_result': best_result.config}
+
+    @property
+    def tune_resources(self):
+        resources = {}
+        for option in ['num_cpus', 'num_gpus']:
+            option_value = self.params.get(option)
+            if option_value:
+                # Remove the 'num_' prefix and trailing s from the option name
+                resources[(lambda s: s[4:-1])(option)] = option_value
+        return resources
+
+    def create_model_from_result(self, result):
+        params = copy.deepcopy(self.params)
+        configuration_fields = ['hyperparameter']
+        configuration = {field: params.pop(field) for field in configuration_fields}
+
+        with tempfile.TemporaryDirectory() as temp_path:
+            archive_path = Path(temp_path, 'archive.zip')
+
+            # Archive tune results
+            # https://docs.ray.io/en/latest/tune/tutorials/tune_get_data_in_and_out.html#getting-data-out-of-tune-using-checkpoints-other-artifacts
+            archive(result.path, archive_path)
+
+            return self.client.create_model({
+                'plugin': self.plugin_release.plugin,
+                'version': self.plugin_release.version,
+                'file': str(archive_path),
+                'configuration': configuration,
+                **params,
+            })
+
+    @staticmethod
+    def convert_tune_params(param_list):
+        """
+        Convert YAML hyperparameter configuration to Ray Tune parameter dictionary.
+
+        Args:
+            param_list (list): List of hyperparameter configurations.
+
+        Returns:
+            dict: Ray Tune parameter dictionary
+        """
+        from ray import tune
+
+        param_space = {}
+
+        for param in param_list:
+            name = param['name']
+            param_type = param['type']
+
+            if param_type == 'loguniform':
+                param_space[name] = tune.loguniform(param['min'], param['max'])
+            elif param_type == 'choice':
+                param_space[name] = tune.choice(param['options'])
+            elif param_type == 'randint':
+                param_space[name] = tune.randint(param['min'], param['max'])
+            # Add more type handlers as needed
+
+        return param_space
+
+    @staticmethod
+    def _tune_override_exists(module_path='plugin.tune') -> bool:
+        try:
+            import_string(module_path)
+            return True
+        except ImportError:
+            return False

synapse_sdk/plugins/categories/upload/actions/upload.py

@@ -1,3 +1,6 @@
+import json
+from datetime import datetime
+from enum import Enum
 from typing import Annotated, Dict, List
 
 from pydantic import AfterValidator, BaseModel, field_validator
@@ -16,8 +19,69 @@ from synapse_sdk.utils.pydantic.validators import non_blank
 from synapse_sdk.utils.storage import get_pathlib
 
 
+class UploadStatus(str, Enum):
+    SUCCESS = 'success'
+    FAILED = 'failed'
+
+
 class UploadRun(Run):
-    pass
+    class DataFileLog(BaseModel):
+        """Data file log model."""
+
+        data_file_info: str | None
+        status: UploadStatus
+        created: str
+
+    class DataUnitLog(BaseModel):
+        """Data unit log model."""
+
+        data_unit_id: int | None
+        status: UploadStatus
+        created: str
+
+    class TaskLog(BaseModel):
+        """Task log model."""
+
+        task_id: int | None
+        status: UploadStatus
+        created: str
+
+    def log_data_file(self, data_file_info: dict, status: UploadStatus):
+        """Upload data_file log.
+
+        Args:
+            data_file_info (dict): The json info of the data file.
+            checksum (str): The checksum of the data file.
+            status (DataUnitStatus): The status of the data unit.
+        """
+        now = datetime.now().isoformat()
+        self.log(
+            'upload_data_file',
+            self.DataFileLog(data_file_info=json.dumps(data_file_info), status=status.value, created=now).model_dump(),
+        )
+
+    def log_data_unit(self, data_unit_id: int, status: UploadStatus):
+        """Upload data_unit log.
+
+        Args:
+            data_unit_id (int): The ID of the data unit.
+            status (DataUnitStatus): The status of the data unit.
+        """
+        now = datetime.now().isoformat()
+        self.log(
+            'upload_data_unit',
+            self.DataUnitLog(data_unit_id=data_unit_id, status=status.value, created=now).model_dump(),
+        )
+
+    def log_task(self, task_id: int, status: UploadStatus):
+        """Upload task log.
+
+        Args:
+            task_id (int): The ID of the task.
+            status (UploadStatus): The status of the task.
+        """
+        now = datetime.now().isoformat()
+        self.log('upload_task', self.TaskLog(task_id=task_id, status=status.value, created=now).model_dump())
 
 
 class UploadParams(BaseModel):
@@ -108,6 +172,7 @@ class UploadAction(Action):
     name = 'upload'
     category = PluginCategory.UPLOAD
     method = RunMethod.JOB
+    run_class = UploadRun
     progress_categories = {
         'analyze_collection': {
             'proportion': 5,
@@ -144,9 +209,7 @@ class UploadAction(Action):
         uploader = self.get_uploader(pathlib_cwd)
 
         # Analyze Collection file specifications to determine the data structure for upload.
-        self.run.set_progress(0, 1, category='analyze_collection')
         file_specification_template = self._analyze_collection()
-        self.run.set_progress(1, 1, category='analyze_collection')
 
         # Setup result dict.
         result = {}
@@ -155,57 +218,48 @@ class UploadAction(Action):
         organized_files = uploader.handle_upload_files()
         if not self._validate_organized_files(file_specification_template, organized_files):
             self.run.log_message('Validate organized files failed.')
+            self.run.end_log()
             return result
 
         # Upload files to synapse-backend.
         organized_files_count = len(organized_files)
         if not organized_files_count:
             self.run.log_message('Files not found on the path.', context=Context.WARNING.value)
+            self.run.end_log()
             return result
-
-        self.run.set_progress(0, organized_files_count, category='upload_data_files')
-        self.run.log_message('Uploading data files...')
-        result['uploaded_files'] = self._upload_files(organized_files)
-        self.run.set_progress(organized_files_count, organized_files_count, category='upload_data_files')
-        self.run.log_message('Upload data files completed.')
+        uploaded_files = self._upload_files(organized_files, organized_files_count)
+        result['uploaded_files_count'] = len(uploaded_files)
 
         # Generate data units for the uploaded data.
-        upload_result_count = len(result['uploaded_files'])
+        upload_result_count = len(uploaded_files)
         if not upload_result_count:
             self.run.log_message('No files were uploaded.', context=Context.WARNING.value)
+            self.run.end_log()
             return result
-
-        self.run.set_progress(0, upload_result_count, category='generate_data_units')
-        generated_data_units = self._generate_data_units(result['uploaded_files'])
-        result['generated_data_units'] = generated_data_units
-        self.run.set_progress(upload_result_count, upload_result_count, category='generate_data_units')
+        generated_data_units = self._generate_data_units(uploaded_files, upload_result_count)
+        result['generated_data_units_count'] = len(generated_data_units)
 
         # Setup task with uploaded synapse-backend data units.
         if not len(generated_data_units):
             self.run.log_message('No data units were generated.', context=Context.WARNING.value)
+            self.run.end_log()
             return result
 
-        self.run.set_progress(0, 1, category='generate_tasks')
         if self.config['options']['allow_generate_tasks'] and self.params['is_generate_tasks']:
-            self.run.log_message('Generating tasks with data files...')
-            self._generate_tasks(generated_data_units)
-            self.run.log_message('Generating tasks completed')
+            generated_tasks = self._generate_tasks(generated_data_units)
+            result['generated_tasks_count'] = len(generated_tasks)
         else:
             self.run.log_message('Generating tasks process has passed.')
 
-        self.run.set_progress(1, 1, category='generate_tasks')
-
         # Generate ground truths for the uploaded data.
         # TODO: Need to add ground truths generation logic later.
-        self.run.set_progress(0, 1, category='generate_ground_truths')
         if self.config['options']['allow_generate_ground_truths'] and self.params['is_generate_ground_truths']:
-            self.run.log_message('Generating ground truths...')
-            self._generate_ground_truths()
-            self.run.log_message('Generating ground truths completed')
+            generated_ground_truths = self._generate_ground_truths()
+            result['generated_ground_truths_count'] = len(generated_ground_truths)
         else:
             self.run.log_message('Generating ground truths process has passed.')
-        self.run.set_progress(1, 1, category='generate_ground_truths')
 
+        self.run.end_log()
         return result
 
     def _analyze_collection(self) -> Dict:
@@ -214,9 +268,17 @@ class UploadAction(Action):
         Returns:
             Dict: The file specifications of the collection.
         """
+
+        # Initialize progress
+        self.run.set_progress(0, 1, category='analyze_collection')
+
         client = self.run.client
         collection_id = self.params['collection']
         collection = client.get_dataset(collection_id)
+
+        # Finish progress
+        self.run.set_progress(1, 1, category='analyze_collection')
+
         return collection['file_specifications']
 
     def _validate_organized_files(self, file_specification_template: Dict, organized_files: List) -> bool:
@@ -224,24 +286,35 @@ class UploadAction(Action):
         validator = FileSpecificationValidator(file_specification_template, organized_files)
         return validator.validate()
 
-    def _upload_files(self, organized_files) -> List:
+    def _upload_files(self, organized_files, organized_files_count: int) -> List:
         """Upload files to synapse-backend.
 
         Returns:
             Dict: The result of the upload.
         """
+        # Initialize progress
+        self.run.set_progress(0, organized_files_count, category='upload_data_files')
+        self.run.log_message('Uploading data files...')
+
         client = self.run.client
         collection_id = self.params['collection']
         upload_result = []
         organized_files_count = len(organized_files)
         current_progress = 0
         for organized_file in organized_files:
-            upload_result.append(client.upload_data_file(organized_file, collection_id))
+            uploaded_data_file = client.upload_data_file(organized_file, collection_id)
+            self.run.log_data_file(organized_file, UploadStatus.SUCCESS)
+            upload_result.append(uploaded_data_file)
             self.run.set_progress(current_progress, organized_files_count, category='upload_data_files')
             current_progress += 1
+
+        # Finish progress
+        self.run.set_progress(organized_files_count, organized_files_count, category='upload_data_files')
+        self.run.log_message('Upload data files completed.')
+
         return upload_result
 
-    def _generate_data_units(self, uploaded_files: List) -> List:
+    def _generate_data_units(self, uploaded_files: List, upload_result_count: int) -> List:
         """Generate data units for the uploaded data.
 
         TODO: make batch size configurable.
@@ -249,23 +322,36 @@ class UploadAction(Action):
         Returns:
             Dict: The result of the generate data units process.
         """
+        # Initialize progress
+        self.run.set_progress(0, upload_result_count, category='generate_data_units')
+
         client = self.run.client
 
-        generation_result = []
+        generated_data_units = []
         current_progress = 0
         batches = get_batched_list(uploaded_files, 100)
         batches_count = len(batches)
         for batch in batches:
-            generation_result.append(client.create_data_units(batch))
+            created_data_units = client.create_data_units(batch)
+            generated_data_units.append(created_data_units)
             self.run.set_progress(current_progress, batches_count, category='generate_data_units')
             current_progress += 1
-        return generation_result
+            for created_data_unit in created_data_units:
+                self.run.log_data_unit(created_data_unit['id'], UploadStatus.SUCCESS)
+
+        # Finish progress
+        self.run.set_progress(upload_result_count, upload_result_count, category='generate_data_units')
 
-    def _generate_tasks(self, generated_data_units: List):
+        return sum(generated_data_units, [])
+
+    def _generate_tasks(self, generated_data_units: List) -> List:
         """Setup task with uploaded synapse-backend data units.
 
         TODO: make batch size configurable.
         """
+        # Initialize progress
+        self.run.set_progress(0, 1, category='generate_tasks')
+        self.run.log_message('Generating tasks with data files...')
 
         # Prepare batches for processing
         client = self.run.client
@@ -273,21 +359,37 @@ class UploadAction(Action):
         current_progress = 0
 
         # Generate tasks
+        generated_tasks = []
         generated_data_units_count = len(generated_data_units)
-        for data_units in generated_data_units:
+        for data_unit in generated_data_units:
             tasks_data = []
-            for data_unit in data_units:
-                task_data = {'project': project_id, 'data_unit': data_unit['id']}
-                tasks_data.append(task_data)
-
+            task_data = {'project': project_id, 'data_unit': data_unit['id']}
+            tasks_data.append(task_data)
             if tasks_data:
-                client.create_tasks(tasks_data)
+                created_tasks = client.create_tasks(tasks_data)
+                created_task_ids = [created_task['id'] for created_task in created_tasks]
+                generated_tasks.append(created_task_ids)
+                for created_task_id in created_task_ids:
+                    self.run.log_task(created_task_id, UploadStatus.SUCCESS)
 
             self.run.set_progress(current_progress, generated_data_units_count, category='generate_tasks')
             current_progress += 1
 
+        # Finish progress
+        self.run.log_message('Generating tasks completed')
+        self.run.set_progress(1, 1, category='generate_tasks')
+
+        return sum(generated_tasks, [])
+
     def _generate_ground_truths(self):
         """Generate ground truths for the uploaded data.
 
         TODO: Need to add ground truths generation logic later.
         """
+        # Initialize progress
+        self.run.set_progress(0, 1, category='generate_ground_truths')
+        self.run.log_message('Generating ground truths...')
+
+        # Finish progress
+        self.run.log_message('Generating ground truths completed')
+        self.run.set_progress(1, 1, category='generate_ground_truths')

synapse_sdk/plugins/models.py

@@ -45,6 +45,11 @@ class PluginRelease:
     def package_manager(self):
         return self.config.get('package_manager', 'pip')
 
+    @cached_property
+    def package_manager_options(self):
+        options = {'pip': {}, 'uv': {'uv_pip_install_options': []}}
+        return options[self.package_manager]
+
     @cached_property
     def checksum(self):
         return hash_text(self.code)
@@ -69,7 +74,10 @@ class PluginRelease:
 
             warm_up.options(
                 runtime_env={
-                    self.package_manager: ['-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/requirements.txt'],
+                    self.package_manager: {
+                        'packages': ['-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/requirements.txt']
+                        ** self.package_manager_options
+                    },
                     'working_dir': self.get_url(self.envs['SYNAPSE_PLUGIN_STORAGE']),
                 },
                 scheduling_strategy=strategy,

synapse_sdk/utils/file.py

@@ -150,11 +150,12 @@ def calculate_checksum(file_path, prefix=''):
     return checksum
 
 
-def archive(input_path, output_path):
+def archive(input_path, output_path, append=False):
     input_path = Path(input_path)
     output_path = Path(output_path)
 
-    with zipfile.ZipFile(output_path, mode='w', compression=zipfile.ZIP_DEFLATED) as zipf:
+    mode = 'a' if append and output_path.exists() else 'w'
+    with zipfile.ZipFile(output_path, mode=mode, compression=zipfile.ZIP_DEFLATED) as zipf:
         if input_path.is_file():
             zipf.write(input_path, input_path.name)
         else:

{synapse_sdk-1.0.0a36.dist-info → synapse_sdk-1.0.0a38.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a36
+Version: 1.0.0a38
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a36.dist-info → synapse_sdk-1.0.0a38.dist-info}/RECORD

@@ -21,7 +21,7 @@ synapse_sdk/cli/plugin/create.py,sha256=HpYTpohV1NbSrULaVUlc4jWLWznPrx7glgydTM3s
 synapse_sdk/cli/plugin/publish.py,sha256=sIl1wiuSC3lAUpE3rOF4UDKDy2G5EVLlelMjk2aT05g,1221
 synapse_sdk/cli/plugin/run.py,sha256=xz5LRm3zh8Y9DMjw5FFRFVRWSCWtYfZJskfCmrPikaQ,2598
 synapse_sdk/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/clients/base.py,sha256=RaQN_41NxFKtVsVKB7I7hdKJAR8ScuyCoK_x1D-U0c0,6399
+synapse_sdk/clients/base.py,sha256=emtffTGGtPOSz6tT_NCL9cGE7ZEOv9pTHyc9BRfoe2s,6462
 synapse_sdk/clients/exceptions.py,sha256=ylv7x10eOp4aA3a48jwonnvqvkiYwzJYXjkVkRTAjwk,220
 synapse_sdk/clients/utils.py,sha256=8pPJTdzHiRPSbZMoQYHAgR2BAMO6u_R_jMV6a2p34iQ,392
 synapse_sdk/clients/agent/__init__.py,sha256=Pz8_iTbIbnb7ywGJ3feqoZVmO2I3mEbwpWsISIxh0BU,1968
@@ -44,11 +44,11 @@ synapse_sdk/clients/validators/collections.py,sha256=LtnwvutsScubOUcZ2reGHLCzseX
 synapse_sdk/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/enums.py,sha256=ibixwqA3sCNSriG1jAtL54JQc_Zwo3MufwYUqGhVncc,523
 synapse_sdk/plugins/exceptions.py,sha256=Qs7qODp_RRLO9y2otU2T4ryj5LFwIZODvSIXkAh91u0,691
-synapse_sdk/plugins/models.py,sha256=njTQIT-c2d7TsqAN__q1aoYm8hLEPC7ludTj665iN-4,4148
+synapse_sdk/plugins/models.py,sha256=FI_6Hr4q4hGj-GwHjucOfX4HYsUpraGd2yeuy4FjjC0,4438
 synapse_sdk/plugins/upload.py,sha256=VJOotYMayylOH0lNoAGeGHRkLdhP7jnC_A0rFQMvQpQ,3228
 synapse_sdk/plugins/utils.py,sha256=4_K6jIl0WrsXOEhFp94faMOriSsddOhIiaXcawYYUUA,3300
 synapse_sdk/plugins/categories/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/base.py,sha256=cfk7mgidCBnldZ8nCo_Nq_qzP9C1LJYgSXWtYT6hJDw,10046
+synapse_sdk/plugins/categories/base.py,sha256=ATI1VjBWm2rimSkNiiCjfZn7FO4x2oltmh81pJJGL0w,10389
 synapse_sdk/plugins/categories/decorators.py,sha256=Gw6T-UHwpCKrSt596X-g2sZbY_Z1zbbogowClj7Pr5Q,518
 synapse_sdk/plugins/categories/registry.py,sha256=KdQR8SUlLT-3kgYzDNWawS1uJnAhrcw2j4zFaTpilRs,636
 synapse_sdk/plugins/categories/templates.py,sha256=FF5FerhkZMeW1YcKLY5cylC0SkWSYdJODA_Qcm4OGYQ,887
@@ -59,17 +59,19 @@ synapse_sdk/plugins/categories/data_validation/templates/config.yaml,sha256=Hijb
 synapse_sdk/plugins/categories/data_validation/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py,sha256=90I5boUpEXvO3mEuKKBs528ls2A4h8Iw4ReOID2h00Y,139
 synapse_sdk/plugins/categories/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/export/enums.py,sha256=gtyngvQ1DKkos9iKGcbecwTVQQ6sDwbrBPSGPNb5Am0,127
 synapse_sdk/plugins/categories/export/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/export/actions/export.py,sha256=DN1z2zxjzkgAaalEVWyJGXIVkpb4QeppODm1S7XDVic,7693
+synapse_sdk/plugins/categories/export/actions/export.py,sha256=m-V-PsCmcF1rhiv8Rf9s3xaqJgpjC_kwITv5od5epGQ,9918
 synapse_sdk/plugins/categories/export/templates/config.yaml,sha256=N7YmnFROb3s3M35SA9nmabyzoSb5O2t2TRPicwFNN2o,56
 synapse_sdk/plugins/categories/export/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/export/templates/plugin/export.py,sha256=nbjvgFVQpPN5Lo1UnPL5p__BYeejMZLMZ4RT_yd7vJU,4561
+synapse_sdk/plugins/categories/export/templates/plugin/export.py,sha256=39XLGo8ui5FscbwZyX3JwmrJqGGvOYrY3FMYDKXwTOQ,5192
 synapse_sdk/plugins/categories/neural_net/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/actions/deployment.py,sha256=oetIwZoee5vxriPX3r1onmxgwojUyaRTlnBIdaQ1zk8,3895
 synapse_sdk/plugins/categories/neural_net/actions/inference.py,sha256=0a655ELqNVjPFZTJDiw4EUdcMCPGveUEKyoYqpwMFBU,1019
 synapse_sdk/plugins/categories/neural_net/actions/test.py,sha256=JY25eg-Fo6WbgtMkGoo_qNqoaZkp3AQNEypJmeGzEog,320
 synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=kve6iTCg2kUeavMQTR2JFuoYDu-QWZFFlB58ZICQtdM,5406
+synapse_sdk/plugins/categories/neural_net/actions/tune.py,sha256=XJczlLDF8FOJXA-7TXNZa3npWhMsT0wGqQwYW3w5TDo,9475
 synapse_sdk/plugins/categories/neural_net/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/base/inference.py,sha256=R5DASI6-5vzsjDOYxqeGGMBjnav5qHF4hNJT8zNUR3I,1097
 synapse_sdk/plugins/categories/neural_net/templates/config.yaml,sha256=uZVuXjIfsd_pTaSKptHeHn1TN2FIiLrvvpkClToc6po,596
@@ -97,7 +99,7 @@ synapse_sdk/plugins/categories/smart_tool/templates/plugin/__init__.py,sha256=47
 synapse_sdk/plugins/categories/smart_tool/templates/plugin/auto_label.py,sha256=eevNg0nOcYFR4z_L_R-sCvVOYoLWSAH1jwDkAf3YCjY,320
 synapse_sdk/plugins/categories/upload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/upload/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/upload/actions/upload.py,sha256=ry5Whogy5opZ7U_G6hQlbej8ufs5BS_VaJLY648vpF4,11347
+synapse_sdk/plugins/categories/upload/actions/upload.py,sha256=9DIH4Aw70LxDpfhrpD0MfncE1m9oj-v52FpaChkVEnA,14755
 synapse_sdk/plugins/categories/upload/templates/config.yaml,sha256=0PhB2uD-9ufavZs7EiF6xj4aBgZuif9mFFGGfzG7HuY,147
 synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/upload/templates/plugin/upload.py,sha256=dnK8gy33GjG5ettayawDJv1gM3xCm1K6lM-PfeeTjQw,1163
@@ -116,7 +118,7 @@ synapse_sdk/shared/enums.py,sha256=WMZPag9deVF7VCXaQkLk7ly_uX1KwbNzRx9TdvgaeFE,1
 synapse_sdk/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/utils/dataset.py,sha256=zWTzFmv589izFr62BDuApi3r5FpTsdm-5AmriC0AEdM,1865
 synapse_sdk/utils/debug.py,sha256=F7JlUwYjTFZAMRbBqKm6hxOIz-_IXYA8lBInOS4jbS4,100
-synapse_sdk/utils/file.py,sha256=zP8eOZifGiYP9PyC4ivQwxs-ljbtXRtbWN4yOjZF6tc,6658
+synapse_sdk/utils/file.py,sha256=Qb5FihoX1J0wsF2UAckc0d0c3IMHn0NrX9Vt3cXAwt4,6732
 synapse_sdk/utils/module_loading.py,sha256=chHpU-BZjtYaTBD_q0T7LcKWtqKvYBS4L0lPlKkoMQ8,1020
 synapse_sdk/utils/network.py,sha256=wg-oFM0gKK5REqIUO8d-x9yXJfqbnkSbbF0_qyxpwz4,412
 synapse_sdk/utils/string.py,sha256=rEwuZ9SAaZLcQ8TYiwNKr1h2u4CfnrQx7SUL8NWmChg,216
@@ -130,9 +132,9 @@ synapse_sdk/utils/storage/providers/__init__.py,sha256=x7RGwZryT2FpVxS7fGWryRVpq
 synapse_sdk/utils/storage/providers/gcp.py,sha256=i2BQCu1Kej1If9SuNr2_lEyTcr5M_ncGITZrL0u5wEA,363
 synapse_sdk/utils/storage/providers/s3.py,sha256=W94rQvhGRXti3R4mYP7gmU5pcyCQpGFIBLvxxqLVdRM,2231
 synapse_sdk/utils/storage/providers/sftp.py,sha256=_8s9hf0JXIO21gvm-JVS00FbLsbtvly4c-ETLRax68A,1426
-synapse_sdk-1.0.0a36.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0a36.dist-info/METADATA,sha256=q_dbloQbRO4v_kWdEH5BFmftNs0ocAHu5agv_cOvJA8,1160
-synapse_sdk-1.0.0a36.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-synapse_sdk-1.0.0a36.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-1.0.0a36.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0a36.dist-info/RECORD,,
+synapse_sdk-1.0.0a38.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0a38.dist-info/METADATA,sha256=GxLFIbbXbVYUjusRE7MAZefelK_XNYz0oGTOOCNuybU,1160
+synapse_sdk-1.0.0a38.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+synapse_sdk-1.0.0a38.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-1.0.0a38.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0a38.dist-info/RECORD,,