synapse-sdk 1.0.0a16__py3-none-any.whl → 1.0.0a18__py3-none-any.whl

synapse_sdk/clients/backend/ml.py

@@ -9,9 +9,7 @@ class MLClientMixin(BaseClient):
 
     def get_model(self, pk, params=None, url_conversion=None):
         path = f'models/{pk}/'
-        url_conversion = get_default_url_conversion(
-            url_conversion, files_fields=['files', 'parent.files'], is_list=False
-        )
+        url_conversion = get_default_url_conversion(url_conversion, files_fields=['file'], is_list=False)
         return self._get(path, params=params, url_conversion=url_conversion)
 
     def create_model(self, data):

synapse_sdk/plugins/categories/base.py

@@ -147,7 +147,8 @@ class Action:
                 return self.start_by_restapi()
             else:
                 result = self.start()
-                self.post_action_by_job(result)
+                if self.job_id:
+                    self.post_action_by_job(result)
                 return result
         return getattr(self, f'start_by_{self.method.value}')()
 
@@ -227,10 +228,9 @@ class Action:
             raise ActionError('Unable to connect to serve application')
 
     def post_action_by_job(self, result):
-        if self.client:
-            job_client = self.get_job_client()
-            logs = job_client.get_job_logs(self.job_id).split('\n')
-            self.client.update_job(self.job_id, data={'result': result or {}, 'console_logs': logs})
+        job_client = self.get_job_client()
+        logs = job_client.get_job_logs(self.job_id).split('\n')
+        self.client.update_job(self.job_id, data={'result': result or {}, 'console_logs': logs})
 
     def get_job_client(self):
         from ray.dashboard.modules.job.sdk import JobSubmissionClient

synapse_sdk/plugins/categories/neural_net/actions/deployment.py

@@ -11,7 +11,12 @@ class DeploymentAction(Action):
     method = RunMethod.JOB
 
     def get_actor_options(self):
-        return {'runtime_env': self.get_runtime_env()}
+        options = {'runtime_env': self.get_runtime_env()}
+        for option in ['num_cpus', 'num_gpus']:
+            option_value = self.params.get(option)
+            if option_value:
+                options[option] = option_value
+        return options
 
     def start(self):
         from ray import serve

synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -12,7 +12,7 @@ from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
 from synapse_sdk.plugins.enums import PluginCategory, RunMethod
 from synapse_sdk.plugins.models import Run
-from synapse_sdk.utils.file import archive
+from synapse_sdk.utils.file import archive, get_temp_path, unarchive
 from synapse_sdk.utils.pydantic.validators import non_blank
 
 
@@ -83,10 +83,15 @@ class TrainAction(Action):
         self.run.log_message('Preparing dataset for training.')
         input_dataset = self.get_dataset()
 
+        # retrieve checkpoint
+        checkpoint = None
+        if self.params['checkpoint']:
+            self.run.log_message('Retrieving checkpoint.')
+            checkpoint = self.get_model(self.params['checkpoint'])
+
         # train dataset
         self.run.log_message('Starting model training.')
-
-        result = self.entrypoint(self.run, input_dataset, hyperparameter)
+        result = self.entrypoint(self.run, input_dataset, hyperparameter, checkpoint=checkpoint)
 
         # upload model_data
         self.run.log_message('Registering model data.')
@@ -106,6 +111,7 @@ class TrainAction(Action):
         ground_truths, count_dataset = client.list_ground_truth_events(
             params={
                 'fields': ['category', 'files', 'data'],
+                'expand': ['data'],
                 'ground_truth_dataset_versions': self.params['dataset'],
             },
             list_all=True,
@@ -120,11 +126,16 @@ class TrainAction(Action):
 
         return input_dataset
 
-    def create_model(self, path):
-        if not self.client:
-            print(path)
-            return None
+    def get_model(self, model_id):
+        model = self.client.get_model(model_id)
+        model_file = Path(model['file'])
+        output_path = get_temp_path(f'models/{model_file.stem}')
+        if not output_path.exists():
+            unarchive(model_file, output_path)
+        model['path'] = output_path
+        return model
 
+    def create_model(self, path):
         params = copy.deepcopy(self.params)
         configuration_fields = ['hyperparameter']
         configuration = {field: params.pop(field) for field in configuration_fields}

synapse_sdk/plugins/cli/publish.py

@@ -26,8 +26,8 @@ def publish(ctx, host, user_token, tenant, debug_modules):
 
     data = {'plugin': plugin_release.plugin, 'file': str(archive_path), 'debug': debug}
     if debug:
-        if debug_modules:
-            data['debug_meta'] = json.dumps({'modules': debug_modules.split(',')})
+        modules = debug_modules.split(',') if debug_modules else []
+        data['debug_meta'] = json.dumps({'modules': modules})
 
     client = BackendClient(host, user_token, tenant=tenant)
     client.create_plugin_release(data)

synapse_sdk/utils/file.py

@@ -30,14 +30,17 @@ def download_file(url, path_download, name=None, coerce=None):
     return path
 
 
-def files_url_to_path(files, coerce=None):
+def files_url_to_path(files, coerce=None, file_field=None):
     path_download = Path('/tmp/datamaker') / 'media'
     path_download.mkdir(parents=True, exist_ok=True)
-    for file_name in files:
-        if isinstance(files[file_name], str):
-            files[file_name] = download_file(files[file_name], path_download, coerce=coerce)
-        else:
-            files[file_name]['path'] = download_file(files[file_name].pop('url'), path_download, coerce=coerce)
+    if file_field:
+        files[file_field] = download_file(files[file_field], path_download, coerce=coerce)
+    else:
+        for file_name in files:
+            if isinstance(files[file_name], str):
+                files[file_name] = download_file(files[file_name], path_download, coerce=coerce)
+            else:
+                files[file_name]['path'] = download_file(files[file_name].pop('url'), path_download, coerce=coerce)
 
 
 def files_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False, is_async=False):
@@ -51,7 +54,10 @@ def files_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False,
             for files_field in files_fields:
                 try:
                     files = reduce(operator.getitem, files_field.split('.'), obj)
-                    files_url_to_path(files, coerce=coerce)
+                    if isinstance(files, str):
+                        files_url_to_path(obj, coerce=coerce, file_field=files_field)
+                    else:
+                        files_url_to_path(files, coerce=coerce)
                 except KeyError:
                     pass
 
@@ -78,7 +84,7 @@ async def adownload_file(url, path_download, name=None, coerce=None):
 
 
 async def afiles_url_to_path(files, coerce=None):
-    path_download = Path('/tmp/datamaker') / 'media'
+    path_download = get_temp_path('media')
     path_download.mkdir(parents=True, exist_ok=True)
     for file_name in files:
         if isinstance(files[file_name], str):
@@ -138,3 +144,25 @@ def archive(input_path, output_path):
                 if file_path.is_file():  # Only add files, skip directories
                     arcname = file_path.relative_to(input_path.parent)
                     zipf.write(file_path, arcname)
+
+
+def unarchive(file_path, output_path):
+    """
+    Unarchives a ZIP file to a given directory.
+
+    Parameters:
+        file_path (str | Path): The path to the ZIP file.
+        output_path (str): The directory where the files will be extracted.
+    """
+    output_path = Path(output_path)
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    with zipfile.ZipFile(str(file_path), 'r') as zip_ref:
+        zip_ref.extractall(output_path)
+
+
+def get_temp_path(sub_path=None):
+    path = Path('/tmp/datamaker')
+    if sub_path:
+        path = path / sub_path
+    return path

{synapse_sdk-1.0.0a16.dist-info → synapse_sdk-1.0.0a18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a16
+Version: 1.0.0a18
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a16.dist-info → synapse_sdk-1.0.0a18.dist-info}/RECORD

@@ -19,7 +19,7 @@ synapse_sdk/clients/backend/__init__.py,sha256=50MW1CMWGaKnALSv2fOjhJZG55Xb3yrql
 synapse_sdk/clients/backend/annotation.py,sha256=eZc5EidgR_RfMGwvv1r1_mLkPdRd8e52c4zuuMjMX34,979
 synapse_sdk/clients/backend/dataset.py,sha256=a_svyCKgzF7N99l8V4u4wXD8JxiGuLW9z2EBinnz7b8,1738
 synapse_sdk/clients/backend/integration.py,sha256=Jg_8fEmbrgYXfZZcG8cDtLxR6ugPmnbNhPDyRu_Uib0,2160
-synapse_sdk/clients/backend/ml.py,sha256=lg978cCMcPiLN4ByjhrSlJHlw1_kgZQgaNZNKr9zPsI,1054
+synapse_sdk/clients/backend/ml.py,sha256=vNo1FOG9welXGizcnylLSW_-l9iJSQrTI3DhyCCyFKw,1015
 synapse_sdk/clients/ray/__init__.py,sha256=9ZSPXVVxlJ8Wp8ku7l021ENtPjVrGgQDgqifkkVAXgM,187
 synapse_sdk/clients/ray/core.py,sha256=a4wyCocAma2HAm-BHlbZnoVbpfdR-Aad2FM0z6vPFvw,731
 synapse_sdk/clients/ray/serve.py,sha256=rbCpXZYWf0oP8XJ9faa9QFNPYU7h8dltIG8xn9ZconY,907
@@ -30,7 +30,7 @@ synapse_sdk/plugins/models.py,sha256=T3ZuTw7BZwMKpz2QGaErnEPetKRD9d4z3qceJkV363o
 synapse_sdk/plugins/upload.py,sha256=VJOotYMayylOH0lNoAGeGHRkLdhP7jnC_A0rFQMvQpQ,3228
 synapse_sdk/plugins/utils.py,sha256=UYkwxkmrs0-mRgQB63SkTGD1HpIQEIiHndzTcpdNaF4,2936
 synapse_sdk/plugins/categories/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/base.py,sha256=5BrOWXbLiwZw4I1sfElQ5CF12kLDXY97b8MSvpdNIb0,8179
+synapse_sdk/plugins/categories/base.py,sha256=jpwIptMcBBrkrUlzfWUbQDWStm4tOckASGi5Zec-yjA,8179
 synapse_sdk/plugins/categories/decorators.py,sha256=Gw6T-UHwpCKrSt596X-g2sZbY_Z1zbbogowClj7Pr5Q,518
 synapse_sdk/plugins/categories/registry.py,sha256=KdQR8SUlLT-3kgYzDNWawS1uJnAhrcw2j4zFaTpilRs,636
 synapse_sdk/plugins/categories/templates.py,sha256=FF5FerhkZMeW1YcKLY5cylC0SkWSYdJODA_Qcm4OGYQ,887
@@ -48,10 +48,10 @@ synapse_sdk/plugins/categories/import/actions/__init__.py,sha256=47DEQpj8HBSa-_T
 synapse_sdk/plugins/categories/import/actions/import.py,sha256=URn6TOp081odMT5D4NlZ2XEcyKelJx8fxzdoKSkXSAI,320
 synapse_sdk/plugins/categories/neural_net/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/neural_net/actions/deployment.py,sha256=HeNGfRBIjiw_rRwzoApAuPm6nuT0qEj-BlJBpI8nGKc,1505
+synapse_sdk/plugins/categories/neural_net/actions/deployment.py,sha256=qQ1LKGfmpVcJrmtfxJUxf5Pf3L2-KEb57ReKgSXSM7Q,1706
 synapse_sdk/plugins/categories/neural_net/actions/inference.py,sha256=i18bDDjkuF9V8nxxW-T_umNIOD-Jnq_MMjIjZc6W8n8,645
 synapse_sdk/plugins/categories/neural_net/actions/test.py,sha256=JY25eg-Fo6WbgtMkGoo_qNqoaZkp3AQNEypJmeGzEog,320
-synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=7SNhETRw-L-dNEXXANejHMPYCYeqmYjpuUaRFS5vIYA,4554
+synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=ZjBE4uHOdVZ318BuepBt2l0KvSflLr1Fqt3dt4KYXJs,5113
 synapse_sdk/plugins/categories/neural_net/templates/config.yaml,sha256=dXKB1hO53hDZB73xnxLVCNQl8Sm7svMmVmuMrOCQmEU,343
 synapse_sdk/plugins/categories/neural_net/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/templates/plugin/inference.py,sha256=InfqKWJYi6sqiUnfPKHC5KYGhxckDaWZNQ202u-uVP4,366
@@ -76,7 +76,7 @@ synapse_sdk/plugins/categories/smart_tool/templates/config.yaml,sha256=3jxW8daip
 synapse_sdk/plugins/categories/smart_tool/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/smart_tool/templates/plugin/auto_label.py,sha256=eevNg0nOcYFR4z_L_R-sCvVOYoLWSAH1jwDkAf3YCjY,320
 synapse_sdk/plugins/cli/__init__.py,sha256=LPtUO0jqkhKq6xR1grpse7da2R6OoT_BeDyCNyUY0T4,380
-synapse_sdk/plugins/cli/publish.py,sha256=rhUtJsKhYyeDTQlul8mQthfMeNuQCou64faHyaP9y1c,1230
+synapse_sdk/plugins/cli/publish.py,sha256=Kg_MaOeTNCra9BIzWBASsIZxLfhnNkzbRPqdlf6aTyk,1251
 synapse_sdk/plugins/cli/run.py,sha256=lw1KbsL-xTGllF4NtD2cq-Rh6HMbhi-sO862_Ds-sUo,2330
 synapse_sdk/plugins/templates/cookiecutter.json,sha256=NxOWk9A_v1pO0Ny4IYT9Cj5iiJ16--cIQrGC67QdR0I,396
 synapse_sdk/plugins/templates/hooks/post_gen_project.py,sha256=jqlYkY1O2TxIR-Vh3gnwILYy8k-D39Xx66d2KNQVMCs,147
@@ -95,7 +95,7 @@ synapse_sdk/shared/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 synapse_sdk/shared/enums.py,sha256=WMZPag9deVF7VCXaQkLk7ly_uX1KwbNzRx9TdvgaeFE,138
 synapse_sdk/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/utils/debug.py,sha256=F7JlUwYjTFZAMRbBqKm6hxOIz-_IXYA8lBInOS4jbS4,100
-synapse_sdk/utils/file.py,sha256=y1POYiVg1r7DDcB4HAhnB0GdrjjIPDD6qNfzYupgUjM,4277
+synapse_sdk/utils/file.py,sha256=r7KBH2uKkpj7qWTWT75zHGgm5Vfw3XKzwgkrCBLqTUU,5163
 synapse_sdk/utils/module_loading.py,sha256=chHpU-BZjtYaTBD_q0T7LcKWtqKvYBS4L0lPlKkoMQ8,1020
 synapse_sdk/utils/storage.py,sha256=a8OVbd38ATr0El4G4kuV07lr_tJZrpIJBSy4GHb0qZ8,2581
 synapse_sdk/utils/string.py,sha256=rEwuZ9SAaZLcQ8TYiwNKr1h2u4CfnrQx7SUL8NWmChg,216
@@ -103,9 +103,9 @@ synapse_sdk/utils/pydantic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 synapse_sdk/utils/pydantic/config.py,sha256=1vYOcUI35GslfD1rrqhFkNXXJOXt4IDqOPSx9VWGfNE,123
 synapse_sdk/utils/pydantic/errors.py,sha256=0v0T12eQBr1KrFiEOBu6KMaPK4aPEGEC6etPJGoR5b4,1061
 synapse_sdk/utils/pydantic/validators.py,sha256=G47P8ObPhsePmd_QZDK8EdPnik2CbaYzr_N4Z6En8dc,193
-synapse_sdk-1.0.0a16.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0a16.dist-info/METADATA,sha256=Xied9eLdg3w-l21B9xpFvP-GUULIMIg2YEWOH4_9kZA,1049
-synapse_sdk-1.0.0a16.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-synapse_sdk-1.0.0a16.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-1.0.0a16.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0a16.dist-info/RECORD,,
+synapse_sdk-1.0.0a18.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0a18.dist-info/METADATA,sha256=f3pDUz--xd-2W8tTimkIm8Smw4wrMRDI7eVli95La1o,1049
+synapse_sdk-1.0.0a18.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+synapse_sdk-1.0.0a18.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-1.0.0a18.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0a18.dist-info/RECORD,,