synapse-sdk 1.0.0a15__py3-none-any.whl → 1.0.0a16__py3-none-any.whl

synapse_sdk/clients/base.py

@@ -62,7 +62,7 @@ class BaseClient:
         response = self._request('get', path, **kwargs)
         if url_conversion:
             if url_conversion['is_list']:
-                files_url_to_path_from_objs(response['results'], **url_conversion)
+                files_url_to_path_from_objs(response['results'], **url_conversion, is_async=True)
             else:
                 files_url_to_path_from_objs(response, **url_conversion)
         return response
@@ -80,14 +80,14 @@ class BaseClient:
         return self._request('delete', path, **kwargs)
 
     def _list(self, path, url_conversion=None, list_all=False, **kwargs):
-        response = self._get(path, url_conversion, **kwargs)
+        response = self._get(path, **kwargs)
         if list_all:
             return self._list_all(path, url_conversion, **kwargs), response['count']
         else:
             return response
 
-    def _list_all(self, path, url_conversion=None, **kwargs):
-        response = self._get(path, url_conversion, **kwargs)
+    def _list_all(self, path, url_conversion=None, params=None, **kwargs):
+        response = self._get(path, url_conversion, params=params, **kwargs)
         yield from response['results']
         if response['next']:
             yield from self._list_all(response['next'], url_conversion, **kwargs)

synapse_sdk/plugins/categories/base.py

@@ -92,13 +92,15 @@ class Action:
     @property
     def debug_modules(self):
         debug_modules = []
-        for module_path in self.envs.get('SYNAPSE_DEBUG_MODULES', '').split(','):
-            if module_path.startswith('https://'):  # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
-                module_url = module_path
-            else:
-                module_url = build_and_upload(module_path, self.plugin_storage_url)
-            debug_modules.append(module_url)
-        self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
+        if self.envs.get('SYNAPSE_DEBUG_MODULES'):
+            for module_path in self.envs['SYNAPSE_DEBUG_MODULES'].split(','):
+                # TODO ray에서 지원하는 remote uri 형식 (https, s3, gs) 모두 지원
+                if module_path.startswith('https://'):
+                    module_url = module_path
+                else:
+                    module_url = build_and_upload(module_path, self.plugin_storage_url)
+                debug_modules.append(module_url)
+            self.envs['SYNAPSE_DEBUG_MODULES'] = ','.join(debug_modules)
         return debug_modules
 
     def get_run(self):

synapse_sdk/utils/file.py

@@ -1,3 +1,4 @@
+import asyncio
 import hashlib
 import json
 import operator
@@ -6,6 +7,7 @@ from functools import reduce
 from pathlib import Path
 from urllib.parse import urlparse
 
+import aiohttp
 import requests
 import yaml
 
@@ -38,18 +40,69 @@ def files_url_to_path(files, coerce=None):
             files[file_name]['path'] = download_file(files[file_name].pop('url'), path_download, coerce=coerce)
 
 
-def files_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False):
+def files_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False, is_async=False):
+    if is_async:
+        asyncio.run(afiles_url_to_path_from_objs(objs, files_fields, coerce=coerce, is_list=is_list))
+    else:
+        if not is_list:
+            objs = [objs]
+
+        for obj in objs:
+            for files_field in files_fields:
+                try:
+                    files = reduce(operator.getitem, files_field.split('.'), obj)
+                    files_url_to_path(files, coerce=coerce)
+                except KeyError:
+                    pass
+
+
+async def adownload_file(url, path_download, name=None, coerce=None):
+    if name:
+        name += Path(url).suffix
+    else:
+        name = Path(url).name
+
+    name = urlparse(name).path
+    path = Path(path_download) / name
+    if not path.is_file():
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                with open(str(path), 'wb') as file:
+                    while chunk := await response.content.read(1024 * 64):
+                        file.write(chunk)
+
+    if coerce:
+        path = coerce(path)
+
+    return path
+
+
+async def afiles_url_to_path(files, coerce=None):
+    path_download = Path('/tmp/datamaker') / 'media'
+    path_download.mkdir(parents=True, exist_ok=True)
+    for file_name in files:
+        if isinstance(files[file_name], str):
+            files[file_name] = await adownload_file(files[file_name], path_download, coerce=coerce)
+        else:
+            files[file_name]['path'] = await adownload_file(files[file_name].pop('url'), path_download, coerce=coerce)
+
+
+async def afiles_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False):
     if not is_list:
         objs = [objs]
 
+    tasks = []
+
     for obj in objs:
         for files_field in files_fields:
             try:
                 files = reduce(operator.getitem, files_field.split('.'), obj)
-                files_url_to_path(files, coerce=coerce)
+                tasks.append(afiles_url_to_path(files, coerce=coerce))
             except KeyError:
                 pass
 
+    await asyncio.gather(*tasks)
+
 
 def get_dict_from_file(file_path):
     if isinstance(file_path, str):

{synapse_sdk-1.0.0a15.dist-info → synapse_sdk-1.0.0a16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse-sdk
-Version: 1.0.0a15
+Version: 1.0.0a16
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a15.dist-info → synapse_sdk-1.0.0a16.dist-info}/RECORD

@@ -8,7 +8,7 @@ synapse_sdk/loggers.py,sha256=RsDDOiOeUCih1XOkWQJseYdYCX_wt50AZJRe6aPf96Q,4004
 synapse_sdk/cli/__init__.py,sha256=WmYGW1qZEXXIGJe3SGr8QjOStY4svuZKK1Lp_aPvtPs,140
 synapse_sdk/cli/create_plugin.py,sha256=egbW_92WwxfHz50Gy4znX5Bf5fxDdQj3GFyd0l3Y3SY,228
 synapse_sdk/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/clients/base.py,sha256=b34uGFZyV13NVWPfycDK3FNz3W9gT5r4-oLQ_mVrDPQ,3327
+synapse_sdk/clients/base.py,sha256=un2vseaQ7YY6rVQqd5fIyzclg7VYLAEx2TwUkZof3ik,3354
 synapse_sdk/clients/exceptions.py,sha256=ylv7x10eOp4aA3a48jwonnvqvkiYwzJYXjkVkRTAjwk,220
 synapse_sdk/clients/utils.py,sha256=8pPJTdzHiRPSbZMoQYHAgR2BAMO6u_R_jMV6a2p34iQ,392
 synapse_sdk/clients/agent/__init__.py,sha256=Pz8_iTbIbnb7ywGJ3feqoZVmO2I3mEbwpWsISIxh0BU,1968
@@ -30,7 +30,7 @@ synapse_sdk/plugins/models.py,sha256=T3ZuTw7BZwMKpz2QGaErnEPetKRD9d4z3qceJkV363o
 synapse_sdk/plugins/upload.py,sha256=VJOotYMayylOH0lNoAGeGHRkLdhP7jnC_A0rFQMvQpQ,3228
 synapse_sdk/plugins/utils.py,sha256=UYkwxkmrs0-mRgQB63SkTGD1HpIQEIiHndzTcpdNaF4,2936
 synapse_sdk/plugins/categories/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/base.py,sha256=Q14bnUTdPLeNeFodcOBMr9Z0d8LM1VRIq-pl8vTKgTc,8093
+synapse_sdk/plugins/categories/base.py,sha256=5BrOWXbLiwZw4I1sfElQ5CF12kLDXY97b8MSvpdNIb0,8179
 synapse_sdk/plugins/categories/decorators.py,sha256=Gw6T-UHwpCKrSt596X-g2sZbY_Z1zbbogowClj7Pr5Q,518
 synapse_sdk/plugins/categories/registry.py,sha256=KdQR8SUlLT-3kgYzDNWawS1uJnAhrcw2j4zFaTpilRs,636
 synapse_sdk/plugins/categories/templates.py,sha256=FF5FerhkZMeW1YcKLY5cylC0SkWSYdJODA_Qcm4OGYQ,887
@@ -95,7 +95,7 @@ synapse_sdk/shared/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 synapse_sdk/shared/enums.py,sha256=WMZPag9deVF7VCXaQkLk7ly_uX1KwbNzRx9TdvgaeFE,138
 synapse_sdk/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/utils/debug.py,sha256=F7JlUwYjTFZAMRbBqKm6hxOIz-_IXYA8lBInOS4jbS4,100
-synapse_sdk/utils/file.py,sha256=OYch8D5Ub8oaGIIcpO6_R6-McpR3QJk7xlkaffhgd_c,2547
+synapse_sdk/utils/file.py,sha256=y1POYiVg1r7DDcB4HAhnB0GdrjjIPDD6qNfzYupgUjM,4277
 synapse_sdk/utils/module_loading.py,sha256=chHpU-BZjtYaTBD_q0T7LcKWtqKvYBS4L0lPlKkoMQ8,1020
 synapse_sdk/utils/storage.py,sha256=a8OVbd38ATr0El4G4kuV07lr_tJZrpIJBSy4GHb0qZ8,2581
 synapse_sdk/utils/string.py,sha256=rEwuZ9SAaZLcQ8TYiwNKr1h2u4CfnrQx7SUL8NWmChg,216
@@ -103,9 +103,9 @@ synapse_sdk/utils/pydantic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 synapse_sdk/utils/pydantic/config.py,sha256=1vYOcUI35GslfD1rrqhFkNXXJOXt4IDqOPSx9VWGfNE,123
 synapse_sdk/utils/pydantic/errors.py,sha256=0v0T12eQBr1KrFiEOBu6KMaPK4aPEGEC6etPJGoR5b4,1061
 synapse_sdk/utils/pydantic/validators.py,sha256=G47P8ObPhsePmd_QZDK8EdPnik2CbaYzr_N4Z6En8dc,193
-synapse_sdk-1.0.0a15.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0a15.dist-info/METADATA,sha256=Wh0N4QJ0Hy4XMeV9YUvIHoQUGRNYklUvVnPMv9E7Bbs,1049
-synapse_sdk-1.0.0a15.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-synapse_sdk-1.0.0a15.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-1.0.0a15.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0a15.dist-info/RECORD,,
+synapse_sdk-1.0.0a16.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0a16.dist-info/METADATA,sha256=Xied9eLdg3w-l21B9xpFvP-GUULIMIg2YEWOH4_9kZA,1049
+synapse_sdk-1.0.0a16.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+synapse_sdk-1.0.0a16.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-1.0.0a16.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0a16.dist-info/RECORD,,