synapse-sdk 1.0.0a24__py3-none-any.whl → 1.0.0a26__py3-none-any.whl

synapse_sdk/plugins/categories/neural_net/base/inference.py

@@ -1,10 +1,10 @@
 import tempfile
 
 import jwt
-from synapse_sdk.clients.backend import BackendClient
 from fastapi import FastAPI
 from ray import serve
 
+from synapse_sdk.clients.backend import BackendClient
 from synapse_sdk.utils.file import unarchive
 
 app = FastAPI()

synapse_sdk/types.py

@@ -0,0 +1,19 @@
+from typing import Any
+
+from pydantic import HttpUrl
+from pydantic_core import core_schema
+from pydantic_core.core_schema import ValidationInfo
+
+from synapse_sdk.utils.file import download_file, get_temp_path
+
+
+class FileField(str):
+    @classmethod
+    def __get_pydantic_core_schema__(cls, source_type: Any, _handler: callable) -> core_schema.CoreSchema:
+        return core_schema.with_info_before_validator_function(cls.validate, core_schema.str_schema())
+
+    @staticmethod
+    def validate(url: HttpUrl, info: ValidationInfo) -> str:
+        path_download = get_temp_path('media')
+        path_download.mkdir(parents=True, exist_ok=True)
+        return str(download_file(url, path_download))

synapse_sdk/utils/file.py

@@ -5,24 +5,35 @@ import operator
 import zipfile
 from functools import reduce
 from pathlib import Path
-from urllib.parse import urlparse
 
 import aiohttp
 import requests
 import yaml
 
+from synapse_sdk.utils.network import clean_url
+from synapse_sdk.utils.string import hash_text
+
+
+def download_file(url, path_download, name=None, coerce=None, use_cached=True):
+    chunk_size = 1024 * 1024 * 50
+    cleaned_url = clean_url(url)  # remove query params and fragment
 
-def download_file(url, path_download, name=None, coerce=None):
     if name:
-        name += Path(url).suffix
+        use_cached = False
     else:
-        name = Path(url).name
+        name = hash_text(cleaned_url)
+
+    name += Path(cleaned_url).suffix
 
-    name = urlparse(name).path
     path = Path(path_download) / name
-    if not path.is_file():
-        r = requests.get(url, allow_redirects=True)
-        open(str(path), 'wb').write(r.content)
+
+    if not use_cached or not path.is_file():
+        response = requests.get(url, allow_redirects=True, stream=True)
+        response.raise_for_status()
+
+        with path.open('wb') as file:
+            for chunk in response.iter_content(chunk_size=chunk_size):
+                file.write(chunk)
 
     if coerce:
         path = coerce(path)
@@ -62,19 +73,24 @@ def files_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False,
                     pass
 
 
-async def adownload_file(url, path_download, name=None, coerce=None):
+async def adownload_file(url, path_download, name=None, coerce=None, use_cached=True):
+    chunk_size = 1024 * 1024 * 50
+    cleaned_url = clean_url(url)  # remove query params and fragment
+
     if name:
-        name += Path(url).suffix
+        use_cached = False
     else:
-        name = Path(url).name
+        name = hash_text(cleaned_url)
+
+    name += Path(cleaned_url).suffix
 
-    name = urlparse(name).path
     path = Path(path_download) / name
-    if not path.is_file():
+
+    if not use_cached or not path.is_file():
         async with aiohttp.ClientSession() as session:
             async with session.get(url) as response:
-                with open(str(path), 'wb') as file:
-                    while chunk := await response.content.read(1024 * 64):
+                with path.open('wb') as file:
+                    while chunk := await response.content.read(chunk_size):
                         file.write(chunk)
 
     if coerce:

synapse_sdk/utils/network.py

@@ -0,0 +1,16 @@
+from urllib.parse import urlparse, urlunparse
+
+
+def clean_url(url, remove_query_params=True, remove_fragment=True):
+    parsed = urlparse(url)
+    query = '' if remove_query_params else parsed.query
+    fragment = '' if remove_fragment else parsed.fragment
+
+    return urlunparse((
+        parsed.scheme,
+        parsed.netloc,
+        parsed.path,
+        parsed.params,
+        query,
+        fragment,
+    ))

{synapse_sdk-1.0.0a24.dist-info → synapse_sdk-1.0.0a26.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: synapse-sdk
-Version: 1.0.0a24
+Version: 1.0.0a26
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a24.dist-info → synapse_sdk-1.0.0a26.dist-info}/RECORD

@@ -5,6 +5,7 @@ locale/ko/LC_MESSAGES/messages.po,sha256=TFii_RbURDH-Du_9ZQf3wNh-2briGk1IqY33-9G
 synapse_sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/i18n.py,sha256=VXMR-Zm_1hTAg9iPk3YZNNq-T1Bhx1J2fEtRT6kyYbg,766
 synapse_sdk/loggers.py,sha256=RsDDOiOeUCih1XOkWQJseYdYCX_wt50AZJRe6aPf96Q,4004
+synapse_sdk/types.py,sha256=khzn8KpgxFdn1SrpbcuX84m_Md1Mz_HIoUoPq8uok40,698
 synapse_sdk/cli/__init__.py,sha256=WmYGW1qZEXXIGJe3SGr8QjOStY4svuZKK1Lp_aPvtPs,140
 synapse_sdk/cli/create_plugin.py,sha256=egbW_92WwxfHz50Gy4znX5Bf5fxDdQj3GFyd0l3Y3SY,228
 synapse_sdk/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -54,7 +55,7 @@ synapse_sdk/plugins/categories/neural_net/actions/inference.py,sha256=0a655ELqNV
 synapse_sdk/plugins/categories/neural_net/actions/test.py,sha256=JY25eg-Fo6WbgtMkGoo_qNqoaZkp3AQNEypJmeGzEog,320
 synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=4zq2ryWjqDa6MyI2BGe3sAofMh2NeY3XAo-1gEFXBs4,5145
 synapse_sdk/plugins/categories/neural_net/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/neural_net/base/inference.py,sha256=OI0PcDCIdRaRAWc9HbaJyOJtph5BU4OeGU2SwhQQ1vY,1097
+synapse_sdk/plugins/categories/neural_net/base/inference.py,sha256=R5DASI6-5vzsjDOYxqeGGMBjnav5qHF4hNJT8zNUR3I,1097
 synapse_sdk/plugins/categories/neural_net/templates/config.yaml,sha256=dXKB1hO53hDZB73xnxLVCNQl8Sm7svMmVmuMrOCQmEU,343
 synapse_sdk/plugins/categories/neural_net/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/templates/plugin/inference.py,sha256=InfqKWJYi6sqiUnfPKHC5KYGhxckDaWZNQ202u-uVP4,366
@@ -98,17 +99,18 @@ synapse_sdk/shared/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 synapse_sdk/shared/enums.py,sha256=WMZPag9deVF7VCXaQkLk7ly_uX1KwbNzRx9TdvgaeFE,138
 synapse_sdk/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/utils/debug.py,sha256=F7JlUwYjTFZAMRbBqKm6hxOIz-_IXYA8lBInOS4jbS4,100
-synapse_sdk/utils/file.py,sha256=jfPHIwKBDE4akuckJsrVfOXBjWgEncFlzE5IUMSGmuE,5153
+synapse_sdk/utils/file.py,sha256=eF1GDxjPaq2QOiKKCHSzgCkBRYSBSwP4rKOAApAQR3E,5661
 synapse_sdk/utils/module_loading.py,sha256=chHpU-BZjtYaTBD_q0T7LcKWtqKvYBS4L0lPlKkoMQ8,1020
+synapse_sdk/utils/network.py,sha256=wg-oFM0gKK5REqIUO8d-x9yXJfqbnkSbbF0_qyxpwz4,412
 synapse_sdk/utils/storage.py,sha256=a8OVbd38ATr0El4G4kuV07lr_tJZrpIJBSy4GHb0qZ8,2581
 synapse_sdk/utils/string.py,sha256=rEwuZ9SAaZLcQ8TYiwNKr1h2u4CfnrQx7SUL8NWmChg,216
 synapse_sdk/utils/pydantic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/utils/pydantic/config.py,sha256=1vYOcUI35GslfD1rrqhFkNXXJOXt4IDqOPSx9VWGfNE,123
 synapse_sdk/utils/pydantic/errors.py,sha256=0v0T12eQBr1KrFiEOBu6KMaPK4aPEGEC6etPJGoR5b4,1061
 synapse_sdk/utils/pydantic/validators.py,sha256=G47P8ObPhsePmd_QZDK8EdPnik2CbaYzr_N4Z6En8dc,193
-synapse_sdk-1.0.0a24.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0a24.dist-info/METADATA,sha256=5EQXB_zI67ADun4KdzPXSGn9RRr9vn_NZVp77Aq4NSM,1070
-synapse_sdk-1.0.0a24.dist-info/WHEEL,sha256=A3WOREP4zgxI0fKrHUG8DC8013e3dK3n7a6HDbcEIwE,91
-synapse_sdk-1.0.0a24.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-1.0.0a24.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0a24.dist-info/RECORD,,
+synapse_sdk-1.0.0a26.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0a26.dist-info/METADATA,sha256=DnO5NIlrAWVsCnkh8YXP-5ujNpt_NxijYyOeJ_ZQF2M,1070
+synapse_sdk-1.0.0a26.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+synapse_sdk-1.0.0a26.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-1.0.0a26.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0a26.dist-info/RECORD,,

{synapse_sdk-1.0.0a24.dist-info → synapse_sdk-1.0.0a26.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.7.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any