synapse-sdk 1.0.0a24__tar.gz → 1.0.0a26__tar.gz

{synapse_sdk-1.0.0a24/synapse_sdk.egg-info → synapse_sdk-1.0.0a26}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: synapse-sdk
-Version: 1.0.0a24
+Version: 1.0.0a26
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

synapse_sdk-1.0.0a26/synapse_sdk/types.py

@@ -0,0 +1,19 @@
+from typing import Any
+
+from pydantic import HttpUrl
+from pydantic_core import core_schema
+from pydantic_core.core_schema import ValidationInfo
+
+from synapse_sdk.utils.file import download_file, get_temp_path
+
+
+class FileField(str):
+    @classmethod
+    def __get_pydantic_core_schema__(cls, source_type: Any, _handler: callable) -> core_schema.CoreSchema:
+        return core_schema.with_info_before_validator_function(cls.validate, core_schema.str_schema())
+
+    @staticmethod
+    def validate(url: HttpUrl, info: ValidationInfo) -> str:
+        path_download = get_temp_path('media')
+        path_download.mkdir(parents=True, exist_ok=True)
+        return str(download_file(url, path_download))

{synapse_sdk-1.0.0a24 → synapse_sdk-1.0.0a26}/synapse_sdk/utils/file.py

@@ -5,24 +5,35 @@ import operator
 import zipfile
 from functools import reduce
 from pathlib import Path
-from urllib.parse import urlparse
 
 import aiohttp
 import requests
 import yaml
 
+from synapse_sdk.utils.network import clean_url
+from synapse_sdk.utils.string import hash_text
+
+
+def download_file(url, path_download, name=None, coerce=None, use_cached=True):
+    chunk_size = 1024 * 1024 * 50
+    cleaned_url = clean_url(url)  # remove query params and fragment
 
-def download_file(url, path_download, name=None, coerce=None):
     if name:
-        name += Path(url).suffix
+        use_cached = False
     else:
-        name = Path(url).name
+        name = hash_text(cleaned_url)
+
+    name += Path(cleaned_url).suffix
 
-    name = urlparse(name).path
     path = Path(path_download) / name
-    if not path.is_file():
-        r = requests.get(url, allow_redirects=True)
-        open(str(path), 'wb').write(r.content)
+
+    if not use_cached or not path.is_file():
+        response = requests.get(url, allow_redirects=True, stream=True)
+        response.raise_for_status()
+
+        with path.open('wb') as file:
+            for chunk in response.iter_content(chunk_size=chunk_size):
+                file.write(chunk)
 
     if coerce:
         path = coerce(path)
@@ -62,19 +73,24 @@ def files_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False,
                     pass
 
 
-async def adownload_file(url, path_download, name=None, coerce=None):
+async def adownload_file(url, path_download, name=None, coerce=None, use_cached=True):
+    chunk_size = 1024 * 1024 * 50
+    cleaned_url = clean_url(url)  # remove query params and fragment
+
     if name:
-        name += Path(url).suffix
+        use_cached = False
     else:
-        name = Path(url).name
+        name = hash_text(cleaned_url)
+
+    name += Path(cleaned_url).suffix
 
-    name = urlparse(name).path
     path = Path(path_download) / name
-    if not path.is_file():
+
+    if not use_cached or not path.is_file():
         async with aiohttp.ClientSession() as session:
             async with session.get(url) as response:
-                with open(str(path), 'wb') as file:
-                    while chunk := await response.content.read(1024 * 64):
+                with path.open('wb') as file:
+                    while chunk := await response.content.read(chunk_size):
                         file.write(chunk)
 
     if coerce:

synapse_sdk-1.0.0a26/synapse_sdk/utils/network.py

@@ -0,0 +1,16 @@
+from urllib.parse import urlparse, urlunparse
+
+
+def clean_url(url, remove_query_params=True, remove_fragment=True):
+    parsed = urlparse(url)
+    query = '' if remove_query_params else parsed.query
+    fragment = '' if remove_fragment else parsed.fragment
+
+    return urlunparse((
+        parsed.scheme,
+        parsed.netloc,
+        parsed.path,
+        parsed.params,
+        query,
+        fragment,
+    ))

{synapse_sdk-1.0.0a24 → synapse_sdk-1.0.0a26/synapse_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: synapse-sdk
-Version: 1.0.0a24
+Version: 1.0.0a26
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a24 → synapse_sdk-1.0.0a26}/synapse_sdk.egg-info/SOURCES.txt

@@ -11,6 +11,7 @@ locale/ko/LC_MESSAGES/messages.po
 synapse_sdk/__init__.py
 synapse_sdk/i18n.py
 synapse_sdk/loggers.py
+synapse_sdk/types.py
 synapse_sdk.egg-info/PKG-INFO
 synapse_sdk.egg-info/SOURCES.txt
 synapse_sdk.egg-info/dependency_links.txt
@@ -116,6 +117,7 @@ synapse_sdk/utils/__init__.py
 synapse_sdk/utils/debug.py
 synapse_sdk/utils/file.py
 synapse_sdk/utils/module_loading.py
+synapse_sdk/utils/network.py
 synapse_sdk/utils/storage.py
 synapse_sdk/utils/string.py
 synapse_sdk/utils/pydantic/__init__.py

{synapse_sdk-1.0.0a24 → synapse_sdk-1.0.0a26}/synapse_sdk/plugins/categories/neural_net/base/inference.py

@@ -1,10 +1,10 @@
 import tempfile
 
 import jwt
-from synapse_sdk.clients.backend import BackendClient
 from fastapi import FastAPI
 from ray import serve
 
+from synapse_sdk.clients.backend import BackendClient
 from synapse_sdk.utils.file import unarchive
 
 app = FastAPI()