synapse-sdk 1.0.0a13__py3-none-any.whl → 2025.11.7__py3-none-any.whl

synapse_sdk/utils/http.py

@@ -0,0 +1,138 @@
+import logging
+import os
+import tempfile
+import threading
+import time
+import uuid
+from contextlib import contextmanager
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+
+import requests
+
+from synapse_sdk.utils.network import get_available_ports_host
+
+
+class SingleFileHttpServer(SimpleHTTPRequestHandler):
+    """
+    Custom HTTP request handler that serves a single specified file
+    regardless of the request path.
+    """
+
+    def __init__(self, *args, file_path=None, content_type=None, random_path=None, **kwargs):
+        self.file_path = file_path
+        self.content_type = content_type
+        self.random_path = random_path
+        super().__init__(*args, **kwargs)
+
+    def do_GET(self):
+        """Handle GET requests by serving the single file."""
+        try:
+            # Check if the path matches our random path
+            if self.random_path and self.path == f'/{self.random_path}':
+                self.send_response(200)
+                if self.content_type:
+                    self.send_header('Content-type', self.content_type)
+                self.send_header('Content-Length', str(os.path.getsize(self.file_path)))
+                self.end_headers()
+
+                with open(self.file_path, 'rb') as file:
+                    self.wfile.write(file.read())
+            elif self.path == '/':
+                # Redirect root to the random path
+                self.send_response(302)
+                self.send_header('Location', f'/{self.random_path}')
+                self.end_headers()
+            else:
+                self.send_error(404, 'File not found')
+
+        except Exception as e:
+            self.send_error(500, str(e))
+
+
+@contextmanager
+def temp_file_server(image=None, file_path=None, format='JPEG', host='0.0.0.0', port=None, content_type=None):
+    """
+    Context manager that serves a file temporarily via HTTP.
+
+    Args:
+        image: A PIL Image object to serve (optional)
+        file_path: Path to an existing file to serve (optional - used if image not provided)
+        format: Image format when saving a PIL Image (default: "JPEG")
+        host: Host to serve on (default: "0.0.0.0")
+        port: Port to serve on (default: auto-selected free port)
+        content_type: Content type header (default: auto-detected based on format)
+
+    Returns:
+        URL where the file is being served
+
+    Usage:
+        with temp_file_serve(image=my_pillow_img) as url:
+            # use url to access the image
+            print(f"Image available at: {url}")
+    """
+    if image is None and file_path is None:
+        raise ValueError('Either image or file_path must be provided')
+
+    # Use a free port if none specified
+    if port is None:
+        port = get_available_ports_host(start_port=8991, end_port=8999)
+
+    temp_dir = None
+    temp_file_path = None
+
+    try:
+        random_filename = f'{uuid.uuid4().hex}'
+
+        if image is not None:
+            temp_dir = tempfile.mkdtemp()
+            ext_map = {'JPEG': '.jpg', 'PNG': '.png', 'GIF': '.gif', 'WEBP': '.webp'}
+            content_type_map = {'JPEG': 'image/jpeg', 'PNG': 'image/png', 'GIF': 'image/gif', 'WEBP': 'image/webp'}
+
+            ext = ext_map.get(format, '.jpg')
+            if content_type is None:
+                content_type = content_type_map.get(format, 'image/jpeg')
+
+            temp_file_path = os.path.join(temp_dir, f'temp_image{ext}')
+            image.save(temp_file_path, format=format)
+            file_path = temp_file_path
+            random_filename += ext
+        else:
+            _, ext = os.path.splitext(file_path)
+            random_filename += ext
+
+        def handler(*args, **kwargs):
+            return SingleFileHttpServer(
+                *args, file_path=file_path, content_type=content_type, random_path=random_filename, **kwargs
+            )
+
+        server = HTTPServer((host, port), handler)
+
+        server_thread = threading.Thread(target=server.serve_forever)
+        server_thread.daemon = True
+        server_thread.start()
+
+        url = f'http://localhost:{port}/{random_filename}'
+
+        while True:
+            try:
+                response = requests.get(url)
+                if response.status_code == 200:
+                    break
+            except requests.exceptions.ConnectionError:
+                pass
+            time.sleep(0.01)
+
+        try:
+            yield url
+        finally:
+            server.shutdown()
+            server.server_close()
+
+    finally:
+        if temp_dir:
+            try:
+                if temp_file_path and os.path.exists(temp_file_path):
+                    os.unlink(temp_file_path)
+                os.rmdir(temp_dir)
+            except Exception as e:
+                logging.warning(f'Error cleaning up temporary files: {e}')

synapse_sdk/utils/network.py

@@ -0,0 +1,309 @@
+import asyncio
+import queue as queue_module
+import re
+import ssl
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from typing import Any, Dict, Generator, Optional
+from urllib.parse import urlparse, urlunparse
+
+import requests
+
+from synapse_sdk.clients.exceptions import ClientError
+
+
+@dataclass
+class StreamLimits:
+    """Configuration for streaming limits."""
+
+    max_messages: int = 10000
+    max_lines: int = 50000
+    max_bytes: int = 50 * 1024 * 1024  # 50MB
+    max_message_size: int = 10240  # 10KB
+    queue_size: int = 1000
+    exception_queue_size: int = 10
+
+
+def validate_resource_id(resource_id: Any, resource_name: str = 'resource') -> str:
+    """Validate resource ID to prevent injection attacks."""
+    if not resource_id:
+        raise ClientError(400, f'{resource_name} ID cannot be empty')
+
+    # Allow numeric IDs and UUID formats
+    id_str = str(resource_id)
+    if not re.match(r'^[a-zA-Z0-9\-_]+$', id_str):
+        raise ClientError(400, f'Invalid {resource_name} ID format')
+
+    if len(id_str) > 100:
+        raise ClientError(400, f'{resource_name} ID too long')
+
+    return id_str
+
+
+def validate_timeout(timeout: Any, max_timeout: int = 300) -> float:
+    """Validate timeout value with bounds checking."""
+    if not isinstance(timeout, (int, float)) or timeout <= 0:
+        raise ClientError(400, 'Timeout must be a positive number')
+
+    if timeout > max_timeout:
+        raise ClientError(400, f'Timeout cannot exceed {max_timeout} seconds')
+
+    return float(timeout)
+
+
+def sanitize_error_message(error_msg: str, context: str = '') -> str:
+    """Sanitize error messages to prevent information disclosure."""
+    sanitized = str(error_msg)[:100]
+    # Remove any potential sensitive information
+    sanitized = re.sub(r'["\']([^"\']*)["\']', '"[REDACTED]"', sanitized)
+
+    if context:
+        return f'{context}: {sanitized}'
+    return sanitized
+
+
+def http_to_websocket_url(url: str) -> str:
+    """Convert HTTP/HTTPS URL to WebSocket URL safely."""
+    try:
+        parsed = urlparse(url)
+        if parsed.scheme == 'http':
+            ws_scheme = 'ws'
+        elif parsed.scheme == 'https':
+            ws_scheme = 'wss'
+        else:
+            raise ClientError(400, f'Invalid URL scheme: {parsed.scheme}')
+
+        ws_url = urlunparse((ws_scheme, parsed.netloc, parsed.path, parsed.params, parsed.query, parsed.fragment))
+        return ws_url
+    except Exception as e:
+        raise ClientError(400, f'Invalid URL format: {str(e)[:50]}')
+
+
+def check_library_available(library_name: str) -> bool:
+    """Check if optional library is available."""
+    try:
+        __import__(library_name)
+        return True
+    except ImportError:
+        return False
+
+
+class WebSocketStreamManager:
+    """Manages secure WebSocket streaming with rate limiting and error handling."""
+
+    def __init__(self, thread_pool: ThreadPoolExecutor, limits: Optional[StreamLimits] = None):
+        self.thread_pool = thread_pool
+        self.limits = limits or StreamLimits()
+
+    def stream_logs(
+        self, ws_url: str, headers: Dict[str, str], timeout: float, context: str
+    ) -> Generator[str, None, None]:
+        """Stream logs from WebSocket with proper error handling and cleanup."""
+        if not check_library_available('websockets'):
+            raise ClientError(500, 'websockets library not available for WebSocket connections')
+
+        try:
+            import websockets
+
+            # Use bounded queues to prevent memory exhaustion
+            message_queue = queue_module.Queue(maxsize=self.limits.queue_size)
+            exception_queue = queue_module.Queue(maxsize=self.limits.exception_queue_size)
+
+            async def websocket_client():
+                try:
+                    # Add SSL verification and proper timeouts
+                    connect_kwargs = {
+                        'extra_headers': headers,
+                        'close_timeout': timeout,
+                        'ping_timeout': timeout,
+                        'ping_interval': timeout // 2,
+                    }
+
+                    # For secure connections, add SSL context
+                    if ws_url.startswith('wss://'):
+                        ssl_context = ssl.create_default_context()
+                        ssl_context.check_hostname = True
+                        ssl_context.verify_mode = ssl.CERT_REQUIRED
+                        connect_kwargs['ssl'] = ssl_context
+
+                    async with websockets.connect(ws_url, **connect_kwargs) as websocket:
+                        message_count = 0
+
+                        async for message in websocket:
+                            message_count += 1
+                            if message_count > self.limits.max_messages:
+                                exception_queue.put_nowait(ClientError(429, f'Message limit exceeded for {context}'))
+                                break
+
+                            # Validate message size
+                            if len(str(message)) > self.limits.max_message_size:
+                                continue
+
+                            try:
+                                message_queue.put_nowait(f'{message}\n')
+                            except queue_module.Full:
+                                exception_queue.put_nowait(ClientError(429, f'Message queue full for {context}'))
+                                break
+
+                        message_queue.put_nowait(None)  # Signal end
+
+                except websockets.exceptions.ConnectionClosed:
+                    exception_queue.put_nowait(ClientError(503, f'WebSocket connection closed for {context}'))
+                except asyncio.TimeoutError:
+                    exception_queue.put_nowait(ClientError(408, f'WebSocket timed out for {context}'))
+                except Exception as e:
+                    sanitized_error = sanitize_error_message(str(e), context)
+                    exception_queue.put_nowait(ClientError(500, sanitized_error))
+
+            # Use thread pool instead of raw threading
+            future = self.thread_pool.submit(lambda: asyncio.run(websocket_client()))
+
+            # Yield messages with proper cleanup
+            try:
+                while True:
+                    # Check for exceptions first
+                    try:
+                        exception = exception_queue.get_nowait()
+                        raise exception
+                    except queue_module.Empty:
+                        pass
+
+                    # Get message with timeout
+                    try:
+                        message = message_queue.get(timeout=1.0)
+                        if message is None:  # End signal
+                            break
+                        yield message
+                    except queue_module.Empty:
+                        # Check if future is done
+                        if future.done():
+                            try:
+                                future.result()  # This will raise any exception
+                                break  # Normal completion
+                            except Exception:
+                                break  # Error already in queue
+                        continue
+
+            finally:
+                # Cleanup: cancel future if still running
+                if not future.done():
+                    future.cancel()
+
+        except ImportError:
+            raise ClientError(500, 'websockets library not available for WebSocket connections')
+        except Exception as e:
+            if isinstance(e, ClientError):
+                raise
+            sanitized_error = sanitize_error_message(str(e), context)
+            raise ClientError(500, sanitized_error)
+
+
+class HTTPStreamManager:
+    """Manages HTTP streaming with rate limiting and proper resource cleanup."""
+
+    def __init__(self, requests_session: requests.Session, limits: Optional[StreamLimits] = None):
+        self.requests_session = requests_session
+        self.limits = limits or StreamLimits()
+
+    def stream_logs(
+        self, url: str, headers: Dict[str, str], timeout: tuple, context: str
+    ) -> Generator[str, None, None]:
+        """Stream logs from HTTP endpoint with proper error handling and cleanup."""
+        response = None
+        try:
+            # Use timeout for streaming to prevent hanging
+            response = self.requests_session.get(url, headers=headers, stream=True, timeout=timeout)
+            response.raise_for_status()
+
+            # Set up streaming with timeout and size limits
+            line_count = 0
+            total_bytes = 0
+
+            try:
+                for line in response.iter_lines(decode_unicode=True, chunk_size=1024):
+                    if line:
+                        line_count += 1
+                        total_bytes += len(line.encode('utf-8'))
+
+                        # Rate limiting checks
+                        if line_count > self.limits.max_lines:
+                            raise ClientError(429, f'Line limit exceeded for {context}')
+
+                        if total_bytes > self.limits.max_bytes:
+                            raise ClientError(429, f'Size limit exceeded for {context}')
+
+                        # Validate line size
+                        if len(line) > self.limits.max_message_size:
+                            continue
+
+                        yield f'{line}\n'
+
+            except requests.exceptions.ChunkedEncodingError:
+                raise ClientError(503, f'Log stream interrupted for {context}')
+            except requests.exceptions.ReadTimeout:
+                raise ClientError(408, f'Log stream timed out for {context}')
+
+        except requests.exceptions.ConnectTimeout:
+            raise ClientError(408, f'Failed to connect to log stream for {context}')
+        except requests.exceptions.ReadTimeout:
+            raise ClientError(408, f'Log stream read timeout for {context}')
+        except requests.exceptions.ConnectionError as e:
+            if 'Connection refused' in str(e):
+                raise ClientError(503, f'Agent connection refused for {context}')
+            else:
+                sanitized_error = sanitize_error_message(str(e), context)
+                raise ClientError(503, f'Agent connection error: {sanitized_error}')
+        except requests.exceptions.HTTPError as e:
+            if hasattr(e.response, 'status_code'):
+                status_code = e.response.status_code
+            else:
+                status_code = 500
+            raise ClientError(status_code, f'HTTP error streaming logs for {context}')
+        except Exception as e:
+            if isinstance(e, ClientError):
+                raise
+            sanitized_error = sanitize_error_message(str(e), context)
+            raise ClientError(500, sanitized_error)
+        finally:
+            # Ensure response is properly closed
+            if response is not None:
+                try:
+                    response.close()
+                except Exception:
+                    pass  # Ignore cleanup errors
+
+
+def clean_url(url, remove_query_params=True, remove_fragment=True):
+    parsed = urlparse(url)
+    query = '' if remove_query_params else parsed.query
+    fragment = '' if remove_fragment else parsed.fragment
+
+    return urlunparse((
+        parsed.scheme,
+        parsed.netloc,
+        parsed.path,
+        parsed.params,
+        query,
+        fragment,
+    ))
+
+
+def get_available_ports_host(start_port=8900, end_port=8990):
+    import nmap
+
+    nm = nmap.PortScanner()
+
+    scan_range = f'{start_port}-{end_port}'
+    nm.scan(hosts='host.docker.internal', arguments=f'-p {scan_range}')
+
+    try:
+        open_ports = nm['host.docker.internal']['tcp'].keys()
+        open_ports = [int(port) for port in open_ports]
+    except KeyError:
+        open_ports = []
+
+    for port in range(start_port, end_port + 1):
+        if port not in open_ports:
+            return port
+
+    raise IOError(f'No free ports available in range {start_port}-{end_port}')

synapse_sdk/utils/storage/__init__.py

@@ -0,0 +1,72 @@
+from pathlib import Path
+from urllib.parse import urlparse
+
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.utils.storage.registry import STORAGE_PROVIDERS
+
+
+def get_storage(connection_param: str | dict):
+    """Get storage class with connection param.
+
+    Args:
+        connection_param (str | dict): The connection param for the Storage provider.
+
+    Returns:
+        BaseStorage: The storage class object with connection param.
+    """
+    storage_scheme = None
+    if isinstance(connection_param, dict):
+        storage_scheme = connection_param['provider']
+    else:
+        storage_scheme = urlparse(connection_param).scheme
+
+    assert storage_scheme in STORAGE_PROVIDERS.keys(), _(
+        f'Storage provider not supported. Got scheme: {storage_scheme}. '
+        f'Valid schemes: {", ".join(STORAGE_PROVIDERS.keys())}. '
+        f'Full connection_param: {connection_param}'
+    )
+    return STORAGE_PROVIDERS[storage_scheme](connection_param)
+
+
+def get_pathlib(storage_config: str | dict, path_root: str) -> Path:
+    """Get pathlib object with synapse-backend storage config.
+
+    Args:
+        storage_config (str | dict): The storage config by synapse-backend storage api.
+        path_root (ㅔstr): The path root.
+
+    Returns:
+        pathlib.Path: The pathlib object.
+    """
+    storage_class = get_storage(storage_config)
+    return storage_class.get_pathlib(path_root)
+
+
+def get_path_file_count(storage_config: str | dict, path_root: str) -> int:
+    """Get the file count in the path.
+
+    Args:
+        storage_config (str | dict): The storage config by synapse-backend storage api.
+        path (str): The path.
+
+    Returns:
+        int: The file count in the path.
+    """
+    storage_class = get_storage(storage_config)
+    pathlib_obj = storage_class.get_pathlib(path_root)
+    return storage_class.get_path_file_count(pathlib_obj)
+
+
+def get_path_total_size(storage_config: str | dict, path_root: str) -> int:
+    """Get total size of the files in the path.
+
+    Args:
+        storage_config (str | dict): The storage config by synapse-backend storage api.
+        path (str): The path.
+
+    Returns:
+        int: The total size of the files in the path.
+    """
+    storage_class = get_storage(storage_config)
+    pathlib_obj = storage_class.get_pathlib(path_root)
+    return storage_class.get_path_total_size(pathlib_obj)

synapse_sdk/utils/storage/providers/__init__.py

@@ -0,0 +1,183 @@
+from urllib.parse import parse_qs, urlparse
+
+
+class BaseStorage:
+    """Base storage provider class for all storage implementations.
+
+    This is an abstract base class that defines the interface for storage providers.
+    All storage providers (S3, GCP, SFTP, HTTP, FileSystem) must inherit from this
+    class and implement all the abstract methods.
+
+    Args:
+        connection_params (str | dict): The connection parameters. Can be either:
+            - A URL string (e.g., 's3://bucket/path?access_key=key&secret_key=secret')
+            - A dict with 'provider' and 'configuration' keys
+
+    Attributes:
+        url: Parsed URL object if connection_params was a string
+        base_path: Base path for storage operations (provider-specific)
+        options: Storage-specific options
+        query_params: Parsed query parameters from URL or configuration dict
+        OPTION_CASTS: Dictionary mapping option names to type casting functions
+
+    Example:
+        >>> # URL-based initialization
+        >>> storage = SomeStorage('s3://bucket/path?access_key=key&secret_key=secret')
+        >>>
+        >>> # Dict-based initialization
+        >>> config = {
+        ...     'provider': 's3',
+        ...     'configuration': {
+        ...         'bucket_name': 'my-bucket',
+        ...         'access_key': 'key',
+        ...         'secret_key': 'secret'
+        ...     }
+        ... }
+        >>> storage = SomeStorage(config)
+    """
+
+    url = None
+    options = None
+    OPTION_CASTS = {}
+
+    def __init__(self, connection_params: str | dict):
+        """Initialize the storage provider with connection parameters.
+
+        Args:
+            connection_params (str | dict): Connection parameters for the storage provider.
+                If string, should be a valid URL with scheme and query parameters.
+                If dict, should contain 'provider' and 'configuration' keys.
+        """
+        self.url = None
+
+        if isinstance(connection_params, dict):
+            self.query_params = connection_params['configuration']
+        else:
+            self.url = urlparse(connection_params)
+            self.query_params = self.url_querystring_to_dict()
+
+    def url_querystring_to_dict(self):
+        """Parse URL query string into a dictionary with proper type casting.
+
+        Converts query parameters from the URL into a dictionary, applying
+        type casting based on OPTION_CASTS mapping. Single-value lists are
+        flattened to scalar values.
+
+        Returns:
+            dict: Parsed and type-cast query parameters.
+        """
+        query_string = self.url.query
+
+        query_dict = parse_qs(query_string)
+
+        for key, value in query_dict.items():
+            if len(value) == 1:
+                query_dict[key] = value[0]
+
+        return {
+            key: self.OPTION_CASTS[key](value) if key in self.OPTION_CASTS else value
+            for key, value in query_dict.items()
+        }
+
+    def upload(self, source, target):
+        """Upload a file from source to target location.
+
+        Args:
+            source (str): Path to the source file on local filesystem.
+            target (str): Target path in the storage provider.
+
+        Returns:
+            str: URL or identifier of the uploaded file.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+        """
+        raise NotImplementedError
+
+    def exists(self, target):
+        """Check if a file exists at the target location.
+
+        Args:
+            target (str): Target path in the storage provider.
+
+        Returns:
+            bool: True if the file exists, False otherwise.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+        """
+        raise NotImplementedError
+
+    def get_url(self, target):
+        """Get the URL for accessing a file at the target location.
+
+        Args:
+            target (str): Target path in the storage provider.
+
+        Returns:
+            str: URL that can be used to access the file.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+        """
+        raise NotImplementedError
+
+    def get_pathlib(self, path):
+        """Get the path as a pathlib-compatible object.
+
+        Args:
+            path (str): The path to convert, relative to the storage root.
+
+        Returns:
+            pathlib.Path or UPath: A pathlib-compatible object representing the path.
+            The exact type depends on the storage provider implementation.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+
+        Note:
+            Different storage providers may return different path object types:
+            - FileSystemStorage returns pathlib.Path objects
+            - Cloud storage providers typically return UPath objects
+        """
+        raise NotImplementedError
+
+    def get_path_file_count(self, pathlib_obj):
+        """Get the total number of files in the given path.
+
+        Args:
+            pathlib_obj (Path | UPath): The path object to count files in.
+                Should be obtained from get_pathlib().
+
+        Returns:
+            int: The total number of files found recursively in the path.
+                Returns 1 for individual files, 0 for non-existent paths.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+
+        Note:
+            This method counts files recursively, including files in subdirectories.
+            Directories themselves are not counted, only regular files.
+        """
+        raise NotImplementedError
+
+    def get_path_total_size(self, pathlib_obj):
+        """Get the total size of all files in the given path.
+
+        Args:
+            pathlib_obj (Path | UPath): The path object to calculate size for.
+                Should be obtained from get_pathlib().
+
+        Returns:
+            int: The total size in bytes of all files found recursively in the path.
+                Returns the file size for individual files, 0 for non-existent paths.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+
+        Note:
+            This method calculates the total size recursively, including files
+            in subdirectories. Only regular files contribute to the total size.
+        """
+        raise NotImplementedError