synapse-sdk 1.0.0a35__py3-none-any.whl → 2025.11.7__py3-none-any.whl

synapse_sdk/utils/network.py

@@ -1,5 +1,277 @@
+import asyncio
+import queue as queue_module
+import re
+import ssl
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from typing import Any, Dict, Generator, Optional
 from urllib.parse import urlparse, urlunparse
 
+import requests
+
+from synapse_sdk.clients.exceptions import ClientError
+
+
+@dataclass
+class StreamLimits:
+    """Configuration for streaming limits."""
+
+    max_messages: int = 10000
+    max_lines: int = 50000
+    max_bytes: int = 50 * 1024 * 1024  # 50MB
+    max_message_size: int = 10240  # 10KB
+    queue_size: int = 1000
+    exception_queue_size: int = 10
+
+
+def validate_resource_id(resource_id: Any, resource_name: str = 'resource') -> str:
+    """Validate resource ID to prevent injection attacks."""
+    if not resource_id:
+        raise ClientError(400, f'{resource_name} ID cannot be empty')
+
+    # Allow numeric IDs and UUID formats
+    id_str = str(resource_id)
+    if not re.match(r'^[a-zA-Z0-9\-_]+$', id_str):
+        raise ClientError(400, f'Invalid {resource_name} ID format')
+
+    if len(id_str) > 100:
+        raise ClientError(400, f'{resource_name} ID too long')
+
+    return id_str
+
+
+def validate_timeout(timeout: Any, max_timeout: int = 300) -> float:
+    """Validate timeout value with bounds checking."""
+    if not isinstance(timeout, (int, float)) or timeout <= 0:
+        raise ClientError(400, 'Timeout must be a positive number')
+
+    if timeout > max_timeout:
+        raise ClientError(400, f'Timeout cannot exceed {max_timeout} seconds')
+
+    return float(timeout)
+
+
+def sanitize_error_message(error_msg: str, context: str = '') -> str:
+    """Sanitize error messages to prevent information disclosure."""
+    sanitized = str(error_msg)[:100]
+    # Remove any potential sensitive information
+    sanitized = re.sub(r'["\']([^"\']*)["\']', '"[REDACTED]"', sanitized)
+
+    if context:
+        return f'{context}: {sanitized}'
+    return sanitized
+
+
+def http_to_websocket_url(url: str) -> str:
+    """Convert HTTP/HTTPS URL to WebSocket URL safely."""
+    try:
+        parsed = urlparse(url)
+        if parsed.scheme == 'http':
+            ws_scheme = 'ws'
+        elif parsed.scheme == 'https':
+            ws_scheme = 'wss'
+        else:
+            raise ClientError(400, f'Invalid URL scheme: {parsed.scheme}')
+
+        ws_url = urlunparse((ws_scheme, parsed.netloc, parsed.path, parsed.params, parsed.query, parsed.fragment))
+        return ws_url
+    except Exception as e:
+        raise ClientError(400, f'Invalid URL format: {str(e)[:50]}')
+
+
+def check_library_available(library_name: str) -> bool:
+    """Check if optional library is available."""
+    try:
+        __import__(library_name)
+        return True
+    except ImportError:
+        return False
+
+
+class WebSocketStreamManager:
+    """Manages secure WebSocket streaming with rate limiting and error handling."""
+
+    def __init__(self, thread_pool: ThreadPoolExecutor, limits: Optional[StreamLimits] = None):
+        self.thread_pool = thread_pool
+        self.limits = limits or StreamLimits()
+
+    def stream_logs(
+        self, ws_url: str, headers: Dict[str, str], timeout: float, context: str
+    ) -> Generator[str, None, None]:
+        """Stream logs from WebSocket with proper error handling and cleanup."""
+        if not check_library_available('websockets'):
+            raise ClientError(500, 'websockets library not available for WebSocket connections')
+
+        try:
+            import websockets
+
+            # Use bounded queues to prevent memory exhaustion
+            message_queue = queue_module.Queue(maxsize=self.limits.queue_size)
+            exception_queue = queue_module.Queue(maxsize=self.limits.exception_queue_size)
+
+            async def websocket_client():
+                try:
+                    # Add SSL verification and proper timeouts
+                    connect_kwargs = {
+                        'extra_headers': headers,
+                        'close_timeout': timeout,
+                        'ping_timeout': timeout,
+                        'ping_interval': timeout // 2,
+                    }
+
+                    # For secure connections, add SSL context
+                    if ws_url.startswith('wss://'):
+                        ssl_context = ssl.create_default_context()
+                        ssl_context.check_hostname = True
+                        ssl_context.verify_mode = ssl.CERT_REQUIRED
+                        connect_kwargs['ssl'] = ssl_context
+
+                    async with websockets.connect(ws_url, **connect_kwargs) as websocket:
+                        message_count = 0
+
+                        async for message in websocket:
+                            message_count += 1
+                            if message_count > self.limits.max_messages:
+                                exception_queue.put_nowait(ClientError(429, f'Message limit exceeded for {context}'))
+                                break
+
+                            # Validate message size
+                            if len(str(message)) > self.limits.max_message_size:
+                                continue
+
+                            try:
+                                message_queue.put_nowait(f'{message}\n')
+                            except queue_module.Full:
+                                exception_queue.put_nowait(ClientError(429, f'Message queue full for {context}'))
+                                break
+
+                        message_queue.put_nowait(None)  # Signal end
+
+                except websockets.exceptions.ConnectionClosed:
+                    exception_queue.put_nowait(ClientError(503, f'WebSocket connection closed for {context}'))
+                except asyncio.TimeoutError:
+                    exception_queue.put_nowait(ClientError(408, f'WebSocket timed out for {context}'))
+                except Exception as e:
+                    sanitized_error = sanitize_error_message(str(e), context)
+                    exception_queue.put_nowait(ClientError(500, sanitized_error))
+
+            # Use thread pool instead of raw threading
+            future = self.thread_pool.submit(lambda: asyncio.run(websocket_client()))
+
+            # Yield messages with proper cleanup
+            try:
+                while True:
+                    # Check for exceptions first
+                    try:
+                        exception = exception_queue.get_nowait()
+                        raise exception
+                    except queue_module.Empty:
+                        pass
+
+                    # Get message with timeout
+                    try:
+                        message = message_queue.get(timeout=1.0)
+                        if message is None:  # End signal
+                            break
+                        yield message
+                    except queue_module.Empty:
+                        # Check if future is done
+                        if future.done():
+                            try:
+                                future.result()  # This will raise any exception
+                                break  # Normal completion
+                            except Exception:
+                                break  # Error already in queue
+                        continue
+
+            finally:
+                # Cleanup: cancel future if still running
+                if not future.done():
+                    future.cancel()
+
+        except ImportError:
+            raise ClientError(500, 'websockets library not available for WebSocket connections')
+        except Exception as e:
+            if isinstance(e, ClientError):
+                raise
+            sanitized_error = sanitize_error_message(str(e), context)
+            raise ClientError(500, sanitized_error)
+
+
+class HTTPStreamManager:
+    """Manages HTTP streaming with rate limiting and proper resource cleanup."""
+
+    def __init__(self, requests_session: requests.Session, limits: Optional[StreamLimits] = None):
+        self.requests_session = requests_session
+        self.limits = limits or StreamLimits()
+
+    def stream_logs(
+        self, url: str, headers: Dict[str, str], timeout: tuple, context: str
+    ) -> Generator[str, None, None]:
+        """Stream logs from HTTP endpoint with proper error handling and cleanup."""
+        response = None
+        try:
+            # Use timeout for streaming to prevent hanging
+            response = self.requests_session.get(url, headers=headers, stream=True, timeout=timeout)
+            response.raise_for_status()
+
+            # Set up streaming with timeout and size limits
+            line_count = 0
+            total_bytes = 0
+
+            try:
+                for line in response.iter_lines(decode_unicode=True, chunk_size=1024):
+                    if line:
+                        line_count += 1
+                        total_bytes += len(line.encode('utf-8'))
+
+                        # Rate limiting checks
+                        if line_count > self.limits.max_lines:
+                            raise ClientError(429, f'Line limit exceeded for {context}')
+
+                        if total_bytes > self.limits.max_bytes:
+                            raise ClientError(429, f'Size limit exceeded for {context}')
+
+                        # Validate line size
+                        if len(line) > self.limits.max_message_size:
+                            continue
+
+                        yield f'{line}\n'
+
+            except requests.exceptions.ChunkedEncodingError:
+                raise ClientError(503, f'Log stream interrupted for {context}')
+            except requests.exceptions.ReadTimeout:
+                raise ClientError(408, f'Log stream timed out for {context}')
+
+        except requests.exceptions.ConnectTimeout:
+            raise ClientError(408, f'Failed to connect to log stream for {context}')
+        except requests.exceptions.ReadTimeout:
+            raise ClientError(408, f'Log stream read timeout for {context}')
+        except requests.exceptions.ConnectionError as e:
+            if 'Connection refused' in str(e):
+                raise ClientError(503, f'Agent connection refused for {context}')
+            else:
+                sanitized_error = sanitize_error_message(str(e), context)
+                raise ClientError(503, f'Agent connection error: {sanitized_error}')
+        except requests.exceptions.HTTPError as e:
+            if hasattr(e.response, 'status_code'):
+                status_code = e.response.status_code
+            else:
+                status_code = 500
+            raise ClientError(status_code, f'HTTP error streaming logs for {context}')
+        except Exception as e:
+            if isinstance(e, ClientError):
+                raise
+            sanitized_error = sanitize_error_message(str(e), context)
+            raise ClientError(500, sanitized_error)
+        finally:
+            # Ensure response is properly closed
+            if response is not None:
+                try:
+                    response.close()
+                except Exception:
+                    pass  # Ignore cleanup errors
+
 
 def clean_url(url, remove_query_params=True, remove_fragment=True):
     parsed = urlparse(url)
@@ -14,3 +286,24 @@ def clean_url(url, remove_query_params=True, remove_fragment=True):
         query,
         fragment,
     ))
+
+
+def get_available_ports_host(start_port=8900, end_port=8990):
+    import nmap
+
+    nm = nmap.PortScanner()
+
+    scan_range = f'{start_port}-{end_port}'
+    nm.scan(hosts='host.docker.internal', arguments=f'-p {scan_range}')
+
+    try:
+        open_ports = nm['host.docker.internal']['tcp'].keys()
+        open_ports = [int(port) for port in open_ports]
+    except KeyError:
+        open_ports = []
+
+    for port in range(start_port, end_port + 1):
+        if port not in open_ports:
+            return port
+
+    raise IOError(f'No free ports available in range {start_port}-{end_port}')

synapse_sdk/utils/storage/__init__.py

@@ -20,7 +20,11 @@ def get_storage(connection_param: str | dict):
     else:
         storage_scheme = urlparse(connection_param).scheme
 
-    assert storage_scheme in STORAGE_PROVIDERS.keys(), _('Storage provider not supported.')
+    assert storage_scheme in STORAGE_PROVIDERS.keys(), _(
+        f'Storage provider not supported. Got scheme: {storage_scheme}. '
+        f'Valid schemes: {", ".join(STORAGE_PROVIDERS.keys())}. '
+        f'Full connection_param: {connection_param}'
+    )
     return STORAGE_PROVIDERS[storage_scheme](connection_param)
 
 
@@ -29,10 +33,40 @@ def get_pathlib(storage_config: str | dict, path_root: str) -> Path:
 
     Args:
         storage_config (str | dict): The storage config by synapse-backend storage api.
-        path_root (str): The path root.
+        path_root (ㅔstr): The path root.
 
     Returns:
         pathlib.Path: The pathlib object.
     """
     storage_class = get_storage(storage_config)
     return storage_class.get_pathlib(path_root)
+
+
+def get_path_file_count(storage_config: str | dict, path_root: str) -> int:
+    """Get the file count in the path.
+
+    Args:
+        storage_config (str | dict): The storage config by synapse-backend storage api.
+        path (str): The path.
+
+    Returns:
+        int: The file count in the path.
+    """
+    storage_class = get_storage(storage_config)
+    pathlib_obj = storage_class.get_pathlib(path_root)
+    return storage_class.get_path_file_count(pathlib_obj)
+
+
+def get_path_total_size(storage_config: str | dict, path_root: str) -> int:
+    """Get total size of the files in the path.
+
+    Args:
+        storage_config (str | dict): The storage config by synapse-backend storage api.
+        path (str): The path.
+
+    Returns:
+        int: The total size of the files in the path.
+    """
+    storage_class = get_storage(storage_config)
+    pathlib_obj = storage_class.get_pathlib(path_root)
+    return storage_class.get_path_total_size(pathlib_obj)

synapse_sdk/utils/storage/providers/__init__.py

@@ -2,11 +2,52 @@ from urllib.parse import parse_qs, urlparse
 
 
 class BaseStorage:
+    """Base storage provider class for all storage implementations.
+
+    This is an abstract base class that defines the interface for storage providers.
+    All storage providers (S3, GCP, SFTP, HTTP, FileSystem) must inherit from this
+    class and implement all the abstract methods.
+
+    Args:
+        connection_params (str | dict): The connection parameters. Can be either:
+            - A URL string (e.g., 's3://bucket/path?access_key=key&secret_key=secret')
+            - A dict with 'provider' and 'configuration' keys
+
+    Attributes:
+        url: Parsed URL object if connection_params was a string
+        base_path: Base path for storage operations (provider-specific)
+        options: Storage-specific options
+        query_params: Parsed query parameters from URL or configuration dict
+        OPTION_CASTS: Dictionary mapping option names to type casting functions
+
+    Example:
+        >>> # URL-based initialization
+        >>> storage = SomeStorage('s3://bucket/path?access_key=key&secret_key=secret')
+        >>>
+        >>> # Dict-based initialization
+        >>> config = {
+        ...     'provider': 's3',
+        ...     'configuration': {
+        ...         'bucket_name': 'my-bucket',
+        ...         'access_key': 'key',
+        ...         'secret_key': 'secret'
+        ...     }
+        ... }
+        >>> storage = SomeStorage(config)
+    """
+
     url = None
     options = None
     OPTION_CASTS = {}
 
     def __init__(self, connection_params: str | dict):
+        """Initialize the storage provider with connection parameters.
+
+        Args:
+            connection_params (str | dict): Connection parameters for the storage provider.
+                If string, should be a valid URL with scheme and query parameters.
+                If dict, should contain 'provider' and 'configuration' keys.
+        """
         self.url = None
 
         if isinstance(connection_params, dict):
@@ -16,6 +57,15 @@ class BaseStorage:
             self.query_params = self.url_querystring_to_dict()
 
     def url_querystring_to_dict(self):
+        """Parse URL query string into a dictionary with proper type casting.
+
+        Converts query parameters from the URL into a dictionary, applying
+        type casting based on OPTION_CASTS mapping. Single-value lists are
+        flattened to scalar values.
+
+        Returns:
+            dict: Parsed and type-cast query parameters.
+        """
         query_string = self.url.query
 
         query_dict = parse_qs(query_string)
@@ -30,13 +80,104 @@ class BaseStorage:
         }
 
     def upload(self, source, target):
+        """Upload a file from source to target location.
+
+        Args:
+            source (str): Path to the source file on local filesystem.
+            target (str): Target path in the storage provider.
+
+        Returns:
+            str: URL or identifier of the uploaded file.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+        """
         raise NotImplementedError
 
     def exists(self, target):
+        """Check if a file exists at the target location.
+
+        Args:
+            target (str): Target path in the storage provider.
+
+        Returns:
+            bool: True if the file exists, False otherwise.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+        """
         raise NotImplementedError
 
     def get_url(self, target):
+        """Get the URL for accessing a file at the target location.
+
+        Args:
+            target (str): Target path in the storage provider.
+
+        Returns:
+            str: URL that can be used to access the file.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+        """
         raise NotImplementedError
 
     def get_pathlib(self, path):
+        """Get the path as a pathlib-compatible object.
+
+        Args:
+            path (str): The path to convert, relative to the storage root.
+
+        Returns:
+            pathlib.Path or UPath: A pathlib-compatible object representing the path.
+            The exact type depends on the storage provider implementation.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+
+        Note:
+            Different storage providers may return different path object types:
+            - FileSystemStorage returns pathlib.Path objects
+            - Cloud storage providers typically return UPath objects
+        """
+        raise NotImplementedError
+
+    def get_path_file_count(self, pathlib_obj):
+        """Get the total number of files in the given path.
+
+        Args:
+            pathlib_obj (Path | UPath): The path object to count files in.
+                Should be obtained from get_pathlib().
+
+        Returns:
+            int: The total number of files found recursively in the path.
+                Returns 1 for individual files, 0 for non-existent paths.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+
+        Note:
+            This method counts files recursively, including files in subdirectories.
+            Directories themselves are not counted, only regular files.
+        """
+        raise NotImplementedError
+
+    def get_path_total_size(self, pathlib_obj):
+        """Get the total size of all files in the given path.
+
+        Args:
+            pathlib_obj (Path | UPath): The path object to calculate size for.
+                Should be obtained from get_pathlib().
+
+        Returns:
+            int: The total size in bytes of all files found recursively in the path.
+                Returns the file size for individual files, 0 for non-existent paths.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+
+        Note:
+            This method calculates the total size recursively, including files
+            in subdirectories. Only regular files contribute to the total size.
+        """
         raise NotImplementedError

synapse_sdk/utils/storage/providers/file_system.py

@@ -0,0 +1,134 @@
+import shutil
+from pathlib import Path
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class FileSystemStorage(BaseStorage):
+    """Storage provider for file system.
+
+    * This storage do not support url based initialization.
+
+    Args:
+        url (str): The URL of the storage provider.
+
+    Examples:
+        >>> # Dict-based initialization
+        >>> config = {
+        ...     'provider': 'filesystem',
+        ...     'configuration': {
+        ...         'location': '/data'
+        ...     }
+        ... }
+        >>> storage = FileSystemStorage(config)
+    """
+
+    def __init__(self, connection_params: str | dict):
+        super().__init__(connection_params)
+        self.base_path = Path(self.query_params['location'])
+
+    def get_pathlib(self, path):
+        """Get the path as a pathlib object.
+
+        Args:
+            path (str): The path to convert.
+
+        Returns:
+            pathlib.Path: The converted path.
+        """
+        if path == '/' or path == '':
+            return self.base_path
+
+        # Strip leading slash to ensure path is relative to base_path.
+        # Path('/data') / '/subdir' would incorrectly resolve to '/subdir' instead of '/data/subdir'
+        if isinstance(path, str) and path.startswith('/'):
+            path = path.lstrip('/')
+
+        return self.base_path / path
+
+    def upload(self, source, target):
+        """Upload a file from source to target location.
+
+        Args:
+            source (str): Path to source file
+            target (str): Target path relative to base path
+
+        Returns:
+            str: URL of uploaded file
+        """
+        source_path = Path(source)
+        target_path = self.base_path / target
+
+        # Create parent directories if they don't exist
+        target_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Copy the file
+        shutil.copy2(source_path, target_path)
+
+        return self.get_url(target)
+
+    def exists(self, target):
+        """Check if target file exists.
+
+        Args:
+            target (str): Target path relative to base path
+
+        Returns:
+            bool: True if file exists, False otherwise
+        """
+        target_path = self.base_path / target
+        return target_path.exists()
+
+    def get_url(self, target):
+        """Get URL for target file.
+
+        Args:
+            target (str): Target path relative to base path
+
+        Returns:
+            str: File URL
+        """
+        target_path = self.base_path / target
+        return f'file://{target_path.absolute()}'
+
+    def get_path_file_count(self, pathlib_obj):
+        """Get the file count in the path.
+
+        Args:
+            pathlib_obj (Path): The path to get file count.
+
+        Returns:
+            int: The file count in the path.
+        """
+        if not pathlib_obj.exists():
+            return 0
+
+        if pathlib_obj.is_file():
+            return 1
+
+        count = 0
+        for item in pathlib_obj.rglob('*'):
+            if item.is_file():
+                count += 1
+        return count
+
+    def get_path_total_size(self, pathlib_obj):
+        """Get the total size of the path.
+
+        Args:
+            pathlib_obj (Path): The path to get total size.
+
+        Returns:
+            int: The total size of the path.
+        """
+        if not pathlib_obj.exists():
+            return 0
+
+        if pathlib_obj.is_file():
+            return pathlib_obj.stat().st_size
+
+        total_size = 0
+        for item in pathlib_obj.rglob('*'):
+            if item.is_file():
+                total_size += item.stat().st_size
+        return total_size