synapse-sdk 1.0.0a11__py3-none-any.whl → 2026.1.1b2__py3-none-any.whl

synapse_sdk/utils/file/download.py

@@ -0,0 +1,286 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import operator
+from functools import reduce
+from pathlib import Path
+from typing import Any, Callable, TypeVar
+from urllib.parse import urlparse, urlunparse
+
+import aiohttp
+import requests
+
+from .io import get_temp_path
+
+T = TypeVar('T')
+
+# Default chunk size: 50MB
+_CHUNK_SIZE = 1024 * 1024 * 50
+
+
+def _hash_text(text: str) -> str:
+    """Generate MD5 hash of text for cache keys."""
+    return hashlib.md5(text.encode('utf-8')).hexdigest()
+
+
+def _clean_url(url: str) -> str:
+    """Remove query params and fragment from URL."""
+    parsed = urlparse(url)
+    return urlunparse((
+        parsed.scheme,
+        parsed.netloc,
+        parsed.path,
+        parsed.params,
+        '',  # no query
+        '',  # no fragment
+    ))
+
+
+def download_file(
+    url: str,
+    path_download: str | Path,
+    *,
+    name: str | None = None,
+    coerce: Callable[[Path], T] | None = None,
+    use_cached: bool = True,
+) -> Path | T:
+    """Download a file from a URL to a specified directory.
+
+    Downloads are streamed in chunks for memory efficiency. Supports caching
+    based on URL hash to avoid redundant downloads.
+
+    Args:
+        url: The URL to download from.
+        path_download: Directory path where the file will be saved.
+        name: Custom filename (without extension). Disables caching if provided.
+        coerce: Optional function to transform the downloaded Path.
+        use_cached: If True, skip download if file already exists.
+
+    Returns:
+        Path to the downloaded file, or coerce(path) if coerce is provided.
+
+    Raises:
+        requests.HTTPError: If the HTTP request fails.
+        OSError: If file write fails.
+
+    Examples:
+        >>> path = download_file('https://example.com/image.jpg', '/tmp/downloads')
+        >>> path = download_file(url, '/tmp', name='my_file')  # Custom name
+        >>> path_str = download_file(url, '/tmp', coerce=str)  # As string
+    """
+    cleaned_url = _clean_url(url)
+
+    if name:
+        use_cached = False
+    else:
+        name = _hash_text(cleaned_url)
+
+    name += Path(cleaned_url).suffix
+    path = Path(path_download) / name
+
+    if not use_cached or not path.is_file():
+        response = requests.get(url, allow_redirects=True, stream=True, timeout=30)
+        response.raise_for_status()
+
+        with path.open('wb') as file:
+            for chunk in response.iter_content(chunk_size=_CHUNK_SIZE):
+                file.write(chunk)
+
+    if coerce:
+        return coerce(path)
+    return path
+
+
+async def adownload_file(
+    url: str,
+    path_download: str | Path,
+    *,
+    name: str | None = None,
+    coerce: Callable[[Path], T] | None = None,
+    use_cached: bool = True,
+) -> Path | T:
+    """Asynchronously download a file from a URL.
+
+    Async version of download_file() using aiohttp for concurrent downloads.
+
+    Args:
+        url: The URL to download from.
+        path_download: Directory path where the file will be saved.
+        name: Custom filename (without extension). Disables caching if provided.
+        coerce: Optional function to transform the downloaded Path.
+        use_cached: If True, skip download if file already exists.
+
+    Returns:
+        Path to the downloaded file, or coerce(path) if coerce is provided.
+
+    Examples:
+        >>> path = await adownload_file('https://example.com/large.zip', '/tmp')
+        >>> paths = await asyncio.gather(*[adownload_file(u, '/tmp') for u in urls])
+    """
+    cleaned_url = _clean_url(url)
+
+    if name:
+        use_cached = False
+    else:
+        name = _hash_text(cleaned_url)
+
+    name += Path(cleaned_url).suffix
+    path = Path(path_download) / name
+
+    if not use_cached or not path.is_file():
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                response.raise_for_status()
+                with path.open('wb') as file:
+                    while chunk := await response.content.read(_CHUNK_SIZE):
+                        file.write(chunk)
+
+    if coerce:
+        return coerce(path)
+    return path
+
+
+def files_url_to_path(
+    files: dict[str, Any],
+    *,
+    coerce: Callable[[Path], Any] | None = None,
+    file_field: str | None = None,
+) -> None:
+    """Convert file URLs to local paths by downloading them in-place.
+
+    Args:
+        files: Dictionary containing file URLs or file objects.
+            - String values: treated as URLs, replaced with local paths
+            - Dict values with 'url' key: 'url' is replaced with 'path'
+        coerce: Function to transform downloaded paths.
+        file_field: If provided, only process this specific field.
+
+    Examples:
+        >>> files = {'image': 'https://example.com/img.jpg'}
+        >>> files_url_to_path(files)
+        >>> files['image']  # Path('/tmp/datamaker/media/abc123.jpg')
+
+        >>> files = {'video': {'url': 'https://example.com/vid.mp4', 'size': 1024}}
+        >>> files_url_to_path(files)
+        >>> files['video']  # {'path': Path(...), 'size': 1024}
+    """
+    path_download = get_temp_path('media')
+    path_download.mkdir(parents=True, exist_ok=True)
+
+    if file_field:
+        files[file_field] = download_file(files[file_field], path_download, coerce=coerce)
+    else:
+        for file_name in files:
+            if isinstance(files[file_name], str):
+                files[file_name] = download_file(files[file_name], path_download, coerce=coerce)
+            else:
+                files[file_name]['path'] = download_file(files[file_name].pop('url'), path_download, coerce=coerce)
+
+
+async def afiles_url_to_path(
+    files: dict[str, Any],
+    *,
+    coerce: Callable[[Path], Any] | None = None,
+) -> None:
+    """Asynchronously convert file URLs to local paths.
+
+    All files are downloaded concurrently for better performance.
+
+    Args:
+        files: Dictionary containing file URLs or file objects.
+        coerce: Function to transform downloaded paths.
+    """
+    path_download = get_temp_path('media')
+    path_download.mkdir(parents=True, exist_ok=True)
+
+    for file_name in files:
+        if isinstance(files[file_name], str):
+            files[file_name] = await adownload_file(files[file_name], path_download, coerce=coerce)
+        else:
+            files[file_name]['path'] = await adownload_file(files[file_name].pop('url'), path_download, coerce=coerce)
+
+
+def files_url_to_path_from_objs(
+    objs: dict[str, Any] | list[dict[str, Any]],
+    files_fields: list[str],
+    *,
+    coerce: Callable[[Path], Any] | None = None,
+    is_list: bool = False,
+    is_async: bool = False,
+) -> None:
+    """Convert file URLs to paths for multiple objects with nested field support.
+
+    Args:
+        objs: Single object or list of objects to process.
+        files_fields: List of field paths (supports dot notation like 'data.files').
+        coerce: Function to transform downloaded paths.
+        is_list: If True, objs is treated as a list.
+        is_async: If True, uses async download for better performance.
+
+    Examples:
+        >>> obj = {'files': {'image': 'https://example.com/img.jpg'}}
+        >>> files_url_to_path_from_objs(obj, files_fields=['files'])
+
+        >>> objs = [{'data': {'files': {...}}}, ...]
+        >>> files_url_to_path_from_objs(objs, ['data.files'], is_list=True, is_async=True)
+    """
+    if is_async:
+        asyncio.run(afiles_url_to_path_from_objs(objs, files_fields, coerce=coerce, is_list=is_list))
+    else:
+        if not is_list:
+            objs = [objs]
+
+        for obj in objs:
+            for files_field in files_fields:
+                try:
+                    files = reduce(operator.getitem, files_field.split('.'), obj)
+                    if isinstance(files, str):
+                        files_url_to_path(obj, coerce=coerce, file_field=files_field)
+                    else:
+                        files_url_to_path(files, coerce=coerce)
+                except KeyError:
+                    pass
+
+
+async def afiles_url_to_path_from_objs(
+    objs: dict[str, Any] | list[dict[str, Any]],
+    files_fields: list[str],
+    *,
+    coerce: Callable[[Path], Any] | None = None,
+    is_list: bool = False,
+) -> None:
+    """Asynchronously convert file URLs to paths for multiple objects.
+
+    All file downloads happen concurrently using asyncio.gather().
+
+    Args:
+        objs: Single object or list of objects to process.
+        files_fields: List of field paths (supports dot notation).
+        coerce: Function to transform downloaded paths.
+        is_list: If True, objs is treated as a list.
+    """
+    if not is_list:
+        objs = [objs]
+
+    tasks = []
+
+    for obj in objs:
+        for files_field in files_fields:
+            try:
+                files = reduce(operator.getitem, files_field.split('.'), obj)
+                tasks.append(afiles_url_to_path(files, coerce=coerce))
+            except KeyError:
+                pass
+
+    await asyncio.gather(*tasks)
+
+
+__all__ = [
+    'download_file',
+    'adownload_file',
+    'files_url_to_path',
+    'afiles_url_to_path',
+    'files_url_to_path_from_objs',
+    'afiles_url_to_path_from_objs',
+]

synapse_sdk/utils/file/io.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import base64
+import json
+import mimetypes
+from collections.abc import Generator
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+# Default chunk size: 50MB
+DEFAULT_CHUNK_SIZE = 1024 * 1024 * 50
+
+
+def read_file_in_chunks(
+    file_path: str | Path,
+    chunk_size: int = DEFAULT_CHUNK_SIZE,
+) -> Generator[bytes, None, None]:
+    """Read a file in chunks, yielding each chunk.
+
+    Memory-efficient generator for processing large files.
+
+    Args:
+        file_path: Path to the file to read.
+        chunk_size: Size of each chunk in bytes (default 50MB).
+
+    Yields:
+        Bytes chunks of the file.
+
+    Raises:
+        FileNotFoundError: If file doesn't exist.
+        PermissionError: If file cannot be read.
+
+    Example:
+        >>> for chunk in read_file_in_chunks('/path/to/large_file.zip'):
+        ...     process(chunk)
+    """
+    path = Path(file_path)
+    with path.open('rb') as f:
+        while chunk := f.read(chunk_size):
+            yield chunk
+
+
+def convert_file_to_base64(file_path: str | Path) -> str:
+    """Convert a file to base64 data URI format.
+
+    Args:
+        file_path: Path to the file to encode.
+
+    Returns:
+        Data URI string: "data:{mime_type};base64,{encoded_content}"
+
+    Raises:
+        FileNotFoundError: If file doesn't exist.
+        ValueError: If MIME type cannot be determined.
+
+    Example:
+        >>> uri = convert_file_to_base64('/path/to/image.png')
+        >>> uri.startswith('data:image/png;base64,')
+        True
+    """
+    path = Path(file_path)
+
+    # Check if already base64 encoded
+    if isinstance(file_path, str) and file_path.startswith('data:'):
+        return file_path
+
+    mime_type, _ = mimetypes.guess_type(str(path))
+    if mime_type is None:
+        raise ValueError(f'Cannot determine MIME type for: {path}')
+
+    content = path.read_bytes()
+    encoded = base64.b64encode(content).decode('ascii')
+
+    return f'data:{mime_type};base64,{encoded}'
+
+
+def get_temp_path(sub_path: str | None = None) -> Path:
+    """Get a temporary directory path for SDK operations.
+
+    Args:
+        sub_path: Optional subdirectory name to append.
+
+    Returns:
+        Path object pointing to /tmp/datamaker or /tmp/datamaker/{sub_path}.
+
+    Examples:
+        >>> get_temp_path()
+        PosixPath('/tmp/datamaker')
+        >>> get_temp_path('media')
+        PosixPath('/tmp/datamaker/media')
+    """
+    path = Path('/tmp/datamaker')
+    if sub_path:
+        path = path / sub_path
+    return path
+
+
+def get_dict_from_file(file_path: str | Path) -> dict[str, Any]:
+    """Load a dictionary from a JSON or YAML file.
+
+    Args:
+        file_path: Path to the file (JSON or YAML).
+
+    Returns:
+        Dictionary parsed from the file.
+
+    Raises:
+        FileNotFoundError: If the file doesn't exist.
+        json.JSONDecodeError: If JSON parsing fails.
+        yaml.YAMLError: If YAML parsing fails.
+    """
+    if isinstance(file_path, str):
+        file_path = Path(file_path)
+
+    with open(file_path) as f:
+        if file_path.suffix in ('.yaml', '.yml'):
+            return yaml.safe_load(f)
+        return json.load(f)
+
+
+__all__ = [
+    'DEFAULT_CHUNK_SIZE',
+    'read_file_in_chunks',
+    'convert_file_to_base64',
+    'get_temp_path',
+    'get_dict_from_file',
+]

synapse_sdk/utils/file/requirements.py

@@ -0,0 +1,36 @@
+"""Requirements file parsing utilities."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def read_requirements(path: str | Path) -> list[str] | None:
+    """Parse requirements.txt file.
+
+    Reads a requirements.txt file and returns a list of requirement strings,
+    filtering out empty lines and comments.
+
+    Args:
+        path: Path to requirements.txt file
+
+    Returns:
+        List of requirement strings, or None if file doesn't exist.
+        Returns None if file exists but contains no valid requirements.
+    """
+    path = Path(path)
+    if not path.exists():
+        return None
+
+    requirements = []
+    with path.open() as f:
+        for line in f:
+            line = line.strip()
+            # Skip empty lines and comments
+            if line and not line.startswith('#'):
+                requirements.append(line)
+
+    return requirements if requirements else None
+
+
+__all__ = ['read_requirements']

synapse_sdk/utils/network.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import urlparse, urlunparse
+
+from synapse_sdk.exceptions import ClientError
+
+
+@dataclass
+class StreamLimits:
+    """Configuration for streaming resource limits.
+
+    Prevents resource exhaustion during long-running streaming operations.
+
+    Attributes:
+        max_messages: Maximum WebSocket messages before termination.
+        max_lines: Maximum lines for HTTP streaming.
+        max_bytes: Maximum total bytes to receive.
+        max_message_size: Maximum size of a single message/line in bytes.
+        queue_size: Bounded queue size for async operations.
+    """
+
+    max_messages: int = 10_000
+    max_lines: int = 50_000
+    max_bytes: int = 50 * 1024 * 1024  # 50MB
+    max_message_size: int = 10_240  # 10KB per message
+    queue_size: int = 1_000
+
+
+# Resource ID validation pattern - alphanumeric, hyphens, underscores
+_RESOURCE_ID_PATTERN = re.compile(r'^[a-zA-Z0-9\-_]+$')
+_MAX_RESOURCE_ID_LENGTH = 100
+
+
+def validate_resource_id(resource_id: Any, resource_name: str = 'resource') -> str:
+    """Validate resource ID to prevent injection attacks.
+
+    Args:
+        resource_id: The ID to validate.
+        resource_name: Human-readable name for error messages.
+
+    Returns:
+        Validated ID as string.
+
+    Raises:
+        ClientError: If ID is invalid (400 status code).
+
+    Example:
+        >>> validate_resource_id('job-abc123', 'job')
+        'job-abc123'
+        >>> validate_resource_id('', 'job')
+        Traceback (most recent call last):
+        ...
+        ClientError: job ID cannot be empty
+    """
+    if not resource_id:
+        raise ClientError(400, f'{resource_name} ID cannot be empty')
+
+    id_str = str(resource_id)
+
+    if not _RESOURCE_ID_PATTERN.match(id_str):
+        raise ClientError(400, f'Invalid {resource_name} ID format')
+
+    if len(id_str) > _MAX_RESOURCE_ID_LENGTH:
+        raise ClientError(400, f'{resource_name} ID too long')
+
+    return id_str
+
+
+def validate_timeout(timeout: Any, max_timeout: float = 300.0) -> float:
+    """Validate timeout value with bounds checking.
+
+    Args:
+        timeout: Timeout value to validate.
+        max_timeout: Maximum allowed timeout in seconds.
+
+    Returns:
+        Validated timeout as float.
+
+    Raises:
+        ClientError: If timeout is invalid (400 status code).
+
+    Example:
+        >>> validate_timeout(30.0)
+        30.0
+        >>> validate_timeout(-1)
+        Traceback (most recent call last):
+        ...
+        ClientError: Timeout must be a positive number
+    """
+    if not isinstance(timeout, (int, float)) or timeout <= 0:
+        raise ClientError(400, 'Timeout must be a positive number')
+
+    if timeout > max_timeout:
+        raise ClientError(400, f'Timeout cannot exceed {max_timeout} seconds')
+
+    return float(timeout)
+
+
+def sanitize_error_message(error_msg: str, context: str = '') -> str:
+    """Sanitize error messages to prevent information disclosure.
+
+    Redacts potentially sensitive information like credentials, paths, etc.
+
+    Args:
+        error_msg: Raw error message.
+        context: Optional context prefix.
+
+    Returns:
+        Sanitized error message.
+
+    Example:
+        >>> sanitize_error_message('Failed with token="secret123"', 'connection')
+        'connection: Failed with token="[REDACTED]"'
+    """
+    sanitized = str(error_msg)[:200]
+    # Redact quoted strings which may contain sensitive data
+    sanitized = re.sub(r'["\']([^"\']*)["\']', '"[REDACTED]"', sanitized)
+
+    if context:
+        return f'{context}: {sanitized}'
+    return sanitized
+
+
+def http_to_websocket_url(url: str) -> str:
+    """Convert HTTP/HTTPS URL to WebSocket URL.
+
+    Args:
+        url: HTTP or HTTPS URL.
+
+    Returns:
+        WebSocket URL (ws:// or wss://).
+
+    Raises:
+        ClientError: If URL scheme is invalid.
+
+    Example:
+        >>> http_to_websocket_url('https://example.com/ws/')
+        'wss://example.com/ws/'
+        >>> http_to_websocket_url('http://localhost:8000/ws/')
+        'ws://localhost:8000/ws/'
+    """
+    try:
+        parsed = urlparse(url)
+
+        if parsed.scheme == 'http':
+            ws_scheme = 'ws'
+        elif parsed.scheme == 'https':
+            ws_scheme = 'wss'
+        elif parsed.scheme in ('ws', 'wss'):
+            return url  # Already a WebSocket URL
+        else:
+            raise ClientError(400, f'Invalid URL scheme: {parsed.scheme}')
+
+        return urlunparse((
+            ws_scheme,
+            parsed.netloc,
+            parsed.path,
+            parsed.params,
+            parsed.query,
+            parsed.fragment,
+        ))
+    except ClientError:
+        raise
+    except Exception as e:
+        raise ClientError(400, f'Invalid URL format: {str(e)[:50]}')