synapse-sdk 1.0.0b22__py3-none-any.whl → 1.0.0b24__py3-none-any.whl

synapse_sdk/plugins/categories/upload/actions/upload/utils.py

@@ -0,0 +1,139 @@
+import json
+import os
+
+
+class PathAwareJSONEncoder(json.JSONEncoder):
+    """Custom JSON encoder that handles Path objects and datetime objects.
+
+    Extends the default JSON encoder to properly serialize Path objects
+    and datetime objects that are commonly used in upload operations.
+
+    Supported object types:
+    - Path objects (converts to string using __fspath__ or as_posix)
+    - Datetime objects (converts using isoformat)
+    - All other standard JSON-serializable types
+
+    Example:
+        >>> data = {"path": Path("/tmp/file.txt"), "timestamp": datetime.now()}
+        >>> json.dumps(data, cls=PathAwareJSONEncoder)
+        '{"path": "/tmp/file.txt", "timestamp": "2023-01-01T12:00:00"}'
+    """
+
+    def default(self, obj):
+        if hasattr(obj, '__fspath__'):
+            return obj.__fspath__()
+        elif hasattr(obj, 'as_posix'):
+            return obj.as_posix()
+        elif hasattr(obj, 'isoformat'):
+            return obj.isoformat()
+        return super().default(obj)
+
+
+class ExcelSecurityConfig:
+    """Configuration class for Excel file security limits.
+
+    Manages security constraints for Excel file processing to prevent
+    resource exhaustion and security vulnerabilities. All limits can
+    be configured via environment variables.
+
+    Attributes:
+        MAX_FILE_SIZE_MB (int): Maximum file size in megabytes
+        MAX_FILE_SIZE_BYTES (int): Maximum file size in bytes
+        MAX_MEMORY_USAGE_MB (int): Maximum memory usage in megabytes
+        MAX_MEMORY_USAGE_BYTES (int): Maximum memory usage in bytes
+        MAX_ROWS (int): Maximum number of rows allowed
+        MAX_COLUMNS (int): Maximum number of columns allowed
+        MAX_FILENAME_LENGTH (int): Maximum filename length
+        MAX_COLUMN_NAME_LENGTH (int): Maximum column name length
+        MAX_METADATA_VALUE_LENGTH (int): Maximum metadata value length
+
+    Environment Variables:
+        EXCEL_MAX_FILE_SIZE_MB: Override default file size limit (default: 10)
+        EXCEL_MAX_MEMORY_MB: Override default memory limit (default: 30)
+        EXCEL_MAX_ROWS: Override default row limit (default: 10000)
+        EXCEL_MAX_COLUMNS: Override default column limit (default: 50)
+        EXCEL_MAX_FILENAME_LENGTH: Override filename length limit (default: 255)
+        EXCEL_MAX_COLUMN_NAME_LENGTH: Override column name length (default: 100)
+        EXCEL_MAX_METADATA_VALUE_LENGTH: Override metadata value length (default: 1000)
+    """
+
+    def __init__(self):
+        self.MAX_FILE_SIZE_MB = int(os.getenv('EXCEL_MAX_FILE_SIZE_MB', '10'))
+        self.MAX_FILE_SIZE_BYTES = self.MAX_FILE_SIZE_MB * 1024 * 1024
+
+        self.MAX_MEMORY_USAGE_MB = int(os.getenv('EXCEL_MAX_MEMORY_MB', '30'))
+        self.MAX_MEMORY_USAGE_BYTES = self.MAX_MEMORY_USAGE_MB * 1024 * 1024
+
+        self.MAX_ROWS = int(os.getenv('EXCEL_MAX_ROWS', '10000'))
+        self.MAX_COLUMNS = int(os.getenv('EXCEL_MAX_COLUMNS', '50'))
+
+        self.MAX_FILENAME_LENGTH = int(os.getenv('EXCEL_MAX_FILENAME_LENGTH', '255'))
+        self.MAX_COLUMN_NAME_LENGTH = int(os.getenv('EXCEL_MAX_COLUMN_NAME_LENGTH', '100'))
+        self.MAX_METADATA_VALUE_LENGTH = int(os.getenv('EXCEL_MAX_METADATA_VALUE_LENGTH', '1000'))
+
+
+class ExcelMetadataUtils:
+    """Utility class for Excel metadata processing and validation.
+
+    Provides helper methods for validating and processing Excel metadata
+    while respecting security constraints defined in ExcelSecurityConfig.
+
+    Args:
+        config (ExcelSecurityConfig): Security configuration instance
+
+    Example:
+        >>> config = ExcelSecurityConfig()
+        >>> utils = ExcelMetadataUtils(config)
+        >>> safe_value = utils.validate_and_truncate_string("long text", 10)
+        >>> is_valid = utils.is_valid_filename_length("file.xlsx")
+    """
+
+    def __init__(self, config: ExcelSecurityConfig):
+        self.config = config
+
+    def validate_and_truncate_string(self, value: str, max_length: int) -> str:
+        """Validate and truncate string to maximum length.
+
+        Converts non-string values to strings, trims whitespace, and
+        truncates to the specified maximum length if necessary.
+
+        Args:
+            value (str): Value to validate and truncate
+            max_length (int): Maximum allowed length
+
+        Returns:
+            str: Validated and truncated string
+
+        Example:
+            >>> utils.validate_and_truncate_string("  long text  ", 5)
+            'long '
+        """
+        if not isinstance(value, str):
+            value = str(value)
+
+        value = value.strip()
+
+        if len(value) > max_length:
+            return value[:max_length]
+
+        return value
+
+    def is_valid_filename_length(self, filename: str) -> bool:
+        """Check if filename length is within security limits.
+
+        Validates that the filename (after trimming whitespace) does not
+        exceed the maximum filename length configured in security settings.
+
+        Args:
+            filename (str): Filename to validate
+
+        Returns:
+            bool: True if filename length is valid, False otherwise
+
+        Example:
+            >>> utils.is_valid_filename_length("file.xlsx")
+            True
+            >>> utils.is_valid_filename_length("x" * 300)
+            False
+        """
+        return len(filename.strip()) <= self.config.MAX_FILENAME_LENGTH

synapse_sdk/plugins/categories/upload/templates/plugin/upload.py

@@ -10,7 +10,12 @@ class Uploader:
     """
 
     def __init__(
-        self, run, path: Path, file_specification: List = None, organized_files: List = None, extra_params: Dict = None
+        self,
+        run,
+        path: Path,
+        file_specification: List = None,
+        organized_files: List = None,
+        extra_params: Dict = None,
     ):
         """Initialize the plugin upload action class.
 

synapse_sdk/plugins/models.py

@@ -124,9 +124,9 @@ class Run:
     context = None
     client = None
 
-    def __init__(self, job_id, context):
+    def __init__(self, job_id, context=None):
         self.job_id = job_id
-        self.context = context
+        self.context = context or {}
         config = get_backend_config()
         if config:
             self.client = BackendClient(
@@ -134,17 +134,23 @@ class Run:
                 access_token=config['token'],
             )
         else:
+            # Handle missing environment variables for test environments
+            envs = self.context.get('envs', {})
+            host = envs.get('SYNAPSE_PLUGIN_RUN_HOST', os.getenv('SYNAPSE_PLUGIN_RUN_HOST', 'http://localhost:8000'))
+            token = envs.get('SYNAPSE_PLUGIN_RUN_USER_TOKEN', os.getenv('SYNAPSE_PLUGIN_RUN_USER_TOKEN'))
+            tenant = envs.get('SYNAPSE_PLUGIN_RUN_TENANT', os.getenv('SYNAPSE_PLUGIN_RUN_TENANT'))
+
             self.client = BackendClient(
-                self.context['envs']['SYNAPSE_PLUGIN_RUN_HOST'],
-                token=self.context['envs'].get('SYNAPSE_PLUGIN_RUN_USER_TOKEN'),
-                tenant=self.context['envs'].get('SYNAPSE_PLUGIN_RUN_TENANT'),
+                host,
+                token=token,
+                tenant=tenant,
             )
         self.set_logger()
 
     def set_logger(self):
         kwargs = {
-            'progress_categories': self.context['progress_categories'],
-            'metrics_categories': self.context['metrics_categories'],
+            'progress_categories': self.context.get('progress_categories'),
+            'metrics_categories': self.context.get('metrics_categories'),
         }
 
         if self.job_id:

synapse_sdk/utils/file/__init__.py

@@ -0,0 +1,39 @@
+# File utilities module
+# Maintains backward compatibility by re-exporting all functions
+
+from .archive import archive, unarchive
+from .checksum import calculate_checksum, get_checksum_from_file
+from .chunking import read_file_in_chunks
+from .download import (
+    adownload_file,
+    afiles_url_to_path,
+    afiles_url_to_path_from_objs,
+    download_file,
+    files_url_to_path,
+    files_url_to_path_from_objs,
+)
+from .encoding import convert_file_to_base64
+from .io import get_dict_from_file, get_temp_path
+
+__all__ = [
+    # Chunking
+    'read_file_in_chunks',
+    # Download
+    'download_file',
+    'adownload_file',
+    'files_url_to_path',
+    'afiles_url_to_path',
+    'files_url_to_path_from_objs',
+    'afiles_url_to_path_from_objs',
+    # Checksum
+    'calculate_checksum',
+    'get_checksum_from_file',
+    # Archive
+    'archive',
+    'unarchive',
+    # Encoding
+    'convert_file_to_base64',
+    # I/O
+    'get_dict_from_file',
+    'get_temp_path',
+]

synapse_sdk/utils/file/archive.py

@@ -0,0 +1,32 @@
+import zipfile
+from pathlib import Path
+
+
+def archive(input_path, output_path, append=False):
+    input_path = Path(input_path)
+    output_path = Path(output_path)
+
+    mode = 'a' if append and output_path.exists() else 'w'
+    with zipfile.ZipFile(output_path, mode=mode, compression=zipfile.ZIP_DEFLATED) as zipf:
+        if input_path.is_file():
+            zipf.write(input_path, input_path.name)
+        else:
+            for file_path in input_path.rglob('*'):
+                if file_path.is_file():  # Only add files, skip directories
+                    arcname = file_path.relative_to(input_path.parent)
+                    zipf.write(file_path, arcname)
+
+
+def unarchive(file_path, output_path):
+    """
+    Unarchives a ZIP file to a given directory.
+
+    Parameters:
+        file_path (str | Path): The path to the ZIP file.
+        output_path (str): The directory where the files will be extracted.
+    """
+    output_path = Path(output_path)
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    with zipfile.ZipFile(str(file_path), 'r') as zip_ref:
+        zip_ref.extractall(output_path)

synapse_sdk/utils/file/checksum.py

@@ -0,0 +1,56 @@
+import hashlib
+from typing import IO, Any, Callable
+
+
+def calculate_checksum(file_path, prefix=''):
+    md5_hash = hashlib.md5()
+    with open(file_path, 'rb') as f:
+        for byte_block in iter(lambda: f.read(4096), b''):
+            md5_hash.update(byte_block)
+    checksum = md5_hash.hexdigest()
+    if prefix:
+        return f'dev-{checksum}'
+    return checksum
+
+
+def get_checksum_from_file(file: IO[Any], digest_mod: Callable[[], Any] = hashlib.sha1) -> str:
+    """
+    Calculate checksum for a file-like object.
+
+    Args:
+        file (IO[Any]): File-like object with read() method that supports reading in chunks
+        digest_mod (Callable[[], Any]): Hash algorithm from hashlib (defaults to hashlib.sha1)
+
+    Returns:
+        str: Hexadecimal digest of the file contents
+
+    Example:
+        ```python
+        import hashlib
+        from io import BytesIO
+        from synapse_sdk.utils.file import get_checksum_from_file
+
+        # With BytesIO
+        data = BytesIO(b'Hello, world!')
+        checksum = get_checksum_from_file(data)
+
+        # With different hash algorithm
+        checksum = get_checksum_from_file(data, digest_mod=hashlib.sha256)
+        ```
+    """
+    digest = digest_mod()
+    chunk_size = 4096
+
+    # Reset file pointer to beginning if possible
+    if hasattr(file, 'seek'):
+        file.seek(0)
+
+    while True:
+        chunk = file.read(chunk_size)
+        if not chunk:
+            break
+        if isinstance(chunk, str):
+            chunk = chunk.encode('utf-8')
+        digest.update(chunk)
+
+    return digest.hexdigest()

synapse_sdk/utils/file/chunking.py

@@ -0,0 +1,31 @@
+def read_file_in_chunks(file_path, chunk_size=1024 * 1024 * 50):
+    """
+    Read a file in chunks for efficient memory usage during file processing.
+
+    This function is particularly useful for large files or when you need to process
+    files in chunks, such as for uploading or hashing.
+
+    Args:
+        file_path (str | Path): Path to the file to read
+        chunk_size (int, optional): Size of each chunk in bytes. Defaults to 50MB (1024 * 1024 * 50)
+
+    Yields:
+        bytes: File content chunks
+
+    Raises:
+        FileNotFoundError: If the file doesn't exist
+        PermissionError: If the file can't be read due to permissions
+        OSError: If there's an OS-level error reading the file
+
+    Example:
+        ```python
+        from synapse_sdk.utils.file import read_file_in_chunks
+
+        # Read a file in 10MB chunks
+        for chunk in read_file_in_chunks('large_file.bin', chunk_size=1024*1024*10):
+            process_chunk(chunk)
+        ```
+    """
+    with open(file_path, 'rb') as file:
+        while chunk := file.read(chunk_size):
+            yield chunk

synapse_sdk/utils/file/download.py

@@ -0,0 +1,124 @@
+import asyncio
+import operator
+from functools import reduce
+from pathlib import Path
+
+import aiohttp
+import requests
+
+from synapse_sdk.utils.network import clean_url
+from synapse_sdk.utils.string import hash_text
+
+from .io import get_temp_path
+
+
+def download_file(url, path_download, name=None, coerce=None, use_cached=True):
+    chunk_size = 1024 * 1024 * 50
+    cleaned_url = clean_url(url)  # remove query params and fragment
+
+    if name:
+        use_cached = False
+    else:
+        name = hash_text(cleaned_url)
+
+    name += Path(cleaned_url).suffix
+
+    path = Path(path_download) / name
+
+    if not use_cached or not path.is_file():
+        response = requests.get(url, allow_redirects=True, stream=True)
+        response.raise_for_status()
+
+        with path.open('wb') as file:
+            for chunk in response.iter_content(chunk_size=chunk_size):
+                file.write(chunk)
+
+    if coerce:
+        path = coerce(path)
+
+    return path
+
+
+def files_url_to_path(files, coerce=None, file_field=None):
+    path_download = get_temp_path('media')
+    path_download.mkdir(parents=True, exist_ok=True)
+    if file_field:
+        files[file_field] = download_file(files[file_field], path_download, coerce=coerce)
+    else:
+        for file_name in files:
+            if isinstance(files[file_name], str):
+                files[file_name] = download_file(files[file_name], path_download, coerce=coerce)
+            else:
+                files[file_name]['path'] = download_file(files[file_name].pop('url'), path_download, coerce=coerce)
+
+
+def files_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False, is_async=False):
+    if is_async:
+        asyncio.run(afiles_url_to_path_from_objs(objs, files_fields, coerce=coerce, is_list=is_list))
+    else:
+        if not is_list:
+            objs = [objs]
+
+        for obj in objs:
+            for files_field in files_fields:
+                try:
+                    files = reduce(operator.getitem, files_field.split('.'), obj)
+                    if isinstance(files, str):
+                        files_url_to_path(obj, coerce=coerce, file_field=files_field)
+                    else:
+                        files_url_to_path(files, coerce=coerce)
+                except KeyError:
+                    pass
+
+
+async def adownload_file(url, path_download, name=None, coerce=None, use_cached=True):
+    chunk_size = 1024 * 1024 * 50
+    cleaned_url = clean_url(url)  # remove query params and fragment
+
+    if name:
+        use_cached = False
+    else:
+        name = hash_text(cleaned_url)
+
+    name += Path(cleaned_url).suffix
+
+    path = Path(path_download) / name
+
+    if not use_cached or not path.is_file():
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                with path.open('wb') as file:
+                    while chunk := await response.content.read(chunk_size):
+                        file.write(chunk)
+
+    if coerce:
+        path = coerce(path)
+
+    return path
+
+
+async def afiles_url_to_path(files, coerce=None):
+    path_download = get_temp_path('media')
+    path_download.mkdir(parents=True, exist_ok=True)
+    for file_name in files:
+        if isinstance(files[file_name], str):
+            files[file_name] = await adownload_file(files[file_name], path_download, coerce=coerce)
+        else:
+            files[file_name]['path'] = await adownload_file(files[file_name].pop('url'), path_download, coerce=coerce)
+
+
+async def afiles_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False):
+    if not is_list:
+        objs = [objs]
+
+    tasks = []
+
+    for obj in objs:
+        for files_field in files_fields:
+            try:
+                files = reduce(operator.getitem, files_field.split('.'), obj)
+                tasks.append(afiles_url_to_path(files, coerce=coerce))
+            except KeyError:
+                pass
+
+    await asyncio.gather(*tasks)

synapse_sdk/utils/file/encoding.py

@@ -0,0 +1,40 @@
+import base64
+import mimetypes
+from pathlib import Path
+
+
+def convert_file_to_base64(file_path):
+    """
+    Convert a file to base64 using pathlib.
+
+    Args:
+        file_path (str): Path to the file to convert
+
+    Returns:
+        str: Base64 encoded string of the file contents
+    """
+    # FIXME base64 is sent sometimes.
+    if file_path.startswith('data:'):
+        return file_path
+
+    # Convert string path to Path object
+    path = Path(file_path)
+
+    try:
+        # Read binary content of the file
+        binary_content = path.read_bytes()
+
+        # Convert to base64
+        base64_encoded = base64.b64encode(binary_content).decode('utf-8')
+
+        # Get the MIME type of the file
+        mime_type, _ = mimetypes.guess_type(path)
+        assert mime_type is not None, 'MIME type cannot be guessed'
+
+        # Convert bytes to string for readable output
+        return f'data:{mime_type};base64,{base64_encoded}'
+
+    except FileNotFoundError:
+        raise FileNotFoundError(f'File not found: {file_path}')
+    except Exception as e:
+        raise Exception(f'Error converting file to base64: {str(e)}')

synapse_sdk/utils/file/io.py

@@ -0,0 +1,22 @@
+import json
+from pathlib import Path
+
+import yaml
+
+
+def get_dict_from_file(file_path):
+    if isinstance(file_path, str):
+        file_path = Path(file_path)
+
+    with open(file_path) as f:
+        if file_path.suffix == '.yaml':
+            return yaml.safe_load(f)
+        else:
+            return json.load(f)
+
+
+def get_temp_path(sub_path=None):
+    path = Path('/tmp/datamaker')
+    if sub_path:
+        path = path / sub_path
+    return path

synapse_sdk/utils/file/video/__init__.py

@@ -0,0 +1,29 @@
+# Video processing utilities
+
+from .transcode import (
+    FFmpegNotFoundError,
+    TranscodeConfig,
+    TranscodingFailedError,
+    UnsupportedFormatError,
+    VideoTranscodeError,
+    atranscode_video,
+    get_video_info,
+    optimize_for_web,
+    transcode_batch,
+    transcode_video,
+    validate_video_format,
+)
+
+__all__ = [
+    'TranscodeConfig',
+    'VideoTranscodeError',
+    'UnsupportedFormatError',
+    'FFmpegNotFoundError',
+    'TranscodingFailedError',
+    'transcode_video',
+    'atranscode_video',
+    'get_video_info',
+    'validate_video_format',
+    'optimize_for_web',
+    'transcode_batch',
+]