synapse-sdk 1.0.0a23__py3-none-any.whl → 2025.12.3__py3-none-any.whl

synapse_sdk/utils/file/video/transcode.py

@@ -0,0 +1,307 @@
+import asyncio
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, Optional
+
+import ffmpeg
+
+
+# Exception classes
+class VideoTranscodeError(Exception):
+    """Base exception for video transcoding errors."""
+
+    pass
+
+
+class UnsupportedFormatError(VideoTranscodeError):
+    """Raised when input format is not supported."""
+
+    pass
+
+
+class FFmpegNotFoundError(VideoTranscodeError):
+    """Raised when FFmpeg is not installed or not in PATH."""
+
+    pass
+
+
+class TranscodingFailedError(VideoTranscodeError):
+    """Raised when FFmpeg transcoding process fails."""
+
+    pass
+
+
+@dataclass
+class TranscodeConfig:
+    """Video transcoding configuration."""
+
+    vcodec: str = 'libx264'  # Video codec
+    preset: str = 'medium'  # Encoding preset (ultrafast to veryslow)
+    crf: int = 28  # Constant Rate Factor (0-51, lower=better quality)
+    acodec: str = 'aac'  # Audio codec
+    audio_bitrate: str = '128k'  # Audio bitrate
+    movflags: str = '+faststart'  # MP4 optimization flags
+    resolution: Optional[str] = None  # Target resolution (e.g., '1920x1080')
+    fps: Optional[int] = None  # Target frame rate
+    start_time: Optional[float] = None  # Trim start time in seconds
+    duration: Optional[float] = None  # Trim duration in seconds
+
+
+# Supported input formats
+SUPPORTED_FORMATS = {'.mp4', '.avi', '.mov', '.mkv', '.webm', '.flv', '.wmv', '.mpeg', '.mpg', '.m4v', '.3gp', '.ogv'}
+
+
+def _check_ffmpeg_available():
+    """Check if FFmpeg is available in PATH."""
+    if not shutil.which('ffmpeg'):
+        raise FFmpegNotFoundError(
+            'FFmpeg is not installed or not found in PATH. Please install FFmpeg to use video transcoding features.'
+        )
+
+
+def validate_video_format(video_path: str | Path) -> bool:
+    """
+    Check if video format is supported for transcoding.
+
+    Args:
+        video_path (str | Path): Path to the video file
+
+    Returns:
+        bool: True if format is supported, False otherwise
+    """
+    path = Path(video_path)
+    return path.suffix.lower() in SUPPORTED_FORMATS
+
+
+def get_video_info(video_path: str | Path) -> dict:
+    """
+    Extract video metadata (resolution, duration, codecs, etc.).
+
+    Args:
+        video_path (str | Path): Path to the video file
+
+    Returns:
+        dict: Video metadata information
+
+    Raises:
+        VideoTranscodeError: If unable to probe video file
+    """
+    _check_ffmpeg_available()
+
+    try:
+        probe = ffmpeg.probe(str(video_path))
+
+        video_info = {}
+
+        # Get format information
+        if 'format' in probe:
+            format_info = probe['format']
+            video_info['duration'] = float(format_info.get('duration', 0))
+            video_info['size'] = int(format_info.get('size', 0))
+            video_info['bitrate'] = int(format_info.get('bit_rate', 0))
+
+        # Get stream information
+        video_streams = [stream for stream in probe['streams'] if stream['codec_type'] == 'video']
+        audio_streams = [stream for stream in probe['streams'] if stream['codec_type'] == 'audio']
+
+        if video_streams:
+            video_stream = video_streams[0]
+            video_info['width'] = int(video_stream.get('width', 0))
+            video_info['height'] = int(video_stream.get('height', 0))
+            video_info['video_codec'] = video_stream.get('codec_name', '')
+            video_info['fps'] = eval(video_stream.get('r_frame_rate', '0/1'))
+
+        if audio_streams:
+            audio_stream = audio_streams[0]
+            video_info['audio_codec'] = audio_stream.get('codec_name', '')
+            video_info['channels'] = int(audio_stream.get('channels', 0))
+            video_info['sample_rate'] = int(audio_stream.get('sample_rate', 0))
+
+        return video_info
+
+    except Exception as e:
+        raise VideoTranscodeError(f'Failed to probe video file: {str(e)}')
+
+
+def _build_ffmpeg_stream(input_path: str | Path, output_path: str | Path, config: TranscodeConfig):
+    """Build FFmpeg stream with configuration."""
+    stream = ffmpeg.input(str(input_path))
+
+    # Apply start time and duration trimming
+    if config.start_time is not None or config.duration is not None:
+        kwargs = {}
+        if config.start_time is not None:
+            kwargs['ss'] = config.start_time
+        if config.duration is not None:
+            kwargs['t'] = config.duration
+        stream = ffmpeg.input(str(input_path), **kwargs)
+
+    # Apply video filters
+    if config.resolution or config.fps:
+        if config.resolution:
+            width, height = config.resolution.split('x')
+            stream = ffmpeg.filter(stream, 'scale', width, height)
+        if config.fps:
+            stream = ffmpeg.filter(stream, 'fps', fps=config.fps)
+
+    # Build output with encoding parameters
+    output_kwargs = {
+        'vcodec': config.vcodec,
+        'preset': config.preset,
+        'crf': config.crf,
+        'acodec': config.acodec,
+        'audio_bitrate': config.audio_bitrate,
+        'movflags': config.movflags,
+    }
+
+    return ffmpeg.output(stream, str(output_path), **output_kwargs)
+
+
+def transcode_video(
+    input_path: str | Path,
+    output_path: str | Path,
+    config: Optional[TranscodeConfig] = None,
+    progress_callback: Optional[Callable[[float], None]] = None,
+) -> Path:
+    """
+    Transcode video with specified configuration.
+
+    Args:
+        input_path (str | Path): Path to input video file
+        output_path (str | Path): Path to output video file
+        config (Optional[TranscodeConfig]): Transcoding configuration
+        progress_callback (Optional[Callable[[float], None]]): Progress callback function
+
+    Returns:
+        Path: Path to the transcoded video file
+
+    Raises:
+        UnsupportedFormatError: If input format is not supported
+        FFmpegNotFoundError: If FFmpeg is not available
+        TranscodingFailedError: If transcoding fails
+    """
+    _check_ffmpeg_available()
+
+    input_path = Path(input_path)
+    output_path = Path(output_path)
+
+    if not validate_video_format(input_path):
+        raise UnsupportedFormatError(f'Unsupported video format: {input_path.suffix}')
+
+    if config is None:
+        config = TranscodeConfig()
+
+    # Ensure output directory exists
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    try:
+        # Build FFmpeg command
+        stream = _build_ffmpeg_stream(input_path, output_path, config)
+
+        # Run FFmpeg
+        if progress_callback:
+            # Get video duration for progress calculation
+            video_info = get_video_info(input_path)
+            total_duration = video_info.get('duration', 0)
+
+            # Run with progress monitoring
+            process = ffmpeg.run_async(stream, pipe_stderr=True, overwrite_output=True)
+
+            while True:
+                output = process.stderr.readline()
+                if output == b'' and process.poll() is not None:
+                    break
+                if output:
+                    line = output.decode('utf-8')
+                    # Parse progress from FFmpeg output
+                    if 'time=' in line and total_duration > 0:
+                        try:
+                            time_str = line.split('time=')[1].split()[0]
+                            hours, minutes, seconds = time_str.split(':')
+                            current_time = int(hours) * 3600 + int(minutes) * 60 + float(seconds)
+                            progress = min(current_time / total_duration, 1.0)
+                            progress_callback(progress)
+                        except (ValueError, IndexError):
+                            pass
+
+            if process.returncode != 0:
+                raise TranscodingFailedError('FFmpeg process failed')
+        else:
+            # Run without progress monitoring
+            ffmpeg.run(stream, overwrite_output=True, quiet=True)
+
+        return output_path
+
+    except ffmpeg.Error as e:
+        error_message = e.stderr.decode('utf-8') if e.stderr else str(e)
+        raise TranscodingFailedError(f'Transcoding failed: {error_message}')
+    except Exception as e:
+        raise VideoTranscodeError(f'Unexpected error during transcoding: {str(e)}')
+
+
+def optimize_for_web(video_path: str | Path, output_path: str | Path) -> Path:
+    """
+    Quick optimization for web streaming with default settings.
+
+    Args:
+        video_path (str | Path): Path to input video file
+        output_path (str | Path): Path to output video file
+
+    Returns:
+        Path: Path to the optimized video file
+    """
+    config = TranscodeConfig(
+        preset='fast',  # Faster encoding for web optimization
+        crf=23,  # Better quality for web
+        movflags='+faststart+frag_keyframe+empty_moov',  # Advanced web optimization
+    )
+    return transcode_video(video_path, output_path, config)
+
+
+async def atranscode_video(
+    input_path: str | Path, output_path: str | Path, config: Optional[TranscodeConfig] = None
+) -> Path:
+    """
+    Async version of transcode_video.
+
+    Args:
+        input_path (str | Path): Path to input video file
+        output_path (str | Path): Path to output video file
+        config (Optional[TranscodeConfig]): Transcoding configuration
+
+    Returns:
+        Path: Path to the transcoded video file
+    """
+    loop = asyncio.get_event_loop()
+    return await loop.run_in_executor(None, transcode_video, input_path, output_path, config)
+
+
+def transcode_batch(
+    video_paths: list[Path], output_dir: Path, config: Optional[TranscodeConfig] = None, max_workers: int = 4
+) -> list[Path]:
+    """
+    Process multiple videos concurrently.
+
+    Args:
+        video_paths (list[Path]): List of input video file paths
+        output_dir (Path): Directory for output files
+        config (Optional[TranscodeConfig]): Transcoding configuration
+        max_workers (int): Maximum number of concurrent workers
+
+    Returns:
+        list[Path]: List of paths to transcoded video files
+    """
+    import concurrent.futures
+
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    def process_video(video_path):
+        output_path = output_dir / f'{video_path.stem}_transcoded.mp4'
+        return transcode_video(video_path, output_path, config)
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        results = list(executor.map(process_video, video_paths))
+
+    return results

synapse_sdk/utils/file.py.backup

@@ -0,0 +1,301 @@
+import asyncio
+import base64
+import hashlib
+import json
+import mimetypes
+import operator
+import zipfile
+from functools import reduce
+from pathlib import Path
+from typing import IO, Any, Callable
+
+import aiohttp
+import requests
+import yaml
+
+from synapse_sdk.utils.network import clean_url
+from synapse_sdk.utils.string import hash_text
+
+
+def read_file_in_chunks(file_path, chunk_size=1024 * 1024 * 50):
+    """
+    Read a file in chunks for efficient memory usage during file processing.
+
+    This function is particularly useful for large files or when you need to process
+    files in chunks, such as for uploading or hashing.
+
+    Args:
+        file_path (str | Path): Path to the file to read
+        chunk_size (int, optional): Size of each chunk in bytes. Defaults to 50MB (1024 * 1024 * 50)
+
+    Yields:
+        bytes: File content chunks
+
+    Raises:
+        FileNotFoundError: If the file doesn't exist
+        PermissionError: If the file can't be read due to permissions
+        OSError: If there's an OS-level error reading the file
+
+    Example:
+        ```python
+        from synapse_sdk.utils.file import read_file_in_chunks
+
+        # Read a file in 10MB chunks
+        for chunk in read_file_in_chunks('large_file.bin', chunk_size=1024*1024*10):
+            process_chunk(chunk)
+        ```
+    """
+    with open(file_path, 'rb') as file:
+        while chunk := file.read(chunk_size):
+            yield chunk
+
+
+def download_file(url, path_download, name=None, coerce=None, use_cached=True):
+    chunk_size = 1024 * 1024 * 50
+    cleaned_url = clean_url(url)  # remove query params and fragment
+
+    if name:
+        use_cached = False
+    else:
+        name = hash_text(cleaned_url)
+
+    name += Path(cleaned_url).suffix
+
+    path = Path(path_download) / name
+
+    if not use_cached or not path.is_file():
+        response = requests.get(url, allow_redirects=True, stream=True)
+        response.raise_for_status()
+
+        with path.open('wb') as file:
+            for chunk in response.iter_content(chunk_size=chunk_size):
+                file.write(chunk)
+
+    if coerce:
+        path = coerce(path)
+
+    return path
+
+
+def files_url_to_path(files, coerce=None, file_field=None):
+    path_download = get_temp_path('media')
+    path_download.mkdir(parents=True, exist_ok=True)
+    if file_field:
+        files[file_field] = download_file(files[file_field], path_download, coerce=coerce)
+    else:
+        for file_name in files:
+            if isinstance(files[file_name], str):
+                files[file_name] = download_file(files[file_name], path_download, coerce=coerce)
+            else:
+                files[file_name]['path'] = download_file(files[file_name].pop('url'), path_download, coerce=coerce)
+
+
+def files_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False, is_async=False):
+    if is_async:
+        asyncio.run(afiles_url_to_path_from_objs(objs, files_fields, coerce=coerce, is_list=is_list))
+    else:
+        if not is_list:
+            objs = [objs]
+
+        for obj in objs:
+            for files_field in files_fields:
+                try:
+                    files = reduce(operator.getitem, files_field.split('.'), obj)
+                    if isinstance(files, str):
+                        files_url_to_path(obj, coerce=coerce, file_field=files_field)
+                    else:
+                        files_url_to_path(files, coerce=coerce)
+                except KeyError:
+                    pass
+
+
+async def adownload_file(url, path_download, name=None, coerce=None, use_cached=True):
+    chunk_size = 1024 * 1024 * 50
+    cleaned_url = clean_url(url)  # remove query params and fragment
+
+    if name:
+        use_cached = False
+    else:
+        name = hash_text(cleaned_url)
+
+    name += Path(cleaned_url).suffix
+
+    path = Path(path_download) / name
+
+    if not use_cached or not path.is_file():
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                with path.open('wb') as file:
+                    while chunk := await response.content.read(chunk_size):
+                        file.write(chunk)
+
+    if coerce:
+        path = coerce(path)
+
+    return path
+
+
+async def afiles_url_to_path(files, coerce=None):
+    path_download = get_temp_path('media')
+    path_download.mkdir(parents=True, exist_ok=True)
+    for file_name in files:
+        if isinstance(files[file_name], str):
+            files[file_name] = await adownload_file(files[file_name], path_download, coerce=coerce)
+        else:
+            files[file_name]['path'] = await adownload_file(files[file_name].pop('url'), path_download, coerce=coerce)
+
+
+async def afiles_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False):
+    if not is_list:
+        objs = [objs]
+
+    tasks = []
+
+    for obj in objs:
+        for files_field in files_fields:
+            try:
+                files = reduce(operator.getitem, files_field.split('.'), obj)
+                tasks.append(afiles_url_to_path(files, coerce=coerce))
+            except KeyError:
+                pass
+
+    await asyncio.gather(*tasks)
+
+
+def get_dict_from_file(file_path):
+    if isinstance(file_path, str):
+        file_path = Path(file_path)
+
+    with open(file_path) as f:
+        if file_path.suffix == '.yaml':
+            return yaml.safe_load(f)
+        else:
+            return json.load(f)
+
+
+def calculate_checksum(file_path, prefix=''):
+    md5_hash = hashlib.md5()
+    with open(file_path, 'rb') as f:
+        for byte_block in iter(lambda: f.read(4096), b''):
+            md5_hash.update(byte_block)
+    checksum = md5_hash.hexdigest()
+    if prefix:
+        return f'dev-{checksum}'
+    return checksum
+
+
+def get_checksum_from_file(file: IO[Any], digest_mod: Callable[[], Any] = hashlib.sha1) -> str:
+    """
+    Calculate checksum for a file-like object.
+
+    Args:
+        file (IO[Any]): File-like object with read() method that supports reading in chunks
+        digest_mod (Callable[[], Any]): Hash algorithm from hashlib (defaults to hashlib.sha1)
+
+    Returns:
+        str: Hexadecimal digest of the file contents
+
+    Example:
+        ```python
+        import hashlib
+        from io import BytesIO
+        from synapse_sdk.utils.file import get_checksum_from_file
+
+        # With BytesIO
+        data = BytesIO(b'Hello, world!')
+        checksum = get_checksum_from_file(data)
+
+        # With different hash algorithm
+        checksum = get_checksum_from_file(data, digest_mod=hashlib.sha256)
+        ```
+    """
+    digest = digest_mod()
+    chunk_size = 4096
+
+    # Reset file pointer to beginning if possible
+    if hasattr(file, 'seek'):
+        file.seek(0)
+
+    while True:
+        chunk = file.read(chunk_size)
+        if not chunk:
+            break
+        if isinstance(chunk, str):
+            chunk = chunk.encode('utf-8')
+        digest.update(chunk)
+
+    return digest.hexdigest()
+
+
+def archive(input_path, output_path, append=False):
+    input_path = Path(input_path)
+    output_path = Path(output_path)
+
+    mode = 'a' if append and output_path.exists() else 'w'
+    with zipfile.ZipFile(output_path, mode=mode, compression=zipfile.ZIP_DEFLATED) as zipf:
+        if input_path.is_file():
+            zipf.write(input_path, input_path.name)
+        else:
+            for file_path in input_path.rglob('*'):
+                if file_path.is_file():  # Only add files, skip directories
+                    arcname = file_path.relative_to(input_path.parent)
+                    zipf.write(file_path, arcname)
+
+
+def unarchive(file_path, output_path):
+    """
+    Unarchives a ZIP file to a given directory.
+
+    Parameters:
+        file_path (str | Path): The path to the ZIP file.
+        output_path (str): The directory where the files will be extracted.
+    """
+    output_path = Path(output_path)
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    with zipfile.ZipFile(str(file_path), 'r') as zip_ref:
+        zip_ref.extractall(output_path)
+
+
+def get_temp_path(sub_path=None):
+    path = Path('/tmp/datamaker')
+    if sub_path:
+        path = path / sub_path
+    return path
+
+
+def convert_file_to_base64(file_path):
+    """
+    Convert a file to base64 using pathlib.
+
+    Args:
+        file_path (str): Path to the file to convert
+
+    Returns:
+        str: Base64 encoded string of the file contents
+    """
+    # FIXME base64 is sent sometimes.
+    if file_path.startswith('data:'):
+        return file_path
+
+    # Convert string path to Path object
+    path = Path(file_path)
+
+    try:
+        # Read binary content of the file
+        binary_content = path.read_bytes()
+
+        # Convert to base64
+        base64_encoded = base64.b64encode(binary_content).decode('utf-8')
+
+        # Get the MIME type of the file
+        mime_type, _ = mimetypes.guess_type(path)
+        assert mime_type is not None, 'MIME type cannot be guessed'
+
+        # Convert bytes to string for readable output
+        return f'data:{mime_type};base64,{base64_encoded}'
+
+    except FileNotFoundError:
+        raise FileNotFoundError(f'File not found: {file_path}')
+    except Exception as e:
+        raise Exception(f'Error converting file to base64: {str(e)}')