synapse-sdk 1.0.0a35__py3-none-any.whl → 2025.11.7__py3-none-any.whl

synapse_sdk/utils/converters/yolo/to_dm.py

@@ -0,0 +1,267 @@
+import os
+from typing import IO, Any, Dict, List, Tuple
+
+import yaml
+from PIL import Image
+
+from synapse_sdk.utils.converters import ToDMConverter
+
+
+class YOLOToDMConverter(ToDMConverter):
+    """Convert YOLO formatted datasets to DM format."""
+
+    IMG_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.bmp']
+    CLASS_NAMES = []
+
+    def __init__(
+        self,
+        root_dir: str = None,
+        is_categorized_dataset: bool = False,
+        is_single_conversion: bool = False,
+        class_names: List[str] = None,
+    ):
+        super().__init__(root_dir, is_categorized_dataset, is_single_conversion)
+
+        # Set class names
+        self.CLASS_NAMES = class_names
+
+        # Get class names from dataset.yaml if not provided
+        if not class_names:
+            dataset_yaml_path = os.path.join(self.root_dir, 'dataset.yaml')
+            if not os.path.exists(dataset_yaml_path):
+                raise FileNotFoundError(f'No dataset.yaml file found in {self.root_dir}.')
+            with open(dataset_yaml_path, 'r', encoding='utf-8') as f:
+                dataset_yaml = yaml.safe_load(f)
+                self.CLASS_NAMES = dataset_yaml.get('names', [])
+
+    def convert(self):
+        """Convert YOLO dataset to DM format."""
+        if self.is_categorized_dataset:
+            splits = self._validate_splits(['train', 'valid'], ['test'])
+            all_split_data = {}
+            for split, split_dir in splits.items():
+                split_data = self._convert_yolo_split_to_dm(split_dir)
+                all_split_data[split] = split_data
+            self.converted_data = all_split_data
+            return all_split_data
+        else:
+            split_data = self._convert_yolo_split_to_dm(self.root_dir)
+            self.converted_data = split_data
+            return split_data
+
+    def _find_image_path(self, images_dir, base):
+        """Find the image file corresponding to the base name in the images directory."""
+        for ext in self.IMG_EXTENSIONS:
+            img_path = os.path.join(images_dir, base + ext)
+            if os.path.exists(img_path):
+                return img_path
+        return None
+
+    @staticmethod
+    def _get_image_size(image_path: str) -> Tuple[int, int]:
+        """Get the size of the image at the given path."""
+        with Image.open(image_path) as img:
+            return img.size
+
+    def _parse_yolo_line(self, line: str, class_names: List[str], img_size: Tuple[int, int]):
+        """Parse a single line from a YOLO label file."""
+        parts = line.strip().split()
+        if len(parts) < 5:
+            return None  # skip malformed
+
+        class_idx = int(parts[0])
+        class_name = class_names[class_idx] if class_idx < len(class_names) else f'class_{class_idx}'
+        img_w, img_h = img_size
+
+        # Check if it's a polygon (more than 5 values and even number of coordinates after class_id)
+        if len(parts) > 5 and (len(parts) - 1) % 2 == 0:
+            # Polygon format: class_id x1 y1 x2 y2 x3 y3 ... (normalized coordinates)
+            coords = []
+            for i in range(1, len(parts), 2):
+                x_norm = float(parts[i])
+                y_norm = float(parts[i + 1])
+                # Convert normalized coordinates to absolute coordinates
+                x_abs = int(x_norm * img_w)
+                y_abs = int(y_norm * img_h)
+                coords.append([x_abs, y_abs])
+
+            return {'type': 'polygon', 'classification': class_name, 'data': coords}
+
+        # Standard bounding box format
+        elif len(parts) == 5:
+            x_center, y_center, width, height = map(float, parts[1:5])
+
+            # Denormalize YOLO (x_center, y_center, w, h) to (left, top, w, h)
+            left = int((x_center - width / 2) * img_w)
+            top = int((y_center - height / 2) * img_h)
+            abs_w = int(width * img_w)
+            abs_h = int(height * img_h)
+
+            return {'type': 'bounding_box', 'classification': class_name, 'data': [left, top, abs_w, abs_h]}
+
+        # Keypoint format: class_id x_center y_center w h x1 y1 v1 x2 y2 v2 ...
+        elif len(parts) > 5 and (len(parts) - 5) % 3 == 0:
+            x_center, y_center, width, height = map(float, parts[1:5])
+
+            # Denormalize bounding box
+            left = int((x_center - width / 2) * img_w)
+            top = int((y_center - height / 2) * img_h)
+            abs_w = int(width * img_w)
+            abs_h = int(height * img_h)
+
+            keypoints = []
+            for i in range(5, len(parts), 3):
+                xk = int(float(parts[i]) * img_w)
+                yk = int(float(parts[i + 1]) * img_h)
+                vk = int(parts[i + 2])
+                keypoints.append([xk, yk, vk])
+
+            return {
+                'type': 'keypoint',
+                'classification': class_name,
+                'data': keypoints,
+                'bounding_box': [left, top, abs_w, abs_h],
+            }
+
+        return None
+
+    def _convert_yolo_split_to_dm(self, split_dir: str) -> Dict[str, Any]:
+        """Convert a single YOLO split directory to DM format."""
+        # Find image and label directories
+        images_dir = None
+        for candidate in ['images', 'img', 'imgs']:
+            candidate_path = os.path.join(split_dir, candidate)
+            if os.path.isdir(candidate_path):
+                images_dir = candidate_path
+                break
+        if images_dir is None:
+            raise FileNotFoundError(f"No images directory found in {split_dir} (tried 'images', 'img', 'imgs').")
+
+        labels_dir = os.path.join(split_dir, 'labels')
+        if not os.path.isdir(labels_dir):
+            raise FileNotFoundError(f"No labels directory found in {split_dir} (expected 'labels').")
+
+        # Build DM data
+        result = {}
+        for label_filename in os.listdir(labels_dir):
+            if not label_filename.endswith('.txt'):
+                continue
+            base = os.path.splitext(label_filename)[0]
+            img_path = self._find_image_path(images_dir, base)
+            if img_path is None:
+                print(f'[WARNING] Image not found for label {label_filename}, skipping.')
+                continue
+            img_size = self._get_image_size(img_path)
+            label_path = os.path.join(labels_dir, label_filename)
+            with open(label_path, 'r', encoding='utf-8') as f:
+                label_lines = [line.strip() for line in f if line.strip()]
+
+            # Prepare DM annotation structure
+            dm_img = {
+                'bounding_box': [],
+                'polygon': [],
+                'keypoint': [],
+                'relation': [],
+                'group': [],
+            }
+
+            for line in label_lines:
+                ann = self._parse_yolo_line(line, self.CLASS_NAMES, img_size)
+                if ann is None:
+                    continue
+
+                if ann['type'] == 'bounding_box':
+                    dm_img['bounding_box'].append({
+                        'id': self._generate_unique_id(),
+                        'classification': ann['classification'],
+                        'attrs': [],
+                        'data': ann['data'],
+                    })
+                elif ann['type'] == 'polygon':
+                    dm_img['polygon'].append({
+                        'id': self._generate_unique_id(),
+                        'classification': ann['classification'],
+                        'attrs': [],
+                        'data': ann['data'],
+                    })
+                elif ann['type'] == 'keypoint':
+                    dm_img['keypoint'].append({
+                        'id': self._generate_unique_id(),
+                        'classification': ann['classification'],
+                        'attrs': [],
+                        'data': ann['data'],
+                        'bounding_box': ann['bounding_box'],
+                    })
+
+            dm_json = {'images': [dm_img]}
+            result[os.path.basename(img_path)] = (dm_json, img_path)
+        return result
+
+    def convert_single_file(self, data: List[str], original_file: IO) -> Dict[str, Any]:
+        """Convert a single YOLO label data and corresponding image to DM format.
+
+        Args:
+            data: List of YOLO label lines (strings from .txt file content)
+            original_file: File object for the corresponding original image
+
+        Returns:
+            Dictionary containing DM format data for the single file
+        """
+        if not self.is_single_conversion:
+            raise RuntimeError('convert_single_file is only available when is_single_conversion=True')
+
+        img_path = getattr(original_file, 'name', None)
+        if not img_path:
+            raise ValueError('original_file must have a "name" attribute representing its path or filename.')
+
+        if hasattr(self, '_get_image_size'):
+            img_size = self._get_image_size(img_path)
+        else:
+            raise AttributeError('Converter missing _get_image_size method for file objects.')
+
+        # data is already a list of label lines
+        label_lines = [line.strip() for line in data if line.strip()]
+
+        # Prepare DM annotation structure
+        dm_img = {
+            'bounding_box': [],
+            'polygon': [],
+            'keypoint': [],
+            'relation': [],
+            'group': [],
+        }
+
+        for line in label_lines:
+            ann = self._parse_yolo_line(line, self.CLASS_NAMES, img_size)
+            if ann is None:
+                continue
+
+            if ann['type'] == 'bounding_box':
+                dm_img['bounding_box'].append({
+                    'id': self._generate_unique_id(),
+                    'classification': ann['classification'],
+                    'attrs': [],
+                    'data': ann['data'],
+                })
+            elif ann['type'] == 'polygon':
+                dm_img['polygon'].append({
+                    'id': self._generate_unique_id(),
+                    'classification': ann['classification'],
+                    'attrs': [],
+                    'data': ann['data'],
+                })
+            elif ann['type'] == 'keypoint':
+                dm_img['keypoint'].append({
+                    'id': self._generate_unique_id(),
+                    'classification': ann['classification'],
+                    'attrs': [],
+                    'data': ann['data'],
+                    'bounding_box': ann['bounding_box'],
+                })
+
+        dm_json = {'images': [dm_img]}
+        return {
+            'dm_json': dm_json,
+            'image_path': img_path,
+            'image_name': os.path.basename(img_path),
+        }

synapse_sdk/utils/dataset.py

@@ -0,0 +1,46 @@
+import logging
+
+
+def remap_class_labels(dataset, data_type, class_pattern, new_class):
+    """
+    Remaps class labels in a dataset by replacing specific classification patterns.
+
+    This function finds items of a specified data type (e.g., 'pcd', 'image', 'video')
+    where the classification matches a given pattern (e.g., 'truck', 'car'), and
+    remaps them to a new class label (e.g., 'vehicle'). Useful for consolidating
+    or standardizing class labels in ground truth data.
+
+    Args:
+        dataset (dict): The dataset containing ground truth labels
+        data_type (str): Type of data to process (e.g., 'pcd', 'image', 'video')
+        class_pattern (dict): Pattern to identify target classifications (e.g., {'name': 'truck'})
+        new_class (dict): New classification to apply (e.g., {'name': 'vehicle'})
+
+    Returns:
+        dict: Dataset with remapped class labels
+    """
+    updated_dataset = dataset.copy()
+
+    if data_type not in updated_dataset:
+        logging.log(logging.WARNING, f"Data type '{data_type}' not found in dataset.")
+        return updated_dataset
+
+    def matches_pattern(classification, pattern):
+        """Check if a classification matches the specified pattern."""
+        for key, pattern_value in pattern.items():
+            if key not in classification:
+                return False
+
+            if isinstance(pattern_value, dict) and isinstance(classification[key], dict):
+                if not matches_pattern(classification[key], pattern_value):
+                    return False
+            elif classification[key] != pattern_value:
+                return False
+
+        return True
+
+    for item in updated_dataset[data_type]:
+        if 'classification' in item and matches_pattern(item['classification'], class_pattern):
+            item['classification'] = new_class.copy()
+
+    return updated_dataset

synapse_sdk/utils/encryption.py

@@ -0,0 +1,158 @@
+"""Encryption utilities for plugin code security."""
+
+import base64
+import os
+import zipfile
+from io import BytesIO
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+from cryptography.fernet import Fernet
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+
+
+def generate_key(password: str, salt: bytes) -> bytes:
+    """Generate encryption key from password.
+
+    * Argument iterations should be at least 500,000.
+    """
+    kdf = PBKDF2HMAC(
+        algorithm=hashes.SHA256(),
+        length=32,
+        salt=salt,
+        iterations=700000,
+    )
+    return base64.urlsafe_b64encode(kdf.derive(password.encode()))
+
+
+def encrypt_data(data: bytes, password: str) -> Dict:
+    """Encrypt data with password."""
+    salt = os.urandom(16)
+    key = generate_key(password, salt)
+    fernet = Fernet(key)
+    encrypted_data = fernet.encrypt(data)
+
+    return {
+        'encrypted_data': base64.b64encode(encrypted_data).decode(),
+        'salt': base64.b64encode(salt).decode(),
+    }
+
+
+def decrypt_data(encrypted_package: Dict, password: str) -> bytes:
+    """Decrypt data with password."""
+    salt = base64.b64decode(encrypted_package['salt'])
+    encrypted_data = base64.b64decode(encrypted_package['encrypted_data'])
+
+    key = generate_key(password, salt)
+    fernet = Fernet(key)
+    return fernet.decrypt(encrypted_data)
+
+
+def is_plugin_directory(path: Path) -> bool:
+    """Check if directory contains a Synapse plugin."""
+    config_file = path / 'config.yaml'
+    plugin_dir = path / 'plugin'
+
+    return config_file.exists() and plugin_dir.exists() and plugin_dir.is_dir()
+
+
+def get_plugin_files(plugin_path: Path) -> List[Tuple[Path, str]]:
+    """Get all plugin files with their relative paths."""
+    plugin_files = []
+
+    # Essential plugin files
+    essential_patterns = [
+        'config.yaml',
+        'requirements.txt',
+        'README.md',
+        'pyproject.toml',
+    ]
+
+    for pattern in essential_patterns:
+        file_path = plugin_path / pattern
+        if file_path.exists():
+            plugin_files.append((file_path, pattern))
+
+    # Plugin source code
+    plugin_dir = plugin_path / 'plugin'
+    if plugin_dir.exists():
+        for file_path in plugin_dir.rglob('*'):
+            if file_path.is_file() and not file_path.name.startswith('.'):
+                relative_path = f'plugin/{file_path.relative_to(plugin_dir)}'
+                plugin_files.append((file_path, relative_path))
+
+    # Additional common directories
+    for additional_dir in ['tests', 'docs', 'data']:
+        dir_path = plugin_path / additional_dir
+        if dir_path.exists() and dir_path.is_dir():
+            for file_path in dir_path.rglob('*'):
+                if file_path.is_file() and not file_path.name.startswith('.'):
+                    relative_path = f'{additional_dir}/{file_path.relative_to(dir_path)}'
+                    plugin_files.append((file_path, relative_path))
+
+    return plugin_files
+
+
+def create_plugin_archive(plugin_path: Path) -> bytes:
+    """Create a zip archive of the plugin."""
+    plugin_files = get_plugin_files(plugin_path)
+
+    archive_buffer = BytesIO()
+    with zipfile.ZipFile(archive_buffer, 'w', zipfile.ZIP_DEFLATED) as zip_file:
+        for file_path, archive_path in plugin_files:
+            zip_file.write(file_path, archive_path)
+
+    return archive_buffer.getvalue()
+
+
+def encrypt_plugin(plugin_path: Path, password: Optional[str] = None) -> Dict:
+    """Encrypt a plugin directory."""
+    if not is_plugin_directory(plugin_path):
+        raise ValueError(f'Directory {plugin_path} is not a valid plugin directory')
+
+    # Generate password if not provided
+    if password is None:
+        password = base64.urlsafe_b64encode(os.urandom(32)).decode()
+
+    # Create plugin archive
+    archive_data = create_plugin_archive(plugin_path)
+
+    # Encrypt the archive
+    encrypted_package = encrypt_data(archive_data, password)
+
+    # Add metadata
+    encrypted_package.update({
+        'plugin_name': plugin_path.name,
+        'plugin_path': str(plugin_path),
+        'encryption_method': 'fernet',
+        'archive_format': 'zip',
+    })
+
+    return encrypted_package, password
+
+
+def get_plugin_info(plugin_path: Path) -> Optional[Dict]:
+    """Get basic plugin information."""
+    if not is_plugin_directory(plugin_path):
+        return None
+
+    info = {'name': plugin_path.name, 'path': str(plugin_path), 'is_plugin': True}
+
+    # Try to read config.yaml for additional info
+    config_file = plugin_path / 'config.yaml'
+    if config_file.exists():
+        try:
+            import yaml
+
+            with open(config_file, 'r', encoding='utf-8') as f:
+                config = yaml.safe_load(f)
+                info.update({
+                    'version': config.get('version', 'unknown'),
+                    'description': config.get('description', ''),
+                    'category': config.get('category', 'unknown'),
+                })
+        except Exception:
+            pass  # Continue without config details if parsing fails
+
+    return info

synapse_sdk/utils/file/__init__.py

@@ -0,0 +1,39 @@
+# File utilities module
+# Maintains backward compatibility by re-exporting all functions
+
+from .archive import archive, unarchive
+from .checksum import calculate_checksum, get_checksum_from_file
+from .chunking import read_file_in_chunks
+from .download import (
+    adownload_file,
+    afiles_url_to_path,
+    afiles_url_to_path_from_objs,
+    download_file,
+    files_url_to_path,
+    files_url_to_path_from_objs,
+)
+from .encoding import convert_file_to_base64
+from .io import get_dict_from_file, get_temp_path
+
+__all__ = [
+    # Chunking
+    'read_file_in_chunks',
+    # Download
+    'download_file',
+    'adownload_file',
+    'files_url_to_path',
+    'afiles_url_to_path',
+    'files_url_to_path_from_objs',
+    'afiles_url_to_path_from_objs',
+    # Checksum
+    'calculate_checksum',
+    'get_checksum_from_file',
+    # Archive
+    'archive',
+    'unarchive',
+    # Encoding
+    'convert_file_to_base64',
+    # I/O
+    'get_dict_from_file',
+    'get_temp_path',
+]

synapse_sdk/utils/file/archive.py

@@ -0,0 +1,32 @@
+import zipfile
+from pathlib import Path
+
+
+def archive(input_path, output_path, append=False):
+    input_path = Path(input_path)
+    output_path = Path(output_path)
+
+    mode = 'a' if append and output_path.exists() else 'w'
+    with zipfile.ZipFile(output_path, mode=mode, compression=zipfile.ZIP_DEFLATED) as zipf:
+        if input_path.is_file():
+            zipf.write(input_path, input_path.name)
+        else:
+            for file_path in input_path.rglob('*'):
+                if file_path.is_file():  # Only add files, skip directories
+                    arcname = file_path.relative_to(input_path.parent)
+                    zipf.write(file_path, arcname)
+
+
+def unarchive(file_path, output_path):
+    """
+    Unarchives a ZIP file to a given directory.
+
+    Parameters:
+        file_path (str | Path): The path to the ZIP file.
+        output_path (str): The directory where the files will be extracted.
+    """
+    output_path = Path(output_path)
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    with zipfile.ZipFile(str(file_path), 'r') as zip_ref:
+        zip_ref.extractall(output_path)

synapse_sdk/utils/file/checksum.py

@@ -0,0 +1,56 @@
+import hashlib
+from typing import IO, Any, Callable
+
+
+def calculate_checksum(file_path, prefix=''):
+    md5_hash = hashlib.md5()
+    with open(file_path, 'rb') as f:
+        for byte_block in iter(lambda: f.read(4096), b''):
+            md5_hash.update(byte_block)
+    checksum = md5_hash.hexdigest()
+    if prefix:
+        return f'dev-{checksum}'
+    return checksum
+
+
+def get_checksum_from_file(file: IO[Any], digest_mod: Callable[[], Any] = hashlib.sha1) -> str:
+    """
+    Calculate checksum for a file-like object.
+
+    Args:
+        file (IO[Any]): File-like object with read() method that supports reading in chunks
+        digest_mod (Callable[[], Any]): Hash algorithm from hashlib (defaults to hashlib.sha1)
+
+    Returns:
+        str: Hexadecimal digest of the file contents
+
+    Example:
+        ```python
+        import hashlib
+        from io import BytesIO
+        from synapse_sdk.utils.file import get_checksum_from_file
+
+        # With BytesIO
+        data = BytesIO(b'Hello, world!')
+        checksum = get_checksum_from_file(data)
+
+        # With different hash algorithm
+        checksum = get_checksum_from_file(data, digest_mod=hashlib.sha256)
+        ```
+    """
+    digest = digest_mod()
+    chunk_size = 4096
+
+    # Reset file pointer to beginning if possible
+    if hasattr(file, 'seek'):
+        file.seek(0)
+
+    while True:
+        chunk = file.read(chunk_size)
+        if not chunk:
+            break
+        if isinstance(chunk, str):
+            chunk = chunk.encode('utf-8')
+        digest.update(chunk)
+
+    return digest.hexdigest()

synapse_sdk/utils/file/chunking.py

@@ -0,0 +1,31 @@
+def read_file_in_chunks(file_path, chunk_size=1024 * 1024 * 50):
+    """
+    Read a file in chunks for efficient memory usage during file processing.
+
+    This function is particularly useful for large files or when you need to process
+    files in chunks, such as for uploading or hashing.
+
+    Args:
+        file_path (str | Path): Path to the file to read
+        chunk_size (int, optional): Size of each chunk in bytes. Defaults to 50MB (1024 * 1024 * 50)
+
+    Yields:
+        bytes: File content chunks
+
+    Raises:
+        FileNotFoundError: If the file doesn't exist
+        PermissionError: If the file can't be read due to permissions
+        OSError: If there's an OS-level error reading the file
+
+    Example:
+        ```python
+        from synapse_sdk.utils.file import read_file_in_chunks
+
+        # Read a file in 10MB chunks
+        for chunk in read_file_in_chunks('large_file.bin', chunk_size=1024*1024*10):
+            process_chunk(chunk)
+        ```
+    """
+    with open(file_path, 'rb') as file:
+        while chunk := file.read(chunk_size):
+            yield chunk