synapse-sdk 1.0.0b5__py3-none-any.whl → 2025.12.3__py3-none-any.whl

synapse_sdk/plugins/utils/ray_gcs.py

@@ -0,0 +1,66 @@
+"""Utilities for converting HTTP URLs to Ray GCS URLs."""
+
+import tempfile
+from pathlib import Path
+
+
+def convert_http_to_ray_gcs(http_url: str) -> str:
+    """Convert HTTP URL to Ray GCS URL by downloading and uploading to Ray's GCS.
+
+    Ray's working_dir only accepts certain protocols (gcs://, s3://, gs://, https://).
+    When SYNAPSE_PLUGIN_STORAGE is an HTTP URL (Django media server), this function
+    converts it to a Ray-compatible gcs:// URL by:
+    1. Downloading the file from HTTP
+    2. Uploading it to Ray's Global Control Store (GCS)
+    3. Returning the content-addressable gcs:// URI
+
+    Args:
+        http_url: HTTP/HTTPS URL to plugin zip file
+
+    Returns:
+        gcs:// URL that Ray can use for working_dir
+        Example: "gcs://_ray_pkg_abc123def456.zip"
+
+    Raises:
+        RuntimeError: If Ray is not initialized or not installed
+        requests.exceptions.RequestException: If HTTP download fails
+
+    Note:
+        - Ray must be initialized (ray.init()) before calling this function
+        - The gcs:// URI is content-addressable (same file = same URI)
+        - Ray automatically deduplicates uploads via package_exists()
+    """
+    try:
+        import ray
+    except ImportError:
+        raise RuntimeError(
+            'Ray is not installed but is required for HTTP → GCS conversion. Install ray with: pip install ray'
+        )
+
+    if not ray.is_initialized():
+        raise RuntimeError(
+            'Ray must be initialized before converting HTTP URLs to GCS. '
+            'Call ray.init() before submitting jobs with HTTP storage.'
+        )
+
+    from ray._private.runtime_env.packaging import (
+        get_uri_for_package,
+        package_exists,
+        upload_package_to_gcs,
+    )
+
+    from synapse_sdk.plugins.upload import download_file
+
+    # Download HTTP file to temporary location
+    with tempfile.TemporaryDirectory() as temp_dir:
+        local_path = Path(download_file(http_url, temp_dir))
+
+        # Generate content-addressable gcs:// URI based on file content
+        gcs_uri = get_uri_for_package(local_path)
+
+        # Check if already exists in Ray GCS (deduplication)
+        if not package_exists(gcs_uri):
+            # Upload to Ray's Global Control Store
+            upload_package_to_gcs(gcs_uri, local_path.read_bytes())
+
+        return gcs_uri

synapse_sdk/shared/__init__.py

@@ -0,0 +1,25 @@
+import os
+
+
+def needs_sentry_init():
+    return os.getenv('SENTRY_DSN') is not None
+
+
+def init_sentry():
+    import sentry_sdk
+    from sentry_sdk.integrations.ray import RayIntegration
+
+    dsn = os.getenv('SENTRY_DSN')
+    if dsn is None:
+        return
+
+    sentry_sdk.init(
+        dsn=dsn,
+        environment=os.getenv('DEPLOYMENT_TARGET', 'development'),
+        integrations=[RayIntegration()],
+        send_default_pii=True,
+    )
+
+
+def worker_process_setup_hook(*_, **__):
+    init_sentry()

synapse_sdk/utils/converters/dm/__init__.py

@@ -1,56 +1,57 @@
-from abc import ABC, abstractmethod
+"""
+DM Schema V1/V2 Bidirectional Converter
 
-from synapse_sdk.shared.enums import SupportedTools
+"""
 
+from typing import Any
 
-class BaseDMConverter(ABC):
-    """Base class for DM format converters."""
+from .types import (
+    AnnotationMeta,
+    V2AnnotationData,
+    V2ConversionResult,
+)
 
-    SUPPORTED_TOOLS = SupportedTools.get_all_values()
 
-    def __init__(self, file_type=None):
-        """Initialize the base converter.
+def convert_v1_to_v2(v1_data: dict[str, Any]) -> V2ConversionResult:
+    """Convert DM Schema V1 data to V2 (separated result)
 
-        Args:
-            file_type (str, optional): Type of file being converted (image, video, pcd, text, audio)
-        """
-        self.file_type = file_type
-        self.tool_processors = self._setup_tool_processors()
+    Args:
+        v1_data: DM Schema V1 format data
 
-    def _setup_tool_processors(self):
-        """Setup tool processor mapping dynamically based on file_type."""
-        if not self.file_type:
-            return {}
+    Returns:
+        V2ConversionResult: Separated conversion result
+            - annotation_data: V2 common annotation structure
+            - annotation_meta: Preserved V1 top-level structure
+    """
+    from .from_v1 import DMV1ToV2Converter
 
-        processors = {}
-        tools = SupportedTools.get_tools_for_file_type(self.file_type)
+    converter = DMV1ToV2Converter()
+    return converter.convert(v1_data)
 
-        for tool in tools:
-            # For other tools, use generic method names
-            method_name = f'_convert_{tool.method_name}'
 
-            if hasattr(self, method_name):
-                processors[tool.annotation_tool] = getattr(self, method_name)
+def convert_v2_to_v1(
+    v2_data: V2ConversionResult | dict[str, Any],
+    annotation_meta: AnnotationMeta | None = None,
+) -> dict[str, Any]:
+    """Convert DM Schema V2 data to V1
 
-        return processors
+    Args:
+        v2_data: DM Schema V2 format data
+        annotation_meta: Optional V1 top-level structure passed separately
 
-    @abstractmethod
-    def convert(self):
-        """Convert data from one format to another."""
+    Returns:
+        DM Schema V1 format data
+    """
+    from .to_v1 import DMV2ToV1Converter
 
-    def _handle_unknown_tool(self, tool_type, item_id=None):
-        """Handle unknown tool types with consistent warning message."""
-        warning_msg = f"Warning: Unknown tool type '{tool_type}'"
-        if item_id:
-            warning_msg += f' for item {item_id}'
-        print(warning_msg)
+    converter = DMV2ToV1Converter()
+    return converter.convert(v2_data, annotation_meta)
 
-    def _extract_media_type_info(self, media_id):
-        """Extract media type information from media ID."""
-        media_type = media_id.split('_')[0] if '_' in media_id else media_id
-        media_type_plural = media_type + 's' if not media_type.endswith('s') else media_type
-        return media_type, media_type_plural
 
-    def _singularize_media_type(self, media_type_plural):
-        """Convert plural media type to singular."""
-        return media_type_plural.rstrip('s')
+__all__ = [
+    'convert_v1_to_v2',
+    'convert_v2_to_v1',
+    'V2ConversionResult',
+    'V2AnnotationData',
+    'AnnotationMeta',
+]

synapse_sdk/utils/converters/dm/base.py

@@ -0,0 +1,137 @@
+"""
+DM Schema V1/V2 Converter Base Class
+
+Created: 2025-12-11
+"""
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
+from .types import MEDIA_TYPE_MAP, SUPPORTED_FILE_TYPES
+from .utils import detect_file_type, extract_media_type_info
+
+if TYPE_CHECKING:
+    from .tools import ToolProcessor
+
+
+class BaseDMConverter(ABC):
+    """DM Schema Converter Base Class
+
+    Abstract base class for all DM converters.
+
+    Attributes:
+        file_type: File type to process (None for auto-detection)
+        SUPPORTED_FILE_TYPES: Tuple of supported file types
+        MEDIA_TYPE_MAP: Media type mapping dictionary
+
+    Example:
+        >>> class MyConverter(BaseDMConverter):
+        ...     def convert(self, data):
+        ...         # implementation
+        ...         pass
+    """
+
+    SUPPORTED_FILE_TYPES = SUPPORTED_FILE_TYPES
+    MEDIA_TYPE_MAP = MEDIA_TYPE_MAP
+
+    def __init__(self, file_type: str | None = None) -> None:
+        """
+        Args:
+            file_type: File type to process (None for auto-detection)
+
+        Raises:
+            ValueError: Unsupported file type
+        """
+        if file_type is not None and file_type not in self.SUPPORTED_FILE_TYPES:
+            raise ValueError(
+                f'Unsupported file type: {file_type}. Supported types: {", ".join(self.SUPPORTED_FILE_TYPES)}'
+            )
+        self.file_type = file_type
+        self._tool_processors: dict[str, 'ToolProcessor'] = {}
+        self._setup_tool_processors()
+
+    @abstractmethod
+    def _setup_tool_processors(self) -> None:
+        """Register tool processors
+
+        Subclasses implement this to register supported tool processors.
+
+        Example:
+            >>> def _setup_tool_processors(self):
+            ...     from .tools import BoundingBoxProcessor, PolygonProcessor
+            ...     self.register_processor(BoundingBoxProcessor())
+            ...     self.register_processor(PolygonProcessor())
+        """
+        ...
+
+    def register_processor(self, processor: 'ToolProcessor') -> None:
+        """Register a tool processor
+
+        Use this method to register processors when adding new tool support.
+        Allows extension without modifying existing code (AR-001).
+
+        Args:
+            processor: ToolProcessor implementation
+
+        Example:
+            >>> class KeypointProcessor:
+            ...     tool_name = "keypoint"
+            ...     def to_v2(self, v1_annotation, v1_data): ...
+            ...     def to_v1(self, v2_annotation): ...
+            >>> converter.register_processor(KeypointProcessor())
+        """
+        self._tool_processors[processor.tool_name] = processor
+
+    def get_processor(self, tool_name: str) -> 'ToolProcessor | None':
+        """Get a registered tool processor
+
+        Args:
+            tool_name: Tool name (e.g., 'bounding_box', 'polygon')
+
+        Returns:
+            Registered processor or None
+        """
+        return self._tool_processors.get(tool_name)
+
+    @abstractmethod
+    def convert(self, data: dict[str, Any]) -> dict[str, Any]:
+        """Perform data conversion
+
+        Args:
+            data: Input data (V1 or V2)
+
+        Returns:
+            Converted data (V2 or V1)
+
+        Raises:
+            ValueError: Data cannot be converted
+        """
+        ...
+
+    def _detect_file_type(self, data: dict[str, Any], is_v2: bool = False) -> str:
+        """Auto-detect file type from data
+
+        Args:
+            data: Input data
+            is_v2: Whether the format is V2
+
+        Returns:
+            Detected file type ('image', 'video', etc.)
+
+        Raises:
+            ValueError: Unable to detect file type
+        """
+        if self.file_type:
+            return self.file_type
+        return detect_file_type(data, is_v2)
+
+    def _extract_media_type_info(self, media_id: str) -> tuple[str, str]:
+        """Extract type information from media ID
+
+        Args:
+            media_id: Media ID (e.g., 'image_1', 'video_2')
+
+        Returns:
+            (singular, plural) tuple (e.g., ('image', 'images'))
+        """
+        return extract_media_type_info(media_id)