polystore 0.1.2__py3-none-any.whl

polystore/base.py

@@ -0,0 +1,318 @@
+"""
+Abstract base classes for storage backends.
+
+This module defines the fundamental interfaces for storage backends,
+independent of specific implementations. It establishes the contract
+that all storage backends must fulfill.
+"""
+
+import logging
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, Union
+
+from .exceptions import StorageResolutionError
+from .registry import AutoRegisterMeta
+
+logger = logging.getLogger(__name__)
+
+
+class DataSink(ABC):
+    """
+    Abstract base class for data destinations.
+
+    Defines the minimal interface for sending data to any destination,
+    whether storage, streaming, or other data handling systems.
+    """
+
+    @abstractmethod
+    def save(self, data: Any, identifier: Union[str, Path], **kwargs) -> None:
+        """
+        Send data to the destination.
+
+        Args:
+            data: The data to send
+            identifier: Unique identifier for the data (path-like for compatibility)
+            **kwargs: Backend-specific arguments
+
+        Raises:
+            TypeError: If identifier is not a valid type
+            ValueError: If data cannot be sent to destination
+        """
+        pass
+
+    @abstractmethod
+    def save_batch(
+        self, data_list: List[Any], identifiers: List[Union[str, Path]], **kwargs
+    ) -> None:
+        """
+        Send multiple data objects to the destination in a single operation.
+
+        Args:
+            data_list: List of data objects to send
+            identifiers: List of unique identifiers (must match length of data_list)
+            **kwargs: Backend-specific arguments
+
+        Raises:
+            ValueError: If data_list and identifiers have different lengths
+            TypeError: If any identifier is not a valid type
+            ValueError: If any data cannot be sent to destination
+        """
+        pass
+
+
+class DataSource(ABC):
+    """
+    Abstract base class for read-only data sources.
+
+    Defines the minimal interface for loading data from any source,
+    whether filesystem, virtual workspace, remote storage, or databases.
+    """
+
+    @abstractmethod
+    def load(self, file_path: Union[str, Path], **kwargs) -> Any:
+        """
+        Load data from a file path.
+
+        Args:
+            file_path: Path to the file to load
+            **kwargs: Backend-specific arguments
+
+        Raises:
+            FileNotFoundError: If the file does not exist
+            TypeError: If file_path is not a valid type
+            ValueError: If the data cannot be loaded
+        """
+        pass
+
+    @abstractmethod
+    def load_batch(self, file_paths: List[Union[str, Path]], **kwargs) -> List[Any]:
+        """
+        Load multiple files in a single batch operation.
+
+        Args:
+            file_paths: List of file paths to load
+            **kwargs: Backend-specific arguments
+
+        Raises:
+            FileNotFoundError: If any file does not exist
+            TypeError: If any file_path is not a valid type
+            ValueError: If any data cannot be loaded
+        """
+        pass
+
+    @abstractmethod
+    def list_files(
+        self,
+        directory: Union[str, Path],
+        pattern: Optional[str] = None,
+        extensions: Optional[Set[str]] = None,
+        recursive: bool = False,
+        **kwargs,
+    ) -> List[str]:
+        """
+        List files in a directory.
+
+        Args:
+            directory: Directory to list files from
+            pattern: Optional glob pattern to filter files
+            extensions: Optional set of file extensions to filter (e.g., {'.tif', '.png'})
+            recursive: Whether to search recursively
+            **kwargs: Backend-specific arguments
+
+        Returns:
+            List of file paths (absolute or relative depending on backend)
+        """
+        pass
+
+    @abstractmethod
+    def exists(self, path: Union[str, Path]) -> bool:
+        """Check if a path exists."""
+        pass
+
+    @abstractmethod
+    def is_file(self, path: Union[str, Path]) -> bool:
+        """Check if a path is a file."""
+        pass
+
+    @abstractmethod
+    def is_dir(self, path: Union[str, Path]) -> bool:
+        """Check if a path is a directory."""
+        pass
+
+    @abstractmethod
+    def list_dir(self, path: Union[str, Path]) -> List[str]:
+        """List immediate entries in a directory (names only)."""
+        pass
+
+
+class VirtualBackend(DataSink):
+    """
+    Abstract base for backends that provide virtual filesystem semantics.
+
+    Virtual backends generate file listings on-demand without real filesystem operations.
+    Examples: OMERO (generates filenames from plate structure), S3 (lists objects), HTTP APIs.
+    """
+
+    @abstractmethod
+    def load(self, file_path: Union[str, Path], **kwargs) -> Any:
+        """
+        Load data from virtual path.
+
+        Args:
+            file_path: Virtual path to load
+            **kwargs: Backend-specific context
+
+        Returns:
+            The loaded data
+
+        Raises:
+            FileNotFoundError: If the virtual path does not exist
+            TypeError: If required kwargs are missing
+            ValueError: If the data cannot be loaded
+        """
+        pass
+
+    @abstractmethod
+    def load_batch(self, file_paths: List[Union[str, Path]], **kwargs) -> List[Any]:
+        """
+        Load multiple virtual paths in a single batch operation.
+
+        Args:
+            file_paths: List of virtual paths to load
+            **kwargs: Backend-specific context
+
+        Returns:
+            List of loaded data objects in the same order as file_paths
+
+        Raises:
+            FileNotFoundError: If any virtual path does not exist
+            TypeError: If required kwargs are missing
+            ValueError: If any data cannot be loaded
+        """
+        pass
+
+    @abstractmethod
+    def list_files(
+        self,
+        directory: Union[str, Path],
+        pattern: Optional[str] = None,
+        extensions: Optional[Set[str]] = None,
+        recursive: bool = False,
+        **kwargs,
+    ) -> List[str]:
+        """
+        Generate virtual file listing.
+
+        Args:
+            directory: Virtual directory path
+            pattern: Optional file pattern filter
+            extensions: Optional set of file extensions to filter
+            recursive: Whether to list recursively
+            **kwargs: Backend-specific context
+
+        Returns:
+            List of virtual filenames
+
+        Raises:
+            TypeError: If required kwargs are missing
+            ValueError: If directory is invalid
+        """
+        pass
+
+    @property
+    def requires_filesystem_validation(self) -> bool:
+        """
+        Whether this backend requires filesystem validation.
+
+        Virtual backends return False - they don't have real filesystem paths.
+        Real backends return True - they need path validation.
+
+        Returns:
+            False for virtual backends
+        """
+        return False
+
+
+class BackendBase(metaclass=AutoRegisterMeta):
+    """
+    Base class for all storage backends (read-only and read-write).
+
+    Defines the registry and common interface for backend discovery.
+    Concrete backends should inherit from StorageBackend or ReadOnlyBackend.
+    """
+
+    __registry_key__ = "_backend_type"
+
+    @property
+    @abstractmethod
+    def requires_filesystem_validation(self) -> bool:
+        """Whether this backend requires filesystem validation."""
+        pass
+
+
+class ReadOnlyBackend(BackendBase, DataSource):
+    """
+    Abstract base class for read-only storage backends with auto-registration.
+
+    Use this for backends that only need to read data (virtual workspaces,
+    read-only mounts, archive viewers, etc.).
+    """
+
+    @property
+    def requires_filesystem_validation(self) -> bool:
+        """
+        Whether this backend requires filesystem validation.
+
+        Returns:
+            False for virtual/remote backends, True for local filesystem
+        """
+        return False
+
+
+class StorageBackend(BackendBase, DataSource, DataSink):
+    """
+    Abstract base class for read-write storage backends.
+
+    Extends DataSource (read) and DataSink (write) with file system operations
+    for backends that provide persistent storage with file-like semantics.
+
+    Concrete implementations are automatically registered via AutoRegisterMeta.
+    """
+
+    @property
+    def requires_filesystem_validation(self) -> bool:
+        """
+        Whether this backend requires filesystem validation.
+
+        Returns:
+            True for real filesystem backends (default for StorageBackend)
+        """
+        return True
+
+    def exists(self, path: Union[str, Path]) -> bool:
+        """
+        Check if a path exists (is a valid file or directory).
+
+        Args:
+            path: Path to check
+
+        Returns:
+            bool: True if path resolves to a real object
+        """
+        try:
+            return self.is_file(path)
+        except (FileNotFoundError, NotADirectoryError, StorageResolutionError):
+            pass
+        except IsADirectoryError:
+            # Path exists but is a directory
+            try:
+                return self.is_dir(path)
+            except (FileNotFoundError, NotADirectoryError, StorageResolutionError):
+                return False
+
+        # If is_file failed for other reasons, try is_dir
+        try:
+            return self.is_dir(path)
+        except (FileNotFoundError, NotADirectoryError, StorageResolutionError):
+            return False