diffract-core 0.2.1__py3-none-any.whl

diffract/core/data/proxy.py

@@ -0,0 +1,246 @@
+"""Generic data proxy for lazy-loading storage-backed entities.
+
+This module provides the base DataProxy class that can be specialized
+for different data types (parameters, relations, etc.).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, TypeVar
+
+from diffract.core.storage.interface import DEFAULT_TABLE
+
+if TYPE_CHECKING:
+    from .interface import IDataRepository
+
+TMetadata = TypeVar("TMetadata")
+T = TypeVar("T")
+
+
+@dataclass(kw_only=True)
+class DataProxy[TMetadata]:
+    """Generic lazy-loading proxy for storage-backed data entities.
+
+    Provides efficient access to data through intelligent caching and
+    storage management. Entities are loaded on-demand and can be
+    prefetched for batch operations. The proxy pattern allows working
+    with large datasets without memory constraints.
+
+    Type Parameters:
+        TMetadata: The metadata type for this proxy (e.g., ParameterMetadata).
+
+    Example:
+        >>> proxy = DataProxy.create_and_store(
+        ...     meta=metadata,
+        ...     repository=repository,
+        ... )
+        >>> value = proxy.get_field("weights")
+        >>> proxy.set_field("computed_metric", computed_value)
+
+    Attributes:
+        meta: Immutable entity metadata.
+        _repository: Repository that owns storage/cache/metadata managers.
+    """
+
+    meta: TMetadata
+    _repository: IDataRepository[TMetadata, DataProxy[TMetadata]] = field(repr=False)
+
+    @classmethod
+    def get_table(cls) -> str:
+        """Return the storage table name for this proxy type.
+
+        Override in subclasses to specify a different table.
+        """
+        return DEFAULT_TABLE
+
+    @classmethod
+    def create_and_store(
+        cls,
+        meta: TMetadata,
+        repository: IDataRepository[TMetadata, DataProxy[TMetadata]],
+    ) -> DataProxy[TMetadata]:
+        """Create proxy and store metadata in index.
+
+        Factory method that creates a new proxy and immediately
+        stores the metadata to the metadata index.
+
+        Args:
+            meta: Entity metadata.
+            repository: Repository owning storage, cache, and metadata managers.
+
+        Returns:
+            New proxy with metadata stored.
+        """
+        # Store full metadata in the index
+        repository.metadata_index.upsert(
+            cls.get_table(),
+            **meta.to_dict(),
+        )
+
+        proxy = cls(
+            meta=meta,
+            _repository=repository,
+        )
+        repository.append(proxy)
+        return proxy
+
+    def set_field(self, name: str, value: Any) -> None:
+        """Store a named field for this entity.
+
+        Args:
+            name: Field name to store.
+            value: Value to store (must be serializable).
+        """
+        self._repository.storage_manager.set_field(
+            obj_uid=self.meta.uid,
+            field_name=name,
+            value=value,
+            table=self.get_table(),
+        )
+        self.prefetch_field(name, value)
+
+    def get_field(
+        self, name: str, default: T | None = None, auto_prefetch: bool = True
+    ) -> Any | T:
+        """Retrieve a named field for this entity.
+
+        Checks cache first for performance, falls back to storage if needed.
+        Optionally prefetches the field to cache for future access.
+
+        Args:
+            name: Field name to retrieve.
+            default: Default value if field doesn't exist.
+            auto_prefetch: Whether to cache the value after loading.
+
+        Returns:
+            Field value or default if not found.
+
+        Raises:
+            KeyError: If field doesn't exist and no default provided.
+        """
+        cache = self._repository.cache_manager
+        if (cache is not None) and self.is_field_prefetched(name):
+            value = cache.get_field(obj_uid=self.meta.uid, field_name=name)
+        else:
+            try:
+                value = self._repository.storage_manager.get_field(
+                    obj_uid=self.meta.uid, field_name=name, table=self.get_table()
+                )
+
+            except KeyError:
+                if default is not None:
+                    value = default
+                else:
+                    raise
+
+            if auto_prefetch:
+                self.prefetch_field(name, value)
+
+        return value
+
+    def has_field(self, name: str) -> bool:
+        """Check if entity has a named field.
+
+        Checks both cache and storage for field existence.
+
+        Args:
+            name: Field name to check.
+
+        Returns:
+            True if field exists, False otherwise.
+        """
+        cache = self._repository.cache_manager
+        if (cache is not None) and (self.is_field_prefetched(name)):
+            return True
+        return self._repository.storage_manager.has_field(
+            obj_uid=self.meta.uid,
+            field_name=name,
+            table=self.get_table(),
+        )
+
+    def get_field_metadata(self, name: str) -> dict[str, Any] | None:
+        """Return storage-level metadata for the given field if available."""
+        return self._repository.storage_manager.get_field_metadata(
+            obj_uid=self.meta.uid, field_name=name, table=self.get_table()
+        )
+
+    def is_field_prefetched(self, name: str) -> bool:
+        """Check if field is cached for fast access.
+
+        Args:
+            name: Field name to check.
+
+        Returns:
+            True if field is in cache, False otherwise.
+        """
+        cache = self._repository.cache_manager
+        if cache is None:
+            return True
+        return cache.has_field(obj_uid=self.meta.uid, field_name=name)
+
+    def prefetch_field(self, name: str, value: Any) -> None:
+        """Cache a field value for fast access.
+
+        Args:
+            name: Field name to cache.
+            value: Value to cache.
+        """
+        cache = self._repository.cache_manager
+        if cache is not None:
+            cache.set_field(obj_uid=self.meta.uid, field_name=name, value=value)
+
+    def try_prefetch_field(self, name: str) -> bool:
+        """Attempt to prefetch a field into cache.
+
+        Args:
+            name: Field name to prefetch.
+
+        Returns:
+            True if field was prefetched or already cached, False if not found.
+        """
+        if not self.is_field_prefetched(name):
+            try:
+                value = self._repository.storage_manager.get_field(
+                    obj_uid=self.meta.uid, field_name=name, table=self.get_table()
+                )
+
+            except KeyError:
+                return False
+
+            self.prefetch_field(name, value)
+
+        return True
+
+    def erase_field(self, name: str) -> None:
+        """Remove a field from storage and cache.
+
+        Args:
+            name: Field name to remove.
+        """
+        self._repository.storage_manager.erase_field(
+            obj_uid=self.meta.uid,
+            field_name=name,
+            table=self.get_table(),
+        )
+        cache = self._repository.cache_manager
+        if (cache is not None) and self.is_field_prefetched(name):
+            cache.erase_field(obj_uid=self.meta.uid, field_name=name)
+
+    def list_fields(self) -> list[str]:
+        """List all field names for this entity.
+
+        Returns:
+            List of field names stored for this entity.
+        """
+        return self._repository.storage_manager.list_fields(
+            obj_uid=self.meta.uid, table=self.get_table()
+        )
+
+    def __hash__(self) -> int:
+        """Return hash based on unique identifier.
+
+        Returns:
+            Hash of the entity UID.
+        """
+        return hash(self.meta.uid)

diffract/core/data/repository.py

@@ -0,0 +1,227 @@
+"""Generic data repository for managing storage, metadata index, and cache.
+
+This module provides the base DataRepository class that can be specialized
+for different data types (parameters, relations, etc.).
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any, ClassVar, Self, TypeVar
+
+from diffract.core.parallel import ParallelContext
+from diffract.core.storage.interface import DEFAULT_TABLE
+
+from .view import DataView
+
+if TYPE_CHECKING:
+    import types
+    from collections.abc import Iterable, Iterator
+
+    from diffract.core.cache.interface import ICacheManager
+    from diffract.core.data.metadata.interface import IMetadataIndex
+    from diffract.core.storage.interface import IStorageManager
+
+TMetadata = TypeVar("TMetadata")
+TProxy = TypeVar("TProxy")
+
+logger = logging.getLogger(__name__)
+
+
+class DataRepository[TMetadata, TProxy]:
+    """Generic repository owning storage/cache/metadata and managing entity membership.
+
+    Provides the infrastructure for persistent storage, metadata indexing, and
+    caching of metadata-bearing entities. Subclasses specify the metadata type,
+    proxy class, view class, storage table, and metadata schema.
+
+    Type Parameters:
+        TMetadata: The metadata type for entities in this repository.
+        TProxy: The proxy type for entities in this repository.
+    """
+
+    # Override in subclasses
+    METADATA_CLASS: type[TMetadata]
+    PROXY_CLASS: type[TProxy]
+    VIEW_CLASS: type[DataView[TMetadata, TProxy]]
+    TABLE: str = DEFAULT_TABLE
+
+    # Schema for MetadataIndex - override in subclasses
+    METADATA_COLUMNS: ClassVar[dict[str, type]] = {}
+    METADATA_INDEXES: ClassVar[list[str]] = []
+
+    def __init__(
+        self,
+        storage_manager: IStorageManager,
+        metadata_index: IMetadataIndex,
+        cache_manager: ICacheManager | None = None,
+    ) -> None:
+        self._storage_manager = storage_manager
+        self._metadata_index = metadata_index
+        self._cache_manager = cache_manager
+        self._proxy_cache: dict[str, TProxy] = {}
+
+    @property
+    def storage_manager(self) -> IStorageManager:
+        """Return the storage manager owned by this repository."""
+        return self._storage_manager
+
+    @property
+    def metadata_index(self) -> IMetadataIndex:
+        """Return the metadata index owned by this repository."""
+        return self._metadata_index
+
+    @property
+    def cache_manager(self) -> ICacheManager | None:
+        """Return the cache manager owned by this repository."""
+        return self._cache_manager
+
+    def __enter__(self) -> Self:
+        """Enter batch write context for repository storage."""
+        self._storage_manager.__enter__()
+        self._metadata_index.__enter__()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: types.TracebackType | None,
+    ) -> None:
+        """Exit batch write context for repository storage."""
+        self._metadata_index.__exit__(exc_type, exc_val, exc_tb)
+        self._storage_manager.__exit__(exc_type, exc_val, exc_tb)
+
+    def __len__(self) -> int:
+        """Return number of entities in the repository."""
+        return self._metadata_index.count(self.TABLE)
+
+    def __iter__(self) -> Iterator[TProxy]:
+        """Iterate entity proxies in repository order."""
+        for uid in self._metadata_index.list_uids(self.TABLE):
+            yield self.get_proxy(uid)
+
+    def __getitem__(self, index: int) -> TProxy:
+        """Return entity proxy by positional index."""
+        uids = self._metadata_index.list_uids(self.TABLE)
+        return self.get_proxy(uids[index])
+
+    def get_proxy(self, uid: str) -> TProxy:
+        """Return entity proxy by uid with lazy loading."""
+        if uid in self._proxy_cache:
+            return self._proxy_cache[uid]
+
+        meta_dict = self._metadata_index.get(self.TABLE, uid)
+        if meta_dict is None:
+            raise KeyError(f"Entity with uid '{uid}' not found in {self.TABLE}")
+
+        meta = self.METADATA_CLASS.from_dict(meta_dict)
+        proxy = self.PROXY_CLASS(meta=meta, _repository=self)
+        self._proxy_cache[uid] = proxy
+        return proxy
+
+    def has_uid(self, uid: str) -> bool:
+        """Check if entity with given uid exists."""
+        return self._metadata_index.get(self.TABLE, uid) is not None
+
+    def append(self, proxy: TProxy) -> None:
+        """Add an entity proxy to repository (cache only, metadata already in index)."""
+        self._proxy_cache[proxy.meta.uid] = proxy
+
+    def extend(self, proxies: Iterable[TProxy]) -> None:
+        """Add multiple entity proxies to repository cache."""
+        for proxy in proxies:
+            self.append(proxy)
+
+    def remove_by_uid(self, uid: str) -> None:
+        """Remove an entity from repository by uid."""
+        self._proxy_cache.pop(uid, None)
+        self._metadata_index.delete(self.TABLE, uid)
+        self._storage_manager.erase_obj(uid, table=self.TABLE)
+
+    def clear(self, erase: bool = False) -> None:
+        """Clear repository membership and optionally erase storage data.
+
+        Args:
+            erase: If True, also erase underlying storage data and cache.
+                   If False, only clear membership (metadata index and proxy cache).
+        """
+        if erase:
+            if self._cache_manager is not None:
+                self._cache_manager.clear()
+
+            uids = self._metadata_index.list_uids(self.TABLE)
+            with self._storage_manager:
+                for uid in uids:
+                    self._storage_manager.erase_obj(uid, table=self.TABLE)
+
+        self._metadata_index.clear(self.TABLE)
+        self._proxy_cache.clear()
+
+    def query(
+        self,
+        where: dict[str, Any] | None = None,
+        where_in: dict[str, list[Any]] | None = None,
+        where_like: dict[str, str] | None = None,
+        order_by: list[str] | None = None,
+        limit: int | None = None,
+    ) -> list[str]:
+        """Query UIDs from metadata index."""
+        return self._metadata_index.query(
+            self.TABLE,
+            where=where,
+            where_in=where_in,
+            where_like=where_like,
+            order_by=order_by,
+            limit=limit,
+        )
+
+    def list_uids(self) -> list[str]:
+        """List all entity UIDs in the repository."""
+        return self._metadata_index.list_uids(self.TABLE)
+
+    def prefetch_proxies(
+        self,
+        uids: list[str],
+        parallel: ParallelContext | None = None,
+    ) -> None:
+        """Batch prefetch proxies into cache."""
+        uids_to_load = [uid for uid in uids if uid not in self._proxy_cache]
+        if not uids_to_load:
+            return
+
+        meta_batch = self._metadata_index.get_batch(self.TABLE, uids_to_load)
+
+        for uid, meta_dict in zip(uids_to_load, meta_batch, strict=True):
+            if meta_dict is not None:
+                meta = self.METADATA_CLASS.from_dict(meta_dict)
+                self._proxy_cache[uid] = self.PROXY_CLASS(meta=meta, _repository=self)
+
+    @classmethod
+    def define_schema(cls, metadata_index: IMetadataIndex) -> None:
+        """Define metadata schema in the index. Override in subclasses."""
+        metadata_index.define_table(
+            cls.TABLE,
+            columns=cls.METADATA_COLUMNS,
+            indexes=cls.METADATA_INDEXES,
+        )
+
+    @classmethod
+    def initialize(
+        cls,
+        storage_manager: IStorageManager,
+        metadata_index: IMetadataIndex,
+        cache_manager: ICacheManager | None = None,
+        parallel: ParallelContext | None = None,
+    ) -> Self:
+        """Initialize repository with schema definition."""
+        cls.define_schema(metadata_index)
+        return cls(
+            storage_manager=storage_manager,
+            metadata_index=metadata_index,
+            cache_manager=cache_manager,
+        )
+
+    def create_view(self) -> DataView[TMetadata, TProxy]:
+        """Create a view over all entities in the repository."""
+        return self.VIEW_CLASS(repository=self, uids=None)

diffract/core/data/utils.py

@@ -0,0 +1,33 @@
+"""Utility functions for the data module."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable, Iterable
+
+from diffract.core.constants import REGEX_PREFIX
+
+
+def build_matcher(patterns: Iterable[str]) -> Callable[[str], bool]:
+    """Build a matcher function from exact strings and regex patterns.
+
+    Args:
+        patterns: Strings to match. Prefix with REGEX_PREFIX ("re:") for regex.
+
+    Returns:
+        Function that returns True if input matches any pattern.
+    """
+    exact: set[str] = set()
+    compiled: list[re.Pattern[str]] = []
+    for pattern in patterns:
+        if pattern.startswith(REGEX_PREFIX):
+            compiled.append(re.compile(pattern.removeprefix(REGEX_PREFIX)))
+        else:
+            exact.add(pattern)
+
+    def _matches(value: str) -> bool:
+        if value in exact:
+            return True
+        return any(pattern.fullmatch(value) is not None for pattern in compiled)
+
+    return _matches