datamint 2.3.2__py3-none-any.whl → 2.3.3__py3-none-any.whl

datamint/apihandler/dto/annotation_dto.py

@@ -17,7 +17,11 @@ Classes:
 import json
 from typing import Any, TypeAlias, Literal
 import logging
-from enum import Enum
+import sys
+if sys.version_info >= (3, 11):
+    from enum import StrEnum
+else:
+    from backports.strenum import StrEnum
 from medimgkit.dicom_utils import pixel_to_patient
 import pydicom
 import numpy as np
@@ -31,7 +35,7 @@ CoordinateSystem: TypeAlias = Literal['pixel', 'patient']
 """
 
 
-class AnnotationType(Enum):
+class AnnotationType(StrEnum):
     SEGMENTATION = 'segmentation'
     AREA = 'area'
     DISTANCE = 'distance'

datamint/configs.py

@@ -18,6 +18,12 @@ _LOGGER = logging.getLogger(__name__)
 
 DIRS = PlatformDirs(appname='datamintapi')
 CONFIG_FILE = os.path.join(DIRS.user_config_dir, 'datamintapi.yaml')
+try:
+    DATAMINT_DATA_DIR = os.path.join(os.path.expanduser("~"), '.datamint')
+except Exception as e:
+    _LOGGER.error(f"Could not determine home directory: {e}")
+    DATAMINT_DATA_DIR = None
+    
 
 
 def get_env_var_name(key: str) -> str:

datamint/dataset/base_dataset.py

@@ -19,6 +19,7 @@ from pathlib import Path
 from datamint.entities import Annotation, DatasetInfo
 import cv2
 from datamint.entities import Resource
+import datamint.configs
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -54,7 +55,7 @@ class DatamintBaseDataset:
         exclude_frame_label_names: List of frame label names to exclude. If None, no frame labels will be excluded.
     """
 
-    DATAMINT_DEFAULT_DIR = ".datamint"
+    
     DATAMINT_DATASETS_DIR = "datasets"
 
     def __init__(
@@ -183,8 +184,7 @@ class DatamintBaseDataset:
         """Setup root and dataset directories."""
         if root is None:
             root = os.path.join(
-                os.path.expanduser("~"),
-                self.DATAMINT_DEFAULT_DIR,
+                datamint.configs.DATAMINT_DATA_DIR,
                 self.DATAMINT_DATASETS_DIR
             )
             os.makedirs(root, exist_ok=True)

datamint/entities/__init__.py

@@ -7,14 +7,16 @@ from .project import Project
 from .resource import Resource
 from .user import User  # new export
 from .datasetinfo import DatasetInfo
+from .cache_manager import CacheManager
 
 __all__ = [
     'Annotation',
     'BaseEntity',
+    'CacheManager',
     'Channel',
     'ChannelResourceData',
+    'DatasetInfo',
     'Project',
     'Resource',
-    "User",
-    'DatasetInfo',
+    'User',
 ]

datamint/entities/annotation.py

@@ -5,11 +5,20 @@ This module defines the Annotation model used to represent annotation
 records returned by the DataMint API.
 """
 
-from typing import Any
+from typing import TYPE_CHECKING, Any
 import logging
+import os
+
 from .base_entity import BaseEntity, MISSING_FIELD
-from pydantic import Field
+from .cache_manager import CacheManager
+from pydantic import PrivateAttr
 from datetime import datetime
+from datamint.api.dto import AnnotationType
+from datamint.types import ImagingData
+
+if TYPE_CHECKING:
+    from datamint.api.endpoints.annotations_api import AnnotationsApi
+    from .resource import Resource
 
 logger = logging.getLogger(__name__)
 
@@ -21,6 +30,8 @@ _FIELD_MAPPING = {
     'index': 'frame_index',
 }
 
+_ANNOTATION_CACHE_KEY = "annotation_data"
+
 
 class Annotation(BaseEntity):
     """Pydantic Model representing a DataMint annotation.
@@ -60,7 +71,7 @@ class Annotation(BaseEntity):
     identifier: str
     scope: str
     frame_index: int | None
-    annotation_type: str
+    annotation_type: AnnotationType
     text_value: str | None
     numeric_value: float | int | None
     units: str | None
@@ -83,7 +94,66 @@ class Annotation(BaseEntity):
     annotation_worklist_name: str | None
     user_info: dict | None
     values: list | None = MISSING_FIELD
-    file: str | None = None  # Add file field for segmentations
+    file: str | None = None
+
+    _api: 'AnnotationsApi' = PrivateAttr()
+
+    def __init__(self, **data):
+        """Initialize the annotation entity."""
+        super().__init__(**data)
+        self._cache: CacheManager = CacheManager('annotations')
+        self._resource: 'Resource | None' = None
+
+    @property
+    def resource(self) -> 'Resource':
+        """Lazily load and cache the associated Resource entity."""
+        if self._resource is None:
+            self._resource = self._api._get_resource(self)
+        return self._resource
+
+    def fetch_file_data(
+        self,
+        save_path: os.PathLike | str | None = None,
+        auto_convert: bool = True,
+        use_cache: bool = False,
+    ) -> bytes | ImagingData:
+        # Version info for cache validation
+        version_info = self._generate_version_info()
+
+        # Try to get from cache
+        img_data = None
+        if use_cache:
+            img_data = self._cache.get(self.id, _ANNOTATION_CACHE_KEY, version_info)
+
+        if img_data is None:
+            # Fetch from server using download_resource_file
+            logger.debug(f"Fetching image data from server for resource {self.id}")
+            img_data = self._api.download_file(
+                self,
+                fpath_out=save_path
+            )
+            # Cache the data
+            if use_cache:
+                self._cache.set(self.id, _ANNOTATION_CACHE_KEY, img_data, version_info)
+
+        if auto_convert:
+            return self._api.convert_format(img_data)
+
+        return img_data
+
+    def _generate_version_info(self) -> dict:
+        """Helper to generate version info for caching."""
+        return {
+            'created_at': self.created_at,
+            'deleted_at': self.deleted_at,
+            'associated_file': self.associated_file,
+        }
+
+    def invalidate_cache(self) -> None:
+        """Invalidate all cached data for this annotation."""
+        self._cache.invalidate(self.id)
+        self._resource = None
+        logger.debug(f"Invalidated cache for annotation {self.id}")
 
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> 'Annotation':

datamint/entities/base_entity.py

@@ -1,7 +1,11 @@
 import logging
 import sys
-from typing import Any
-from pydantic import ConfigDict, BaseModel
+from typing import Any, TYPE_CHECKING
+from pydantic import ConfigDict, BaseModel, PrivateAttr
+
+if TYPE_CHECKING:
+    from datamint.api.client import Api
+    from datamint.api.entity_base_api import EntityBaseApi
 
 if sys.version_info >= (3, 11):
     from typing import Self
@@ -22,9 +26,14 @@ class BaseEntity(BaseModel):
     This class provides common functionality for all entities, such as
     serialization and deserialization from dictionaries, as well as
     handling unknown fields gracefully.
+
+    The API client is automatically injected by the Api class when entities
+    are created through API endpoints.
     """
 
-    model_config = ConfigDict(extra='allow')  # Allow extra fields not defined in the model
+    model_config = ConfigDict(extra='allow', arbitrary_types_allowed=True)  # Allow extra fields and arbitrary types
+
+    _api: 'EntityBaseApi[Self] | EntityBaseApi' = PrivateAttr()
 
     def asdict(self) -> dict[str, Any]:
         """Convert the entity to a dictionary, including unknown fields."""
@@ -38,14 +47,46 @@ class BaseEntity(BaseModel):
         """Handle unknown fields by logging a warning once per class/field combination in debug mode."""
         if self.__pydantic_extra__ and _LOGGER.isEnabledFor(logging.DEBUG):
             class_name = self.__class__.__name__
-            
+
             have_to_log = False
             for key in self.__pydantic_extra__.keys():
                 warning_key = (class_name, key)
-                
+
                 if warning_key not in _LOGGED_WARNINGS:
                     _LOGGED_WARNINGS.add(warning_key)
                     have_to_log = True
-            
+
             if have_to_log:
                 _LOGGER.warning(f"Unknown fields {list(self.__pydantic_extra__.keys())} found in {class_name}")
+
+    @staticmethod
+    def is_attr_missing(value: Any) -> bool:
+        """Check if a value is the MISSING_FIELD sentinel."""
+        return value == MISSING_FIELD
+
+    def _refresh(self) -> Self:
+        """Refresh the entity data from the server.
+
+        This method fetches the latest data from the server and updates
+        the current instance with any missing or updated fields.
+
+        Returns:
+            The updated Entity instance (self)
+        """
+        updated_ent = self._api.get_by_id(self._api._entid(self))
+
+        # Update all fields from the fresh data
+        for field_name, field_value in updated_ent.model_dump().items():
+            if field_value != MISSING_FIELD:
+                setattr(self, field_name, field_value)
+
+        return self
+
+    def _ensure_attr(self, attr_name: str) -> None:
+        """Ensure that a given attribute is not MISSING_FIELD, refreshing if necessary.
+
+        Args:
+            attr_name: Name of the attribute to check and ensure
+        """
+        if self.is_attr_missing(getattr(self, attr_name)):
+            self._refresh()

datamint/entities/cache_manager.py

@@ -0,0 +1,302 @@
+"""Cache manager for storing and retrieving entity-related data locally.
+
+This module provides caching functionality for resource data (images, segmentations, etc.)
+with automatic validation against server versions to ensure data freshness.
+"""
+
+import hashlib
+import json
+import logging
+import pickle
+from datetime import datetime
+from pathlib import Path
+from typing import Any, TypeVar, Generic
+from pydantic import BaseModel
+# import appdirs
+import datamint.configs
+
+_LOGGER = logging.getLogger(__name__)
+
+T = TypeVar('T')
+
+
+class CacheManager(Generic[T]):
+    """Manages local caching of entity data with versioning support.
+
+    This class handles storing and retrieving cached data with automatic
+    validation against server versions to ensure data consistency.
+
+    The cache uses a directory structure:
+    - cache_root/
+      - resources/
+        - {resource_id}/
+          - image_data.pkl
+          - metadata.json
+      - annotations/
+        - {annotation_id}/
+          - segmentation_data.pkl
+          - metadata.json
+
+    Attributes:
+        cache_root: Root directory for cache storage
+        entity_type: Type of entity being cached (e.g., 'resources', 'annotations')
+    """
+
+    class ItemMetadata(BaseModel):
+        cached_at: datetime
+        data_path: str
+        data_type: str
+        mimetype: str
+        version_hash: str | None = None
+        version_info: dict | None = None
+        entity_id: str | None = None
+
+    def __init__(self, entity_type: str, cache_root: Path | str | None = None):
+        """Initialize the cache manager.
+
+        Args:
+            entity_type: Type of entity (e.g., 'resources', 'annotations')
+            cache_root: Root directory for cache. If None, uses system cache directory.
+        """
+        self.entity_type = entity_type
+
+        if cache_root is None:
+            # Use platform-specific cache directory
+            # app_cache_dir = appdirs.user_cache_dir('datamint', 'sonance')
+            # cache_root = Path(app_cache_dir) / 'entity_cache'
+            cache_root = Path(datamint.configs.DATAMINT_DATA_DIR)
+        else:
+            cache_root = Path(cache_root)
+
+        self.cache_root = cache_root / entity_type
+
+    def _get_entity_cache_dir(self, entity_id: str) -> Path:
+        """Get the cache directory for a specific entity.
+
+        Args:
+            entity_id: Unique identifier for the entity
+
+        Returns:
+            Path to the entity's cache directory
+        """
+        entity_dir = self.cache_root / entity_id
+        entity_dir = entity_dir.resolve().absolute()
+        entity_dir.mkdir(parents=True, exist_ok=True)
+        return entity_dir
+
+    def _get_metadata_path(self, entity_id: str) -> Path:
+        """Get the path to the metadata file for an entity.
+
+        Args:
+            entity_id: Unique identifier for the entity
+
+        Returns:
+            Path to the metadata file
+        """
+        return self._get_entity_cache_dir(entity_id) / 'metadata.json'
+
+    def _get_data_path(self, entity_id: str, data_key: str) -> Path:
+        """Get the path to a data file for an entity.
+
+        Args:
+            entity_id: Unique identifier for the entity
+            data_key: Key identifying the type of data (e.g., 'image_data', 'segmentation')
+
+        Returns:
+            Path to the data file
+        """
+        return self._get_entity_cache_dir(entity_id) / f"{data_key}.pkl"
+
+    def _compute_version_hash(self, version_info: dict[str, Any]) -> str:
+        """Compute a hash from version information.
+
+        Args:
+            version_info: Dictionary containing version information (e.g., updated_at, size)
+
+        Returns:
+            Hash string representing the version
+        """
+        # Sort keys for consistent hashing
+        sorted_info = json.dumps(version_info, sort_keys=True)
+        return hashlib.sha256(sorted_info.encode()).hexdigest()
+
+    def get(
+        self,
+        entity_id: str,
+        data_key: str,
+        version_info: dict[str, Any] | None = None
+    ) -> T | None:
+        """Retrieve cached data for an entity.
+
+        Args:
+            entity_id: Unique identifier for the entity
+            data_key: Key identifying the type of data
+            version_info: Optional version information from server to validate cache
+
+        Returns:
+            Cached data if valid, None if cache miss or invalid
+        """
+        metadata_path = self._get_metadata_path(entity_id)
+        data_path = self._get_data_path(entity_id, data_key)
+
+        # Check if cache exists
+        if not metadata_path.exists() or not data_path.exists():
+            _LOGGER.debug(f"Cache miss for {entity_id}/{data_key}")
+            return None
+
+        try:
+            # Load or create metadata
+            with open(metadata_path, 'r') as f:
+                jsondata = f.read()
+            cached_metadata = CacheManager.ItemMetadata.model_validate_json(jsondata)
+
+            # Validate version if provided
+            if version_info is not None:
+                server_version = self._compute_version_hash(version_info)
+
+                if server_version != cached_metadata.version_hash:
+                    _LOGGER.debug(
+                        f"Cache version mismatch for {entity_id}/{data_key}. "
+                        f"Server: {server_version}, Cached: {cached_metadata.version_hash}"
+                    )
+                    return None
+
+            data = self._load_data(cached_metadata)
+
+            _LOGGER.debug(f"Cache hit for {entity_id}/{data_key}")
+            return data
+
+        except Exception as e:
+            _LOGGER.warning(f"Error reading cache for {entity_id}/{data_key}: {e}")
+            return None
+
+    def set(
+        self,
+        entity_id: str,
+        data_key: str,
+        data: T,
+        version_info: dict[str, Any] | None = None
+    ) -> None:
+        """Store data in cache for an entity.
+
+        Args:
+            entity_id: Unique identifier for the entity
+            data_key: Key identifying the type of data
+            data: Data to cache
+            version_info: Optional version information from server
+        """
+        metadata_path = self._get_metadata_path(entity_id)
+        data_path = self._get_data_path(entity_id, data_key)
+
+        try:
+            mimetype = self._save_data(data_path, data)
+
+            metadata = CacheManager.ItemMetadata(
+                cached_at=datetime.now(),
+                data_path=str(data_path.absolute()),
+                data_type=type(data).__name__,
+                mimetype=mimetype,
+                entity_id=entity_id
+            )
+
+            # Update metadata for this data key
+
+            if version_info is not None:
+                metadata.version_hash = self._compute_version_hash(version_info)
+                # Store version_info as JSON string to ensure metadata is JSON-serializable
+                metadata.version_info = version_info
+
+            # Save metadata
+            with open(metadata_path, 'w') as f:
+                f.write(metadata.model_dump_json(indent=2))
+
+            _LOGGER.debug(f"Cached data for {entity_id}/{data_key}")
+
+        except Exception as e:
+            _LOGGER.warning(f"Error writing cache for {entity_id}/{data_key}: {e}")
+
+    def _load_data(self,
+                   metadata: 'CacheManager.ItemMetadata') -> T:
+        path = metadata.data_path
+        if metadata.mimetype == 'application/octet-stream':
+            with open(path, 'rb') as f:
+                return f.read()
+        else:
+            with open(path, 'rb') as f:
+                return pickle.load(f)
+
+
+    def _save_data(self, path: Path, data: T) -> str:
+        """
+        Save data and returns the mimetype
+        """
+        if isinstance(data, bytes):
+            with open(path, 'wb') as f:
+                f.write(data)
+            return 'application/octet-stream'
+        else:
+            with open(path, 'wb') as f:
+                pickle.dump(data, f)
+            return 'application/x-python-serialize'
+
+    def invalidate(self, entity_id: str, data_key: str | None = None) -> None:
+        """Invalidate cached data for an entity.
+
+        Args:
+            entity_id: Unique identifier for the entity
+            data_key: Optional key for specific data. If None, invalidates all data for entity.
+        """
+        if data_key is None:
+            # Invalidate entire entity cache
+            entity_dir = self._get_entity_cache_dir(entity_id)
+            if entity_dir.exists():
+                import shutil
+                shutil.rmtree(entity_dir)
+                _LOGGER.debug(f"Invalidated all cache for {entity_id}")
+        else:
+            # Invalidate specific data
+            data_path = self._get_data_path(entity_id, data_key)
+            if data_path.exists():
+                data_path.unlink()
+                _LOGGER.debug(f"Invalidated cache for {entity_id}/{data_key}")
+
+            # Update metadata
+            metadata_path = self._get_metadata_path(entity_id)
+            if metadata_path.exists():
+                with open(metadata_path, 'r') as f:
+                    metadata = json.load(f)
+
+                if data_key in metadata:
+                    del metadata[data_key]
+
+                    with open(metadata_path, 'w') as f:
+                        json.dump(metadata, f, indent=2)
+
+    def clear_all(self) -> None:
+        """Clear all cached data for this entity type."""
+        if self.cache_root.exists():
+            import shutil
+            shutil.rmtree(self.cache_root)
+            self.cache_root.mkdir(parents=True, exist_ok=True)
+            _LOGGER.info(f"Cleared all cache for {self.entity_type}")
+
+    def get_cache_info(self, entity_id: str) -> dict[str, Any]:
+        """Get information about cached data for an entity.
+
+        Args:
+            entity_id: Unique identifier for the entity
+
+        Returns:
+            Dictionary containing cache information
+        """
+        metadata_path = self._get_metadata_path(entity_id)
+
+        if not metadata_path.exists():
+            return {}
+
+        try:
+            with open(metadata_path, 'r') as f:
+                return json.load(f)
+        except Exception as e:
+            _LOGGER.warning(f"Error reading cache info for {entity_id}: {e}")
+            return {}

datamint/entities/datasetinfo.py

@@ -1,14 +1,24 @@
-"""Project entity module for DataMint API."""
+"""Dataset entity module for DataMint API."""
 
 from datetime import datetime
 import logging
+from typing import TYPE_CHECKING, Sequence
+
 from .base_entity import BaseEntity, MISSING_FIELD
 
+if TYPE_CHECKING:
+    from datamint.api.client import Api
+    from .resource import Resource
+    from .project import Project
+
 logger = logging.getLogger(__name__)
 
 
 class DatasetInfo(BaseEntity):
     """Pydantic Model representing a DataMint dataset.
+    
+    This class provides access to dataset information and related entities
+    like resources and projects.
     """
 
     id: str
@@ -20,3 +30,100 @@ class DatasetInfo(BaseEntity):
     updated_at: str | None
     total_resource: int
     resource_ids: list[str]
+
+    def __init__(self, **data):
+        """Initialize the dataset info entity."""
+        super().__init__(**data)
+        self._manager: EntityManager['DatasetInfo'] = EntityManager(self)
+        
+        # Cache for lazy-loaded data
+        self._resources_cache: Sequence['Resource'] | None = None
+        self._projects_cache: Sequence['Project'] | None = None
+    
+    def _inject_api(self, api: 'Api') -> None:
+        """Inject API client into this dataset (called automatically by Api class)."""
+        self._manager.set_api(api)
+
+    def get_resources(
+        self,
+        refresh: bool = False,
+        limit: int | None = None
+    ) -> Sequence['Resource']:
+        """Get all resources in this dataset.
+        
+        Results are cached after the first call unless refresh=True.
+        
+        Args:
+            api: Optional API client. Uses the one from set_api() if not provided.
+            refresh: If True, bypass cache and fetch fresh data
+            
+        Returns:
+            List of Resource instances in this dataset
+            
+        Raises:
+            RuntimeError: If no API client is available
+            
+        Example:
+            >>> dataset = api._datasetsinfo.get_by_id("dataset-id")
+            >>> dataset.set_api(api)
+            >>> resources = dataset.get_resources()
+        """
+        if refresh or self._resources_cache is None:
+            api_client = self._manager._ensure_api(api)
+            
+            # Fetch resources by their IDs
+            resources = []
+            for resource_id in self.resource_ids:
+                try:
+                    resource = api_client.resources.get_by_id(resource_id)
+                    resource.set_api(api_client)
+                    resources.append(resource)
+                except Exception as e:
+                    logger.warning(f"Failed to fetch resource {resource_id}: {e}")
+            
+            self._resources_cache = resources
+        
+        return self._resources_cache
+
+    def get_projects(
+        self,
+        api: 'Api | None' = None,
+        refresh: bool = False
+    ) -> Sequence['Project']:
+        """Get all projects associated with this dataset.
+        
+        Results are cached after the first call unless refresh=True.
+        
+        Args:
+            refresh: If True, bypass cache and fetch fresh data
+            
+        Returns:
+            List of Project instances
+            
+        Raises:
+            RuntimeError: If no API client is available
+            
+        Example:
+            >>> dataset = api.datasetsinfo.get_by_id("dataset-id")
+            >>> projects = dataset.get_projects()
+        """
+        if refresh or self._projects_cache is None:
+            api_client = self._manager.api
+            
+            # Get all projects and filter by dataset_id
+            all_projects = api_client.projects.get_all()
+            projects = [p for p in all_projects if p.dataset_id == self.id]
+            
+            self._projects_cache = projects
+        
+        return self._projects_cache
+    
+    def invalidate_cache(self) -> None:
+        """Invalidate all cached relationship data.
+        
+        This forces fresh data fetches on the next access.
+        """
+        self._resources_cache = None
+        self._projects_cache = None
+        logger.debug(f"Invalidated cache for dataset {self.id}")
+