datamint 2.3.3__py3-none-any.whl → 2.9.0__py3-none-any.whl

datamint/entities/resource.py

@@ -1,20 +1,27 @@
 """Resource entity module for DataMint API."""
 
 from datetime import datetime
-from typing import TYPE_CHECKING, Optional, Any, Sequence
+from typing import TYPE_CHECKING, Optional
+from collections.abc import Sequence
 import logging
+import urllib.parse
+import urllib.request
 
 from .base_entity import BaseEntity, MISSING_FIELD
 from .cache_manager import CacheManager
 from pydantic import PrivateAttr
-from datamint.api.dto import AnnotationType
 import webbrowser
-from datamint.types import ImagingData
+import shutil
+from pathlib import Path
+from datamint.api.base_api import BaseApi
 
 if TYPE_CHECKING:
     from datamint.api.endpoints.resources_api import ResourcesApi
     from .project import Project
-    from .annotation import Annotation
+    from .annotations.annotation import Annotation
+    from datamint.types import ImagingData
+    from datamint.api.dto import AnnotationType
+
 
 logger = logging.getLogger(__name__)
 
@@ -70,47 +77,57 @@ class Resource(BaseEntity):
     location: str
     upload_channel: str
     filename: str
-    modality: str
     mimetype: str
     size: int
-    upload_mechanism: str
     customer_id: str
     status: str
     created_at: str
     created_by: str
     published: bool
     deleted: bool
-    source_filepath: str | None
-    metadata: dict
-    projects: list[dict] = MISSING_FIELD
-    published_on: str | None
-    published_by: str | None
+    upload_mechanism: str | None = None
+    # metadata: dict[str,Any] = {}
+    modality: str | None = None
+    source_filepath: str | None = None
+    # projects: list[dict[str, Any]] | None = None
+    published_on: str | None = None
+    published_by: str | None = None
     tags: list[str] | None = None
-    publish_transforms: Optional[Any] = None
+    # publish_transforms: dict[str, Any] | None = None
     deleted_at: Optional[str] = None
     deleted_by: Optional[str] = None
     instance_uid: Optional[str] = None
     series_uid: Optional[str] = None
     study_uid: Optional[str] = None
     patient_id: Optional[str] = None
-    segmentations: Optional[Any] = None  # TODO: Define proper type when spec available
-    measurements: Optional[Any] = None  # TODO: Define proper type when spec available
-    categories: Optional[Any] = None  # TODO: Define proper type when spec available
-    user_info: Optional[dict] = None
+    # segmentations: Optional[Any] = None  # TODO: Define proper type when spec available
+    # measurements: Optional[Any] = None  # TODO: Define proper type when spec available
+    # categories: Optional[Any] = None  # TODO: Define proper type when spec available
+    user_info: dict[str, str | None] = MISSING_FIELD
 
     _api: 'ResourcesApi' = PrivateAttr()
 
+    def __new__(cls, *args, **kwargs):
+        if cls is Resource and ('local_filepath' in kwargs or 'raw_data' in kwargs):
+            return super().__new__(LocalResource)
+        return super().__new__(cls)
+
     def __init__(self, **data):
         """Initialize the resource entity."""
         super().__init__(**data)
-        self._cache: CacheManager[bytes] = CacheManager[bytes]('resources')
+
+    @property
+    def _cache(self) -> CacheManager[bytes]:
+        if not hasattr(self, '__cache'):
+            self.__cache = CacheManager[bytes]('resources')
+        return self.__cache
 
     def fetch_file_data(
         self,
         auto_convert: bool = True,
         save_path: str | None = None,
         use_cache: bool = False,
-    ) -> bytes | ImagingData:
+    ) -> 'bytes | ImagingData':
         """Get the file data for this resource.
 
         This method automatically caches the file data locally. On subsequent
@@ -119,7 +136,9 @@ class Resource(BaseEntity):
         Args:
             use_cache: If True, uses cached data when available and valid
             auto_convert: If True, automatically converts to appropriate format (pydicom.Dataset, PIL Image, etc.)
-            save_path: Optional path to save the file locally
+            save_path: Optional path to save the file locally. If use_cache is also True,
+                      the file is saved to save_path and cache metadata points to that location
+                      (no duplication - only one file on disk).
 
         Returns:
             File data (format depends on auto_convert and file type)
@@ -127,31 +146,30 @@ class Resource(BaseEntity):
         # Version info for cache validation
         version_info = self._generate_version_info()
 
-        # Try to get from cache
-        img_data = None
-        if use_cache:
-            img_data = self._cache.get(self.id, _IMAGE_CACHEKEY, version_info)
-            if img_data is not None:
-                logger.debug(f"Using cached image data for resource {self.id}")
-
-        if img_data is None:
-            # Fetch from server using download_resource_file
-            logger.debug(f"Fetching image data from server for resource {self.id}")
-            img_data = self._api.download_resource_file(
+        # Download callback for the shared caching logic
+        def download_callback(path: str | None) -> bytes:
+            return self._api.download_resource_file(
                 self,
-                save_path=save_path,
+                save_path=path,
                 auto_convert=False
             )
-            # Cache the data
-            if use_cache:
-                self._cache.set(self.id, _IMAGE_CACHEKEY, img_data, version_info)
+
+        # Use shared caching logic from BaseEntity
+        img_data = self._fetch_and_cache_file_data(
+            cache_manager=self._cache,
+            data_key=_IMAGE_CACHEKEY,
+            version_info=version_info,
+            download_callback=download_callback,
+            save_path=save_path,
+            use_cache=use_cache,
+        )
 
         if auto_convert:
             try:
-                mimetype, ext = self._api._determine_mimetype(img_data, self)
-                img_data = self._api.convert_format(img_data,
-                                                    mimetype=mimetype,
-                                                    file_path=save_path)
+                mimetype, ext = BaseApi._determine_mimetype(img_data, self.mimetype)
+                img_data = BaseApi.convert_format(img_data,
+                                                  mimetype=mimetype,
+                                                  file_path=save_path)
             except Exception as e:
                 logger.error(f"Failed to auto-convert resource {self.id}: {e}")
 
@@ -170,17 +188,36 @@ class Resource(BaseEntity):
         version_info = self._generate_version_info()
         self._cache.set(self.id, _IMAGE_CACHEKEY, data, version_info)
 
+    def is_cached(self) -> bool:
+        """Check if the resource's file data is already cached locally and valid.
+
+        Returns:
+            True if valid cached data exists, False otherwise.
+        """
+        version_info = self._generate_version_info()
+        cached_data = self._cache.get(self.id, _IMAGE_CACHEKEY, version_info)
+        return cached_data is not None
+    
+    @property
+    def filepath_cached(self) -> Path | None:
+        """Get the file path of the cached resource data, if available.
+
+        Returns:
+            Path to the cached file data, or None if not cached.
+        """
+        if self._cache is None:
+            return None
+        version_info = self._generate_version_info()
+        path = self._cache.get_path(self.id, _IMAGE_CACHEKEY, version_info)
+        return path
+
     def fetch_annotations(
         self,
-        annotation_type: AnnotationType | str | None = None
+        annotation_type: 'AnnotationType | str | None' = None
     ) -> Sequence['Annotation']:
         """Get annotations associated with this resource."""
 
-        annotations = self._api.get_annotations(self)
-
-        if annotation_type:
-            annotation_type = AnnotationType(annotation_type)
-            annotations = [a for a in annotations if a.annotation_type == annotation_type]
+        annotations = self._api.get_annotations(self, annotation_type=annotation_type)
         return annotations
 
     # def get_projects(
@@ -193,7 +230,6 @@ class Resource(BaseEntity):
     #     """
     #     return self._api.get_projects(self)
 
-        
     def invalidate_cache(self) -> None:
         """Invalidate cached data for this resource.
         """
@@ -245,7 +281,7 @@ class Resource(BaseEntity):
             f"modality='{self.modality}', status='{self.status}', "
             f"published={self.published})"
         )
-    
+
     @property
     def url(self) -> str:
         """Get the URL to access this resource in the DataMint web application."""
@@ -255,3 +291,231 @@ class Resource(BaseEntity):
     def show(self) -> None:
         """Open the resource in the default web browser."""
         webbrowser.open(self.url)
+
+    @staticmethod
+    def from_local_file(file_path: str | Path):
+        """Create a LocalResource instance from a local file path.
+
+        Args:
+            file_path: Path to the local file
+        """
+        return LocalResource(local_filepath=file_path)
+
+
+class LocalResource(Resource):
+    """Represents a local resource that hasn't been uploaded to DataMint API yet."""
+
+    local_filepath: str | None = None
+    raw_data: bytes | None = None
+
+    @property
+    def filepath_cached(self) -> str | None:
+        """Get the file path of the local resource data.
+
+        Returns:
+            Path to the local file, or None if only raw data is available.
+        """
+        return self.local_filepath
+
+    def __init__(self,
+                 local_filepath: str | Path | None = None,
+                 raw_data: bytes | None = None,
+                 convert_to_bytes: bool = False,
+                 **kwargs):
+        """Initialize a local resource from a local file path, URL, or raw data.
+
+        Args:
+            local_filepath: Path to the local file or URL to an online image
+            raw_data: Raw bytes of the file data
+            convert_to_bytes: If True and local_filepath is provided, read file into raw_data
+        """
+        from medimgkit.format_detection import guess_type, DEFAULT_MIME_TYPE
+        from medimgkit.modality_detector import detect_modality
+
+        if raw_data is None and local_filepath is None:
+            raise ValueError("Either local_filepath or raw_data must be provided.")
+        if raw_data is not None and local_filepath is not None:
+            raise ValueError("Only one of local_filepath or raw_data should be provided.")
+
+        # Check if local_filepath is a URL
+        if local_filepath is not None:
+            local_filepath_str = str(local_filepath)
+            if local_filepath_str.startswith(('http://', 'https://')):
+                # Download content from URL
+                logger.debug(f"Downloading resource from URL: {local_filepath_str}")
+                try:
+                    with urllib.request.urlopen(local_filepath_str) as response:
+                        raw_data = response.read()
+                        # Try to get content-type from response headers
+                        content_type = response.headers.get('Content-Type', '').split(';')[0].strip()
+                except Exception as e:
+                    raise ValueError(f"Failed to download from URL: {local_filepath_str}") from e
+
+                # Extract filename from URL
+                parsed_url = urllib.parse.urlparse(local_filepath_str)
+                url_path = urllib.parse.unquote(parsed_url.path)
+                filename = Path(url_path).name if url_path else 'downloaded_file'
+
+                # Determine mimetype
+                mimetype, _ = guess_type(raw_data)
+                if mimetype is None and content_type:
+                    mimetype = content_type
+                if mimetype is None:
+                    mimetype = DEFAULT_MIME_TYPE
+
+                default_values = {
+                    'id': '',
+                    'resource_uri': '',
+                    'storage': '',
+                    'location': local_filepath_str,
+                    'upload_channel': '',
+                    'filename': filename,
+                    'modality': None,
+                    'mimetype': mimetype,
+                    'size': len(raw_data),
+                    'upload_mechanism': '',
+                    'customer_id': '',
+                    'status': 'local',
+                    'created_at': datetime.now().isoformat(),
+                    'created_by': '',
+                    'published': False,
+                    'deleted': False,
+                    'source_filepath': local_filepath_str,
+                }
+                new_kwargs = kwargs.copy()
+                for key, value in default_values.items():
+                    new_kwargs.setdefault(key, value)
+                super(Resource, self).__init__(
+                    local_filepath=None,
+                    raw_data=raw_data,
+                    **new_kwargs
+                )
+                return
+
+        if convert_to_bytes and local_filepath:
+            with open(local_filepath, 'rb') as f:
+                raw_data = f.read()
+                local_filepath = None
+        if raw_data is not None:
+            # import io
+            if isinstance(raw_data, str):
+                mimetype, _ = guess_type(raw_data.encode())
+            else:
+                mimetype, _ = guess_type(raw_data)
+            default_values = {
+                'id': '',
+                'resource_uri': '',
+                'storage': '',
+                'location': '',
+                'upload_channel': '',
+                'filename': 'raw_data',
+                'modality': None,
+                'mimetype': mimetype if mimetype else DEFAULT_MIME_TYPE,
+                'size': len(raw_data),
+                'upload_mechanism': '',
+                'customer_id': '',
+                'status': 'local',
+                'created_at': datetime.now().isoformat(),
+                'created_by': '',
+                'published': False,
+                'deleted': False,
+                'source_filepath': None,
+            }
+            new_kwargs = kwargs.copy()
+            for key, value in default_values.items():
+                new_kwargs.setdefault(key, value)
+            super().__init__(
+                local_filepath=None,
+                raw_data=raw_data,
+                **new_kwargs
+            )
+        elif local_filepath is not None:
+            file_path = Path(local_filepath)
+            if not file_path.exists():
+                raise FileNotFoundError(f"File not found: {file_path}")
+
+            mimetype, _ = guess_type(file_path)
+            if mimetype is None or mimetype == DEFAULT_MIME_TYPE:
+                logger.warning(f"Could not determine mimetype for file: {file_path}")
+            size = file_path.stat().st_size
+            created_at = datetime.fromtimestamp(file_path.stat().st_ctime).isoformat()
+
+            super().__init__(
+                id="",
+                resource_uri="",
+                storage="",
+                location=str(file_path),
+                upload_channel="",
+                filename=file_path.name,
+                modality=detect_modality(file_path),
+                mimetype=mimetype,
+                size=size,
+                upload_mechanism="",
+                customer_id="",
+                status="local",
+                created_at=created_at,
+                created_by="",
+                published=False,
+                deleted=False,
+                source_filepath=str(file_path),
+                local_filepath=str(file_path),
+                raw_data=None,
+            )
+
+    def fetch_file_data(
+        self, *args,
+        auto_convert: bool = True,
+        save_path: str | None = None,
+        **kwargs,
+    ) -> 'bytes | ImagingData':
+        """Get the file data for this local resource.
+
+        Args:
+            auto_convert: If True, automatically converts to appropriate format (pydicom.Dataset, PIL Image, etc.)
+            save_path: Optional path to save the file locally
+        Returns:
+            File data (format depends on auto_convert and file type)
+        """
+        if self.raw_data is not None:
+            img_data = self.raw_data
+            local_filepath = None
+        else:
+            local_filepath = str(self.local_filepath)
+            with open(local_filepath, 'rb') as f:
+                img_data = f.read()
+
+        if save_path:
+            with open(save_path, 'wb') as f:
+                f.write(img_data)
+
+        if auto_convert:
+            try:
+                mimetype, ext = BaseApi._determine_mimetype(img_data, self.mimetype)
+                img_data = BaseApi.convert_format(img_data,
+                                                  mimetype=mimetype,
+                                                  file_path=local_filepath)
+            except Exception as e:
+                logger.error(f"Failed to auto-convert local resource: {e}")
+                logger.error(e, exc_info=True)
+
+        return img_data
+
+    def __str__(self) -> str:
+        """String representation of the local resource.
+
+        Returns:
+            Human-readable string describing the local resource
+        """
+        return f"LocalResource(filepath='{self.local_filepath}', size={self.size_mb}MB)"
+
+    def __repr__(self) -> str:
+        """Detailed string representation of the local resource.
+
+        Returns:
+            Detailed string representation for debugging
+        """
+        return (
+            f"LocalResource(filepath='{self.local_filepath}', "
+            f"filename='{self.filename}', modality='{self.modality}', "
+            f"size={self.size_mb}MB)"
+        )

datamint/lightning/__init__.py

@@ -0,0 +1 @@
+from .datamintdatamodule import DatamintDataModule

datamint/lightning/datamintdatamodule.py

@@ -0,0 +1,103 @@
+from torch.utils.data import DataLoader
+from datamint import Dataset
+import lightning as L
+from typing import Any
+from copy import copy
+import numpy as np
+
+
+class DatamintDataModule(L.LightningDataModule):
+    """
+    LightningDataModule for Datamint datasets with train/val split.
+    TODO: Add support for test and predict dataloaders.
+    """
+
+    def __init__(
+        self,
+        project_name: str = "./",
+        batch_size: int = 32,
+        image_transform=None,
+        mask_transform=None,
+        alb_transform=None,
+        alb_train_transform=None,
+        alb_val_transform=None,
+        train_split: float = 0.9,
+        val_split: float = 0.1,
+        seed: int = 42,
+        num_workers: int = 4,
+        **dataset_kwargs: Any,
+    ):
+        super().__init__()
+        self.project_name = project_name
+        self.batch_size = batch_size
+        self.image_transform = image_transform
+        self.mask_transform = mask_transform
+
+        if alb_transform is not None and (alb_train_transform is not None or alb_val_transform is not None):
+            raise ValueError("You cannot specify both `alb_transform` and `alb_train_transform`/`alb_val_transform`.")
+
+        # Handle backward compatibility for alb_transform
+        if alb_transform is not None:
+            self.alb_train_transform = alb_transform
+            self.alb_val_transform = alb_transform
+        else:
+            self.alb_train_transform = alb_train_transform
+            self.alb_val_transform = alb_val_transform
+
+        self.train_split = train_split
+        self.val_split = val_split
+        self.seed = seed
+        self.dataset_kwargs = dataset_kwargs
+        self.num_workers = num_workers
+
+        self.dataset = None
+
+    def prepare_data(self) -> None:
+        """Download or update data if needed."""
+        Dataset(
+            project_name=self.project_name,
+            auto_update=True,
+        )
+
+    def setup(self, stage: str = None) -> None:
+        """Set up datasets and perform train/val split."""
+        if self.dataset is None:
+            # Create base dataset for getting indices
+            self.dataset = Dataset(
+                return_as_semantic_segmentation=True,
+                semantic_seg_merge_strategy="union",
+                return_frame_by_frame=True,
+                include_unannotated=False,
+                project_name=self.project_name,
+                image_transform=self.image_transform,
+                mask_transform=self.mask_transform,
+                alb_transform=None,  # No transform for base dataset
+                auto_update=False,
+                **self.dataset_kwargs,
+            )
+
+            indices = list(copy(self.dataset.subset_indices))
+            rs = np.random.RandomState(self.seed)
+            rs.shuffle(indices)
+            train_end = int(self.train_split * len(indices))
+            train_idx = indices[:train_end]
+            val_idx = indices[train_end:]
+
+            self.train_dataset = copy(self.dataset).subset(train_idx)
+            self.train_dataset.alb_transform = self.alb_train_transform
+            self.val_dataset = copy(self.dataset).subset(val_idx)
+            self.val_dataset.alb_transform = self.alb_val_transform
+
+    def train_dataloader(self) -> DataLoader:
+        return self.train_dataset.get_dataloader(batch_size=self.batch_size, num_workers=self.num_workers, shuffle=True)
+
+    def val_dataloader(self) -> DataLoader:
+        return self.val_dataset.get_dataloader(batch_size=self.batch_size, num_workers=self.num_workers, shuffle=False)
+
+    def test_dataloader(self):
+        # Use the same dataloader as validation for testing, because we have so few samples
+        return self.val_dataset.get_dataloader(batch_size=self.batch_size, num_workers=self.num_workers, shuffle=False)
+
+    def predict_dataloader(self):
+        # Use the same dataloader as validation for testing, because we have so few samples
+        return self.val_dataset.get_dataloader(batch_size=self.batch_size, num_workers=self.num_workers, shuffle=False)

datamint/mlflow/__init__.py

@@ -0,0 +1,65 @@
+# Monkey patch mlflow.tracking._tracking_service.utils.get_tracking_uri
+from .tracking.fluent import set_project
+import mlflow.tracking._tracking_service.utils as mlflow_utils
+from functools import wraps
+import logging
+from .env_utils import setup_mlflow_environment, ensure_mlflow_configured
+from typing import TYPE_CHECKING
+
+_LOGGER = logging.getLogger(__name__)
+
+# Store reference to original function
+_original_get_tracking_uri = mlflow_utils.get_tracking_uri
+_SETUP_CALLED_SUCCESSFULLY = False
+
+if mlflow_utils.is_tracking_uri_set():
+    _LOGGER.warning("MLflow tracking URI is already set before patching get_tracking_uri.")
+
+@wraps(_original_get_tracking_uri)
+def _patched_get_tracking_uri(*args, **kwargs):
+    """Patched version of get_tracking_uri that ensures MLflow environment is set up first.
+
+    This wrapper ensures that setup_mlflow_environment is called before any tracking
+    URI operations, guaranteeing proper MLflow configuration.
+
+    Args:
+        *args: Arguments passed to the original get_tracking_uri function.
+        **kwargs: Keyword arguments passed to the original get_tracking_uri function.
+
+    Returns:
+        The result of the original get_tracking_uri function.
+    """
+    global _SETUP_CALLED_SUCCESSFULLY
+    if _SETUP_CALLED_SUCCESSFULLY:
+        return _original_get_tracking_uri(*args, **kwargs)
+    if mlflow_utils.is_tracking_uri_set():
+        _LOGGER.warning("MLflow tracking URI is already set before patching get_tracking_uri.")
+    try:
+        _SETUP_CALLED_SUCCESSFULLY = setup_mlflow_environment(set_mlflow=True)
+    except Exception as e:
+        _SETUP_CALLED_SUCCESSFULLY = False
+        _LOGGER.error("Failed to set up MLflow environment: %s", e)
+    ret = _original_get_tracking_uri(*args, **kwargs)
+    return ret
+
+
+setup_mlflow_environment(set_mlflow=False)
+# Replace the original function with our patched version
+mlflow_utils.get_tracking_uri = _patched_get_tracking_uri
+
+
+if TYPE_CHECKING:
+    from .flavors.model import DatamintModel
+else:
+    import lazy_loader as lazy
+
+    __getattr__, __dir__, __all__ = lazy.attach(
+        __name__,
+        submodules=['flavors.model', 'flavors.datamint_flavor'],
+        submod_attrs={
+            "flavors.model": ["DatamintModel"],
+            "flavors.datamint_flavor": ["log_model", "load_model"],
+        },
+    )
+
+__all__ = ['set_project', 'setup_mlflow_environment', 'ensure_mlflow_configured', 'DatamintModel']

datamint/mlflow/artifact/__init__.py

@@ -0,0 +1 @@
+from .datamint_artifacts_repo import DatamintArtifactsRepository

datamint/mlflow/artifact/datamint_artifacts_repo.py

@@ -0,0 +1,8 @@
+from mlflow.store.artifact.mlflow_artifacts_repo import MlflowArtifactsRepository
+
+
+class DatamintArtifactsRepository(MlflowArtifactsRepository):
+    @classmethod
+    def resolve_uri(cls, artifact_uri, tracking_uri):
+        tracking_uri = tracking_uri.split('datamint://', maxsplit=1)[-1]
+        return super().resolve_uri(artifact_uri, tracking_uri)