lamindb 1.1.1__py3-none-any.whl → 1.2a2__py3-none-any.whl

lamindb/core/storage/objects.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from pathlib import PurePosixPath
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, TypeAlias, TypeVar
 
 from anndata import AnnData
 from pandas import DataFrame
@@ -9,54 +9,87 @@ from pandas import DataFrame
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
 
+SpatialData = TypeVar("SpatialData")
+MuData = TypeVar("MuData")
 
-def _mudata_is_installed():
-    try:
-        import mudata  # noqa: F401c
-    except ImportError:
-        return False
-    return True
+SupportedDataTypes: TypeAlias = AnnData | DataFrame | MuData | SpatialData
 
 
-def infer_suffix(dmem, adata_format: str | None = None):
+def is_package_installed(package_name):
+    import importlib.util
+
+    spec = importlib.util.find_spec(package_name)
+    return spec is not None
+
+
+def infer_suffix(dmem: SupportedDataTypes, format: str | None = None):
     """Infer LaminDB storage file suffix from a data object."""
     if isinstance(dmem, AnnData):
-        if adata_format is not None:
-            if adata_format not in {"h5ad", "zarr", "anndata.zarr"}:
+        if format is not None:
+            if format not in {"h5ad", "zarr", "anndata.zarr"}:
                 raise ValueError(
                     "Error when specifying AnnData storage format, it should be"
-                    f" 'h5ad', 'zarr', not '{adata_format}'. Check 'format'"
+                    f" 'h5ad', 'zarr', not '{format}'. Check 'format'"
                     " or the suffix of 'key'."
                 )
-            return "." + adata_format
+            return "." + format
         return ".h5ad"
-    elif isinstance(dmem, DataFrame):
+
+    if isinstance(dmem, DataFrame):
         return ".parquet"
-    else:
-        if _mudata_is_installed():
-            from mudata import MuData
 
-            if isinstance(dmem, MuData):
-                return ".h5mu"
+    if is_package_installed("mudata"):
+        from mudata import MuData
+
+        if isinstance(dmem, MuData):
+            return ".h5mu"
+
+    if is_package_installed("spatialdata"):
+        from spatialdata import SpatialData
+
+        if isinstance(dmem, SpatialData):
+            if format is not None:
+                if format not in {"spatialdata.zarr"}:
+                    raise ValueError(
+                        "Error when specifying SpatialData storage format, it should be"
+                        f" 'zarr', 'spatialdata.zarr', not '{format}'. Check 'format'"
+                        " or the suffix of 'key'."
+                    )
+                return "." + format
+            return ".zarr"
+    else:
         raise NotImplementedError
 
 
-def write_to_disk(dmem, filepath: UPathStr):
+def write_to_disk(dmem: SupportedDataTypes, filepath: UPathStr) -> None:
+    """Writes the passed in memory data to disk to a specified path."""
     if isinstance(dmem, AnnData):
         suffix = PurePosixPath(filepath).suffix
         if suffix == ".h5ad":
             dmem.write_h5ad(filepath)
+            return
         elif suffix == ".zarr":
             dmem.write_zarr(filepath)
+            return
         else:
             raise NotImplementedError
-    elif isinstance(dmem, DataFrame):
+
+    if isinstance(dmem, DataFrame):
         dmem.to_parquet(filepath)
-    else:
-        if _mudata_is_installed():
-            from mudata import MuData
+        return
+
+    if is_package_installed("mudata"):
+        from mudata import MuData
 
-            if isinstance(dmem, MuData):
-                dmem.write(filepath)
-                return
+        if isinstance(dmem, MuData):
+            dmem.write(filepath)
+            return
+
+    if is_package_installed("spatialdata"):
+        from spatialdata import SpatialData
+
+        if isinstance(dmem, SpatialData):
+            dmem.write(filepath, overwrite=True)
+            return
+    else:
         raise NotImplementedError

lamindb/core/storage/paths.py

@@ -11,13 +11,14 @@ from lamindb_setup.core.upath import (
 )
 
 from lamindb.core._settings import settings
-from lamindb.models import Artifact, Storage
 
 if TYPE_CHECKING:
     from pathlib import Path
 
     from lamindb_setup.core.types import UPathStr
 
+    from lamindb.models.artifact import Artifact
+
 
 AUTO_KEY_PREFIX = ".lamindb/"
 
@@ -70,6 +71,8 @@ def attempt_accessing_path(
 ) -> tuple[UPath, StorageSettings]:
     # check whether the file is in the default db and whether storage
     # matches default storage
+    from lamindb.models import Storage
+
     if (
         artifact._state.db in ("default", None)
         and artifact.storage_id == settings._storage_settings.id

lamindb/curators/__init__.py

@@ -9,12 +9,23 @@
    DataFrameCurator
    AnnDataCurator
 
+CatManager:
+
+.. autosummary::
+   :toctree: .
+
+   CatManager
+   DataFrameCatManager
+   AnnDataCatManager
+   MuDataCatManager
+   TiledbsomaCatManager
+   CurateLookup
+
 """
 
 from __future__ import annotations
 
 import copy
-import random
 import re
 from importlib import resources
 from itertools import chain
@@ -40,14 +51,10 @@ if TYPE_CHECKING:
 
     from lamindb.base.types import FieldAttr
     from lamindb.models import Record
-from lamindb._feature import parse_dtype, parse_dtype_single_cat
 from lamindb.base.types import FieldAttr  # noqa
-from lamindb.core._data import add_labels
-from lamindb.core._feature_manager import parse_staged_feature_sets_from_anndata
 from lamindb.core._settings import settings
 from lamindb.models import (
     Artifact,
-    CanCurate,
     Collection,
     Feature,
     Record,
@@ -55,9 +62,11 @@ from lamindb.models import (
     Schema,
     ULabel,
 )
+from lamindb.models._feature_manager import parse_staged_feature_sets_from_anndata
+from lamindb.models.artifact import add_labels, data_is_anndata
+from lamindb.models.feature import parse_dtype, parse_dtype_single_cat
+from lamindb.models._from_values import _format_values
 
-from .._artifact import data_is_anndata
-from .._from_values import _format_values
 from ..errors import InvalidArgument, ValidationError
 
 if TYPE_CHECKING:
@@ -68,7 +77,7 @@ if TYPE_CHECKING:
     from mudata import MuData
     from spatialdata import SpatialData
 
-    from lamindb._query_set import RecordList
+    from lamindb.models.query_set import RecordList
 
 
 def strip_ansi_codes(text):
@@ -453,24 +462,31 @@ class AnnDataCurator(Curator):
             raise InvalidArgument("dataset must be AnnData-like.")
         if schema.otype != "AnnData":
             raise InvalidArgument("Schema otype must be 'AnnData'.")
-        self._obs_curator = DataFrameCurator(
-            self._dataset.obs, schema._get_component("obs")
-        )
-        self._var_curator = DataFrameCurator(
-            self._dataset.var.T, schema._get_component("var")
-        )
+        # TODO: also support slots other than obs and var
+        self._slots = {
+            slot: DataFrameCurator(
+                (
+                    self._dataset.__getattribute__(slot).T
+                    if slot == "var"
+                    else self._dataset.__getattribute__(slot)
+                ),
+                slot_schema,
+            )
+            for slot, slot_schema in schema.slots.items()
+            if slot in {"obs", "var"}
+        }
 
     @property
     @doc_args(SLOTS_DOCSTRING)
     def slots(self) -> dict[str, DataFrameCurator]:
         """{}"""  # noqa: D415
-        return {"obs": self._obs_curator, "var": self._var_curator}
+        return self._slots
 
     @doc_args(VALIDATE_DOCSTRING)
     def validate(self) -> None:
         """{}"""  # noqa: D415
-        self._obs_curator.validate()
-        self._var_curator.validate()
+        for _, curator in self._slots.items():
+            curator.validate()
 
     @doc_args(SAVE_ARTIFACT_DOCSTRING)
     def save_artifact(
@@ -483,13 +499,18 @@ class AnnDataCurator(Curator):
     ):
         """{}"""  # noqa: D415
         if not self._is_validated:
-            self.validate()  # raises ValidationError if doesn't validate
-        result = parse_dtype_single_cat(self._var_curator._schema.itype, is_itype=True)
+            self.validate()
         return save_artifact(  # type: ignore
             self._dataset,
             description=description,
-            fields=self._obs_curator._cat_manager.categoricals,
-            columns_field=result["field"],
+            fields=self.slots["obs"]._cat_manager.categoricals,
+            columns_field=(
+                parse_dtype_single_cat(self.slots["var"]._schema.itype, is_itype=True)[
+                    "field"
+                ]
+                if "var" in self._slots
+                else None
+            ),
             key=key,
             artifact=self._artifact,
             revises=revises,
@@ -519,8 +540,8 @@ class CatManager:
 
     If you find non-validated values, you have several options:
 
-    - new values found in the data can be registered using :meth:`~lamindb.core.DataFrameCatManager.add_new_from`
-    - non-validated values can be accessed using :meth:`~lamindb.core.DataFrameCatManager.non_validated` and addressed manually
+    - new values found in the data can be registered using :meth:`~lamindb.curators.DataFrameCatManager.add_new_from`
+    - non-validated values can be accessed using :meth:`~lamindb.curators.DataFrameCatManager.non_validated` and addressed manually
     """
 
     def __init__(
@@ -891,7 +912,7 @@ class AnnDataCatManager(CatManager):
     def __init__(
         self,
         data: ad.AnnData | Artifact,
-        var_index: FieldAttr,
+        var_index: FieldAttr | None = None,
         categoricals: dict[str, FieldAttr] | None = None,
         obs_columns: FieldAttr = Feature.name,
         verbosity: str = "hint",
@@ -960,15 +981,16 @@ class AnnDataCatManager(CatManager):
         validated_only: bool = True,
     ):
         """Save variable records."""
-        update_registry(
-            values=list(self._adata.var.index),
-            field=self.var_index,
-            key="var_index",
-            validated_only=validated_only,
-            organism=self._organism,
-            source=self._sources.get("var_index"),
-            exclude=self._exclude.get("var_index"),
-        )
+        if self.var_index is not None:
+            update_registry(
+                values=list(self._adata.var.index),
+                field=self.var_index,
+                key="var_index",
+                validated_only=validated_only,
+                organism=self._organism,
+                source=self._sources.get("var_index"),
+                exclude=self._exclude.get("var_index"),
+            )
 
     def add_new_from(self, key: str, **kwargs):
         """Add validated & new categories.
@@ -1004,15 +1026,19 @@ class AnnDataCatManager(CatManager):
 
         # add all validated records to the current instance
         self._save_from_var_index(validated_only=True)
-        validated_var, non_validated_var = validate_categories(
-            self._adata.var.index,
-            field=self._var_field,
-            key="var_index",
-            source=self._sources.get("var_index"),
-            hint_print=".add_new_from_var_index()",
-            exclude=self._exclude.get("var_index"),
-            organism=self._organism,  # type: ignore
-        )
+        if self.var_index is not None:
+            validated_var, non_validated_var = validate_categories(
+                self._adata.var.index,
+                field=self._var_field,
+                key="var_index",
+                source=self._sources.get("var_index"),
+                hint_print=".add_new_from_var_index()",
+                exclude=self._exclude.get("var_index"),
+                organism=self._organism,  # type: ignore
+            )
+        else:
+            validated_var = True
+            non_validated_var = []
         validated_obs = self._obs_df_curator.validate()
         self._non_validated = self._obs_df_curator._non_validated  # type: ignore
         if len(non_validated_var) > 0:
@@ -1711,7 +1737,7 @@ class TiledbsomaCatManager(CatManager):
         Returns:
             A saved artifact record.
         """
-        from lamindb.core._data import add_labels
+        from lamindb.models.artifact import add_labels
 
         if not self._is_validated:
             self.validate()
@@ -1853,6 +1879,7 @@ class SpatialDataCatManager(CatManager):
         else:
             self._sdata = self._dataset
         self._sample_metadata_key = sample_metadata_key
+        self._write_path = None
         self._var_fields = var_index
         self._verify_accessor_exists(self._var_fields.keys())
         self._categoricals = categoricals
@@ -2134,26 +2161,14 @@ class SpatialDataCatManager(CatManager):
         try:
             settings.verbosity = "warning"
 
-            if self._artifact is None:
-                # Write the SpatialData object to a random path in tmp directory
-                # The Artifact constructor will move it to the cache
-                write_path = (
-                    f"{settings.cache_dir}/{random.randint(10**7, 10**8 - 1)}.zarr"
-                )
-                self._sdata.write(write_path)
-
-                # Create the Artifact and associate Artifact metadata
-                self._artifact = Artifact(
-                    write_path,
-                    description=description,
-                    key=key,
-                    revises=revises,
-                    run=run,
-                )
-                # According to Tim it is not easy to calculate the number of observations.
-                # We would have to write custom code to iterate over labels (which might not even exist at that point)
-                self._artifact.otype = "spatialdata"
-                self._artifact.save()
+            self._artifact = Artifact.from_spatialdata(
+                self._sdata,
+                key=key,
+                description=description,
+                revises=revises,
+                run=run,
+            )
+            self._artifact.save()
 
             # Link schemas
             feature_kwargs = check_registry_organism(
@@ -2171,7 +2186,7 @@ class SpatialDataCatManager(CatManager):
                 """Add Schemas from SpatialData."""
                 if obs_fields is None:
                     obs_fields = {}
-                assert host.otype == "spatialdata"  # noqa: S101
+                assert host.otype == "SpatialData"  # noqa: S101
 
                 feature_sets = {}
 
@@ -2799,7 +2814,7 @@ class DoseHandler:
         return cls.UNIT_MAP.get(unit, unit)
 
     @classmethod
-    def validate_values(cls, values: pd.Series) -> list:
+    def validate_values(cls, values: pd.Series) -> list[str]:
         """Validate pert_dose values with strict case checking."""
         errors = []
 
@@ -2843,7 +2858,7 @@ class TimeHandler:
         return unit[0].lower()
 
     @classmethod
-    def validate_values(cls, values: pd.Series) -> list:
+    def validate_values(cls, values: pd.Series) -> list[str]:
         """Validate pert_time values."""
         errors = []
 
@@ -3197,8 +3212,8 @@ def validate_categories(
     exclude: str | list | None = None,
     hint_print: str | None = None,
     curator: CatManager | None = None,
-) -> tuple[bool, list]:
-    """Validate ontology terms in a pandas series using LaminDB registries.
+) -> tuple[bool, list[str]]:
+    """Validate ontology terms using LaminDB registries.
 
     Args:
         values: The values to validate.
@@ -3210,8 +3225,8 @@ def validate_categories(
         standardize: Whether to standardize the values.
         hint_print: The hint to print that suggests fixing non-validated values.
     """
-    from lamindb._from_values import _format_values
     from lamindb.core._settings import settings
+    from lamindb.models._from_values import _format_values
 
     model_field = f"{field.field.model.__name__}.{field.field.name}"
 
@@ -3346,7 +3361,7 @@ def validate_categories_in_df(
 def save_artifact(
     data: pd.DataFrame | ad.AnnData | MuData,
     fields: dict[str, FieldAttr] | dict[str, dict[str, FieldAttr]],
-    columns_field: FieldAttr | dict[str, FieldAttr],
+    columns_field: FieldAttr | dict[str, FieldAttr] | None = None,
     description: str | None = None,
     organism: str | None = None,
     key: str | None = None,
@@ -3372,8 +3387,7 @@ def save_artifact(
     Returns:
         The saved Artifact.
     """
-    from .._artifact import data_is_anndata, data_is_mudata
-    from ..core._data import add_labels
+    from ..models.artifact import add_labels, data_is_anndata, data_is_mudata
 
     if artifact is None:
         if data_is_anndata(data):
@@ -3395,7 +3409,7 @@ def save_artifact(
     artifact.schema = schema
     artifact.save()
 
-    if organism is not None:
+    if organism is not None and columns_field is not None:
         feature_kwargs = check_registry_organism(
             (
                 list(columns_field.values())[0].field.model
@@ -3532,8 +3546,8 @@ def update_registry(
         exclude: Values to exclude from inspect.
         kwargs: Additional keyword arguments to pass to the registry model to create new records.
     """
-    from lamindb._save import save as ln_save
     from lamindb.core._settings import settings
+    from lamindb.models.save import save as ln_save
 
     registry = field.field.model
     filter_kwargs = check_registry_organism(registry, organism)
@@ -3621,7 +3635,7 @@ def log_saved_labels(
     validated_only: bool = True,
 ) -> None:
     """Log the saved labels."""
-    from .._from_values import _format_values
+    from ..models._from_values import _format_values
 
     model_field = colors.italic(model_field)
     for k, labels in labels_saved.items():
@@ -3667,12 +3681,14 @@ def _save_organism(name: str):
     return organism
 
 
-def _ref_is_name(field: FieldAttr) -> bool | None:
+def _ref_is_name(field: FieldAttr | None) -> bool | None:
     """Check if the reference field is a name field."""
-    from .._can_curate import get_name_field
+    from ..models.can_curate import get_name_field
 
-    name_field = get_name_field(field.field.model)
-    return field.field.name == name_field
+    if field is not None:
+        name_field = get_name_field(field.field.model)
+        return field.field.name == name_field
+    return None
 
 
 # backward compat constructors ------------------
@@ -3721,7 +3737,7 @@ def from_anndata(
 @classmethod  # type: ignore
 def from_mudata(
     cls,
-    mdata: MuData,
+    mdata: MuData | UPathStr,
     var_index: dict[str, dict[str, FieldAttr]],
     categoricals: dict[str, FieldAttr] | None = None,
     verbosity: str = "hint",
@@ -3761,7 +3777,7 @@ def from_tiledbsoma(
 @classmethod  # type: ignore
 def from_spatialdata(
     cls,
-    sdata,
+    sdata: SpatialData | UPathStr,
     var_index: dict[str, FieldAttr],
     categoricals: dict[str, dict[str, FieldAttr]] | None = None,
     organism: str | None = None,

lamindb/errors.py

@@ -1,4 +1,4 @@
-"""Exceptions.
+"""Errors.
 
 .. autosummary::
    :toctree: .

lamindb/integrations/_vitessce.py

@@ -7,10 +7,10 @@ from typing import TYPE_CHECKING
 import lamindb_setup as ln_setup
 from lamin_utils import logger
 
-from lamindb._artifact import Artifact
-from lamindb._collection import Collection
-from lamindb._run import Run
-from lamindb._transform import Transform
+from lamindb.models.artifact import Artifact
+from lamindb.models.collection import Collection
+from lamindb.models.run import Run
+from lamindb.models.transform import Transform
 
 if TYPE_CHECKING:
     from vitessce import VitessceConfig