lamindb 1.5.2__py3-none-any.whl → 1.6.0__py3-none-any.whl

lamindb/__init__.py

@@ -36,11 +36,18 @@ Validate and annotate artifacts.
 .. autosummary::
    :toctree: .
 
-   ULabel
    Feature
-   Param
+   ULabel
    Schema
 
+Manage flexible records to track, e.g., samples or donors.
+
+.. autosummary::
+   :toctree: .
+
+   Record
+   Sheet
+
 Manage projects.
 
 .. autosummary::
@@ -50,6 +57,7 @@ Manage projects.
    Collection
    Project
    Space
+   Branch
    Reference
    Person
 
@@ -82,19 +90,26 @@ Low-level functionality.
    :toctree: .
 
    examples
-   curators
-   integrations
    errors
    setup
    base
    core
    models
 
+Backwards compatibility.
+
+.. autosummary::
+   :toctree: .
+
+   Param
+   FeatureSet
+   Curator
+
 """
 
 # ruff: noqa: I001
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "1.5.2"
+__version__ = "1.6.0"
 
 import warnings
 
@@ -125,10 +140,10 @@ if _check_instance_setup(from_module="lamindb"):
     from .curators._legacy import CatManager as Curator
     from .models import (
         Artifact,
+        Branch,
         Collection,
         Feature,
         FeatureSet,  # backward compat
-        Param,
         Person,
         Project,
         Reference,
@@ -139,6 +154,8 @@ if _check_instance_setup(from_module="lamindb"):
         ULabel,
         User,
         Space,
+        Record,
+        Sheet,
     )
     from .models.save import save
     from . import core
@@ -151,3 +168,5 @@ if _check_instance_setup(from_module="lamindb"):
     settings.__doc__ = """Global live settings (:class:`~lamindb.core.Settings`)."""
     context.__doc__ = """Global run context (:class:`~lamindb.core.Context`)."""
     from django.db.models import Q
+
+    Param = Feature  # backward compat

lamindb/_finish.py

@@ -99,7 +99,7 @@ def save_run_logs(run: Run, save_run: bool = False) -> None:
         artifact = Artifact(  # type: ignore
             logs_path,
             description=f"log streams of run {run.uid}",
-            _branch_code=0,
+            kind="__lamindb_run__",
             run=False,
         )
         artifact.save(upload=True, print_progress=False)
@@ -370,14 +370,14 @@ def save_context_core(
                 logger.important("run.environment is already saved, ignoring")
                 overwrite_env = False
             if overwrite_env:
-                hash, _ = hash_file(env_path)
-                artifact = ln.Artifact.filter(hash=hash, _branch_code=0).one_or_none()
+                env_hash, _ = hash_file(env_path)
+                artifact = ln.Artifact.objects.filter(hash=env_hash).one_or_none()
                 new_env_artifact = artifact is None
                 if new_env_artifact:
                     artifact = ln.Artifact(  # type: ignore
                         env_path,
                         description="requirements.txt",
-                        _branch_code=0,
+                        kind="__lamindb_run__",
                         run=False,
                     )
                     artifact.save(upload=True, print_progress=False)
@@ -424,7 +424,7 @@ def save_context_core(
                     report_file = ln.Artifact(  # type: ignore
                         report_path,
                         description=f"Report of run {run.uid}",
-                        _branch_code=0,  # hidden file
+                        kind="__lamindb_run__",  # hidden file
                         run=False,
                     )
                     report_file.save(upload=True, print_progress=False)

lamindb/_tracked.py

@@ -112,7 +112,7 @@ def tracked(uid: str | None = None) -> Callable[[Callable[P, R]], Callable[P, R]
                 filtered_params[key] = value
 
             # Add parameters to the run
-            run.params.add_values(filtered_params)
+            run.features.add_values(filtered_params)
 
             # Set the run in context and execute function
             token = current_tracked_run.set(run)

lamindb/_view.py

@@ -9,7 +9,7 @@ from lamin_utils import colors, logger
 from lamindb_setup import settings
 from lamindb_setup._init_instance import get_schema_module_name
 
-from lamindb.models import Feature, FeatureValue, ParamValue, Record
+from lamindb.models import Feature, FeatureValue, ParamValue, SQLRecord
 
 from .models.feature import serialize_pandas_dtype
 
@@ -106,7 +106,7 @@ def view(
         limit: Display the latest `n` records
         modules: schema module to view. Default's to
             `None` and displays all registry modules.
-        registries: List of Record names. Defaults to
+        registries: List of SQLRecord names. Defaults to
             `None` and lists all registries.
 
     Examples:
@@ -142,8 +142,8 @@ def view(
             registry
             for registry in schema_module.__dict__.values()
             if inspect.isclass(registry)
-            and issubclass(registry, Record)
-            and registry is not Record
+            and issubclass(registry, SQLRecord)
+            and registry is not SQLRecord
         }
         if module_name == "core":
             all_registries.update({FeatureValue, ParamValue})

lamindb/core/_context.py

@@ -426,7 +426,7 @@ class Context:
         # need to save in all cases
         run.save()
         if params is not None:
-            run.params.add_values(params)
+            run.features.add_values(params)
             self._logging_message_track += "\n→ params: " + ", ".join(
                 f"{key}={value}" for key, value in params.items()
             )
@@ -452,8 +452,16 @@ class Context:
             r_or_python = "."
             if self._path is not None:
                 r_or_python = "." if self._path.suffix in {".py", ".ipynb"} else "$"
-            project_str = f', project="{project}"' if project is not None else ""
-            space_str = f', space="{space}"' if space is not None else ""
+            project_str = (
+                f', project="{project if isinstance(project, str) else project.name}"'
+                if project is not None
+                else ""
+            )
+            space_str = (
+                f', space="{space if isinstance(space, str) else space.name}"'
+                if space is not None
+                else ""
+            )
             params_str = (
                 ", params={...}" if params is not None else ""
             )  # do not put the values because typically parameterized by user
@@ -660,14 +668,32 @@ class Context:
                 key = self._path.name
         else:
             if self.uid is not None:
-                assert len(self.uid) == 12, (  # noqa: S101
-                    "uid must be 12 (stem) or 16 (full) characters long"
-                )
+                # the case with length 16 is covered above
+                if not len(self.uid) == 12:
+                    raise InvalidArgument(
+                        f'Please pass an auto-generated uid instead of "{self.uid}". Resolve by running: ln.track("{base62_12()}")'
+                    )
                 aux_transform = (
                     Transform.filter(uid__startswith=self.uid)
                     .order_by("-created_at")
                     .first()
                 )
+            else:
+                # deal with a hash-based match
+                # the user might have a made a copy of the notebook or script
+                # and actually wants to create a new transform
+                if aux_transform is not None and not aux_transform.key.endswith(
+                    self._path.name
+                ):
+                    prompt = f"Found transform with same hash but different key: {aux_transform.key}. Did you rename your {transform_type} to {self._path.name} (1) or intentionally made a copy (2)?"
+                    response = (
+                        "1" if os.getenv("LAMIN_TESTING") == "true" else input(prompt)
+                    )
+                    assert response in {"1", "2"}, (  # noqa: S101
+                        f"Please respond with either 1 or 2, not {response}"
+                    )
+                    if response == "2":
+                        transform_hash = None  # make a new transform
             if aux_transform is not None:
                 if aux_transform.key.endswith(self._path.name):
                     key = aux_transform.key

lamindb/core/_settings.py

@@ -45,7 +45,7 @@ class Settings:
 
     @property
     def creation(self) -> CreationSettings:
-        """Record creation settings.
+        """SQLRecord creation settings.
 
         For example, `ln.settings.creation.search_names = False` will disable
         searching for records with similar names during creation.

lamindb/core/datasets/mini_immuno.py

@@ -104,6 +104,11 @@ def get_dataset1(
         "concentration": ["0.1%", "200 nM", "0.1%"],
         "treatment_time_h": [24, 24, 6],
         "donor": ["D0001", "D0002", None],
+        "donor_ethnicity": [
+            ["African", "African American"],
+            ["African", "West African"],
+            ["Asian"],
+        ],
     }
     # define the dataset-level metadata
     metadata = {
@@ -124,6 +129,9 @@ def get_dataset1(
             dataset_df.attrs[key] = value
         return dataset_df
     else:
+        del dataset_df[
+            "donor_ethnicity"
+        ]  # remove the donor_ethnicity because AnnData save will error
         dataset_ad = ad.AnnData(
             dataset_df.iloc[:, :3], obs=dataset_df.iloc[:, 3:], uns=metadata
         )

lamindb/core/loaders.py

@@ -44,7 +44,7 @@ try:
 except ImportError:
 
     def load_zarr(storepath):  # type: ignore
-        raise ImportError("Please install zarr: pip install 'zarr<=2.18.4'")
+        raise ImportError("Please install zarr: pip install 'lamindb[zarr]'")
 
 
 is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)

lamindb/core/storage/_anndata_accessor.py

@@ -18,7 +18,7 @@ from anndata.compat import _read_attr
 from fsspec.implementations.local import LocalFileSystem
 from fsspec.utils import infer_compression
 from lamin_utils import logger
-from lamindb_setup.core.upath import create_mapper, infer_filesystem
+from lamindb_setup.core.upath import infer_filesystem
 from packaging import version
 from upath import UPath
 
@@ -288,6 +288,8 @@ except ImportError:
 if ZARR_INSTALLED:
     from anndata._io.zarr import read_dataframe_legacy as read_dataframe_legacy_zarr
 
+    from ._zarr import get_zarr_store
+
     ArrayTypes.append(zarr.Array)
     GroupTypes.append(zarr.Group)
     StorageTypes.append(zarr.Group)
@@ -296,14 +298,9 @@ if ZARR_INSTALLED:
     def open(filepath: UPathStr, mode: Literal["r", "r+", "a", "w", "w-"] = "r"):
         assert mode in {"r", "r+", "a", "w", "w-"}, f"Unknown mode {mode}!"  #  noqa: S101
 
-        fs, file_path_str = infer_filesystem(filepath)
+        store = get_zarr_store(filepath)
+        storage = zarr.open(store, mode=mode)
         conn = None
-        if isinstance(fs, LocalFileSystem):
-            # this is faster than through an fsspec mapper for local
-            open_obj = file_path_str
-        else:
-            open_obj = create_mapper(fs, file_path_str, check=True)
-        storage = zarr.open(open_obj, mode=mode)
         return conn, storage
 
     @registry.register("zarr")
@@ -348,7 +345,10 @@ if ZARR_INSTALLED:
     # this is needed because accessing zarr.Group.keys() directly is very slow
     @registry.register("zarr")
     def keys(storage: zarr.Group):
-        paths = storage._store.keys()
+        if hasattr(storage, "_sync_iter"):  # zarr v3
+            paths = storage._sync_iter(storage.store.list())
+        else:
+            paths = storage.store.keys()  # zarr v2
 
         attrs_keys: dict[str, list] = {}
         obs_var_arrays = []

lamindb/core/storage/_valid_suffixes.py

@@ -6,6 +6,7 @@ from lamindb_setup.core.upath import VALID_COMPOSITE_SUFFIXES, VALID_SIMPLE_SUFF
 VALID_COMPOSITE_SUFFIXES.update(
     {
         ".vitessce.json",
+        "spatialdata.zarr",
         ".ome.zarr",
     }
 )

lamindb/core/storage/_zarr.py

@@ -1,47 +1,47 @@
 from __future__ import annotations
 
-import warnings
 from typing import TYPE_CHECKING, Literal
 
-import scipy.sparse as sparse
 import zarr
 from anndata import __version__ as anndata_version
-from anndata._io.specs import write_elem
-from fsspec.implementations.local import LocalFileSystem
 from lamin_utils import logger
-from lamindb_setup.core.upath import S3FSMap, create_mapper, infer_filesystem
+from lamindb_setup.core.upath import LocalPathClasses, S3FSMap, UPath, create_mapper
 from packaging import version
 
 from lamindb.core._compat import with_package
 
-from ._anndata_sizes import _size_elem, _size_raw, size_adata
-
 if version.parse(anndata_version) < version.parse("0.11.0"):
     from anndata._io import read_zarr as read_anndata_zarr
 else:
     from anndata.io import read_zarr as read_anndata_zarr
 
+if version.parse(zarr.__version__) >= version.parse("3.0.0a0"):
+    IS_ZARR_V3 = True
+    from zarr.abc.store import Store
+else:
+    IS_ZARR_V3 = False
+    from zarr.storage import Store  # noqa
 
 if TYPE_CHECKING:
-    from anndata import AnnData
     from fsspec import FSMap
     from lamindb_setup.core.types import UPathStr
 
     from lamindb.core.types import ScverseDataStructures
 
 
-def create_zarr_open_obj(
-    storepath: UPathStr, *, check: bool = True
-) -> str | S3FSMap | FSMap:
+def get_zarr_store(
+    path: UPathStr, *, check: bool = False, create: bool = False
+) -> str | S3FSMap | FSMap | Store:
     """Creates the correct object that can be used to open a zarr file depending on local or remote location."""
-    fs, storepath_str = infer_filesystem(storepath)
-
-    if isinstance(fs, LocalFileSystem):
-        open_obj = storepath_str
+    storepath, storepath_str = UPath(path), str(path)
+    if isinstance(storepath, LocalPathClasses):
+        store = storepath_str
+    elif IS_ZARR_V3:
+        store = zarr.storage.FsspecStore.from_upath(UPath(storepath, asynchronous=True))
     else:
-        open_obj = create_mapper(fs, storepath_str, check=check)
+        store = create_mapper(storepath.fs, storepath_str, check=check, create=create)
 
-    return open_obj
+    return store
 
 
 def _identify_zarr_type_from_storage(
@@ -64,19 +64,17 @@ def identify_zarr_type(
     storepath: UPathStr, *, check: bool = True
 ) -> Literal["anndata", "mudata", "spatialdata", "unknown"]:
     """Identify whether a zarr store is AnnData, SpatialData, or unknown type."""
-    # we can add these cheap suffix-based-checks later
-    # also need to check whether the .spatialdata.zarr suffix
-    # actually becomes a "standard"; currently we don't recognize it
-    # unlike ".anndata.zarr" in VALID_SUFFIXES
-    # suffixes = UPath(storepath).suffixes
-    # if ".spatialdata" in suffixes:
-    #     return "spatialdata"
-    # elif ".anndata" in suffixes:
-    #     return "anndata"
-
-    open_obj = create_zarr_open_obj(storepath, check=check)
+    suffixes = UPath(storepath).suffixes
+    if ".anndata" in suffixes:
+        return "anndata"
+    elif ".mudata" in suffixes:
+        return "mudata"
+    elif ".spatialdata" in suffixes:
+        return "spatialdata"
+
+    store = get_zarr_store(storepath, check=check)
     try:
-        storage = zarr.open(open_obj, mode="r")
+        storage = zarr.open(store, mode="r")
         return _identify_zarr_type_from_storage(storage)
     except Exception as error:
         logger.warning(
@@ -96,11 +94,10 @@ def load_zarr(
         expected_type: If provided, ensures the zarr store is of this type ("anndata", "mudata", "spatialdata")
                        and raises ValueError if it's not
     """
-    open_obj = create_zarr_open_obj(storepath, check=True)
-
+    store = get_zarr_store(storepath, check=True)
     # Open the storage once
     try:
-        storage = zarr.open(open_obj, mode="r")
+        storage = zarr.open(store, mode="r")
     except Exception as error:
         raise ValueError(f"Could not open zarr store: {error}") from None
 
@@ -112,85 +109,13 @@ def load_zarr(
 
     match actual_type:
         case "anndata":
-            scverse_obj = read_anndata_zarr(open_obj)
+            scverse_obj = read_anndata_zarr(store)
         case "mudata":
-            scverse_obj = with_package("mudata", lambda mod: mod.read_zarr(open_obj))
+            scverse_obj = with_package("mudata", lambda mod: mod.read_zarr(store))
         case "spatialdata":
-            scverse_obj = with_package(
-                "spatialdata", lambda mod: mod.read_zarr(open_obj)
-            )
+            scverse_obj = with_package("spatialdata", lambda mod: mod.read_zarr(store))
         case "unknown" | _:
             raise ValueError(
                 "Unable to determine zarr store format and therefore cannot load Artifact."
             )
     return scverse_obj
-
-
-def write_adata_zarr(
-    adata: AnnData, storepath: UPathStr, callback=None, chunks=None, **dataset_kwargs
-) -> None:
-    fs, storepath_str = infer_filesystem(storepath)
-    store = create_mapper(fs, storepath_str, create=True)
-
-    f = zarr.open(store, mode="w")
-
-    adata.strings_to_categoricals()
-    if adata.raw is not None:
-        adata.strings_to_categoricals(adata.raw.var)
-
-    f.attrs.setdefault("encoding-type", "anndata")
-    f.attrs.setdefault("encoding-version", "0.1.0")
-
-    adata_size = None
-    cumulative_val = 0
-
-    def _report_progress(key_write: str | None = None):
-        nonlocal adata_size
-        nonlocal cumulative_val
-
-        if callback is None:
-            return None
-        if adata_size is None:
-            adata_size = size_adata(adata)
-        if key_write is None:
-            # begin or finish
-            if cumulative_val < adata_size:
-                callback(adata_size, adata_size if cumulative_val > 0 else 0)
-            return None
-
-        elem = getattr(adata, key_write, None)
-        if elem is None:
-            return None
-        elem_size = _size_raw(elem) if key_write == "raw" else _size_elem(elem)
-        if elem_size == 0:
-            return None
-
-        cumulative_val += elem_size
-        callback(adata_size, cumulative_val)
-
-    def _write_elem_cb(f, k, elem, dataset_kwargs):
-        write_elem(f, k, elem, dataset_kwargs=dataset_kwargs)
-        _report_progress(k)
-
-    _report_progress(None)
-    with warnings.catch_warnings():
-        warnings.filterwarnings("ignore", category=UserWarning, module="zarr")
-
-        if chunks is not None and not isinstance(adata.X, sparse.spmatrix):
-            _write_elem_cb(
-                f,
-                "X",
-                adata.X,
-                dataset_kwargs=dict(chunks=chunks, **dataset_kwargs),
-            )
-        else:
-            _write_elem_cb(f, "X", adata.X, dataset_kwargs=dataset_kwargs)
-        for elem in ("obs", "var"):
-            _write_elem_cb(f, elem, getattr(adata, elem), dataset_kwargs=dataset_kwargs)
-        for elem in ("obsm", "varm", "obsp", "varp", "layers", "uns"):
-            _write_elem_cb(
-                f, elem, dict(getattr(adata, elem)), dataset_kwargs=dataset_kwargs
-            )
-        _write_elem_cb(f, "raw", adata.raw, dataset_kwargs=dataset_kwargs)
-    # todo: fix size less than total at the end
-    _report_progress(None)

lamindb/curators/__init__.py

@@ -5,8 +5,9 @@
 
    DataFrameCurator
    AnnDataCurator
-   SpatialDataCurator
    MuDataCurator
+   SpatialDataCurator
+   TiledbsomaExperimentCurator
 
 Modules.
 
@@ -21,4 +22,20 @@ from ._legacy import (  # backward compat
     CellxGeneAnnDataCatManager,
     PertAnnDataCatManager,
 )
-from .core import AnnDataCurator, DataFrameCurator, MuDataCurator, SpatialDataCurator
+from .core import (
+    AnnDataCurator,
+    DataFrameCurator,
+    MuDataCurator,
+    SpatialDataCurator,
+    TiledbsomaExperimentCurator,
+)
+
+__all__ = [
+    "CellxGeneAnnDataCatManager",
+    "PertAnnDataCatManager",
+    "AnnDataCurator",
+    "DataFrameCurator",
+    "MuDataCurator",
+    "SpatialDataCurator",
+    "TiledbsomaExperimentCurator",
+]

lamindb/curators/_cellxgene_schemas/__init__.py

@@ -3,7 +3,7 @@ from lamin_utils import logger
 from lamindb_setup.core.upath import UPath
 
 from lamindb.base.types import FieldAttr
-from lamindb.models import Record, ULabel
+from lamindb.models import SQLRecord, ULabel
 from lamindb.models._from_values import _format_values
 
 RESERVED_NAMES = {
@@ -92,11 +92,11 @@ def _add_defaults_to_obs(obs: pd.DataFrame, defaults: dict[str, str]) -> None:
 
 def _create_sources(
     categoricals: dict[str, FieldAttr], schema_version: str, organism: str
-) -> dict[str, Record]:
+) -> dict[str, SQLRecord]:
     """Creates a sources dictionary that can be passed to AnnDataCatManager."""
     import bionty as bt
 
-    def _fetch_bionty_source(entity: str, organism: str) -> Record | None:  # type: ignore
+    def _fetch_bionty_source(entity: str, organism: str) -> SQLRecord | None:  # type: ignore
         """Fetch the Bionty source of the pinned ontology."""
         entity_sources = sources_df.loc[(sources_df.entity == entity)].copy()
         if not entity_sources.empty: