lamindb 1.10.1__tar.gz → 1.10.2__tar.gz

{lamindb-1.10.1 → lamindb-1.10.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: lamindb
-Version: 1.10.1
+Version: 1.10.2
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10,<3.14
@@ -9,6 +9,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+License-File: LICENSE
 Requires-Dist: lamin_utils==0.15.0
 Requires-Dist: lamin_cli==1.6.1
 Requires-Dist: lamindb_setup[aws]==1.9.1

{lamindb-1.10.1 → lamindb-1.10.2}/lamindb/__init__.py

@@ -108,7 +108,7 @@ Backwards compatibility.
 
 # ruff: noqa: I001
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "1.10.1"
+__version__ = "1.10.2"
 
 import warnings
 

{lamindb-1.10.1 → lamindb-1.10.2}/lamindb/core/_settings.py

@@ -10,6 +10,7 @@ from lamindb_setup import settings as setup_settings
 from lamindb_setup._set_managed_storage import set_managed_storage
 from lamindb_setup.core import deprecated
 from lamindb_setup.core._settings_instance import sanitize_git_repo_url
+from lamindb_setup.core._settings_storage import StorageSettings
 
 from .subsettings._annotation_settings import AnnotationSettings, annotation_settings
 from .subsettings._creation_settings import CreationSettings, creation_settings
@@ -18,7 +19,6 @@ if TYPE_CHECKING:
     from collections.abc import Mapping
     from pathlib import Path
 
-    from lamindb_setup.core._settings_storage import StorageSettings
     from upath import UPath
 
 
@@ -193,13 +193,39 @@ class Settings:
 
     @storage.setter
     def storage(self, path_kwargs: str | Path | UPath | tuple[str | UPath, Mapping]):
+        import lamindb as ln
+
         if isinstance(path_kwargs, tuple):
             path, kwargs = path_kwargs
+            # we should ultimately deprecate passing host here, I think
             if isinstance(kwargs, str):
                 kwargs = {"host": kwargs}
         else:
             path, kwargs = path_kwargs, {}
-        set_managed_storage(path, **kwargs)
+        ssettings = StorageSettings(root=path)  # there is no need to pass kwargs here!
+        exists = ln.Storage.filter(root=ssettings.root_as_str).one_or_none()
+        if exists is None:
+            response = input(
+                f"Storage location {ssettings.root_as_str} does not yet exist. Do you want to continue with creating it? (y/n)"
+            )
+            # logger.warning(f"deprecated call because storage location does **not yet** exist; going forward, please create through ln.Storage(root={path}).save() going forward")
+            if response != "y":
+                return None
+            set_managed_storage(path, **kwargs)
+        else:
+            if exists.instance_uid != ln_setup.settings.instance.uid:
+                raise ValueError(
+                    f"Storage {ssettings.root_as_str} exists in another instance ({exists.instance_uid}), cannot write to it from here."
+                )
+            ssettings = StorageSettings(
+                root=exists.root,
+                region=exists.region,
+                uid=exists.uid,
+                instance_id=ln_setup.settings.instance._id,
+            )
+            ln_setup.settings.instance._storage = ssettings
+            kwargs.pop("host", None)  # host is not needed for existing storage
+            settings.storage._set_fs_kwargs(**kwargs)
 
     @property
     def instance_uid(self) -> str:
@@ -223,6 +249,23 @@ class Settings:
 
     @local_storage.setter
     def local_storage(self, local_root: Path):
+        import lamindb as ln
+
+        # note duplication with storage setter!
+        ssettings = StorageSettings(root=local_root)
+        exists = ln.Storage.filter(root=ssettings.root_as_str).one_or_none()
+        if exists is None:
+            response = input(
+                f"Storage location {ssettings.root_as_str} does not yet exist. Do you want to continue with creating it? (y/n)"
+            )
+            # logger.warning(f"deprecated call because storage location does **not yet** exist; going forward, please create through ln.Storage(root={path}).save() going forward")
+            if response != "y":
+                return None
+        else:
+            if exists.instance_uid != ln_setup.settings.instance.uid:
+                raise ValueError(
+                    f"Storage {ssettings.root_as_str} exists in another instance ({exists.instance_uid}), cannot write to it from here."
+                )
         ln_setup.settings.instance.local_storage = local_root
 
     @property

{lamindb-1.10.1 → lamindb-1.10.2}/lamindb/core/storage/_anndata_accessor.py

@@ -13,12 +13,17 @@ from anndata import __version__ as anndata_version
 from anndata._core.index import _normalize_indices
 from anndata._core.views import _resolve_idx
 from anndata._io.h5ad import read_dataframe_legacy as read_dataframe_legacy_h5
-from anndata._io.specs.registry import get_spec, read_elem, read_elem_partial
+from anndata._io.specs.registry import (
+    get_spec,
+    read_elem,
+    read_elem_partial,
+    write_elem,
+)
 from anndata.compat import _read_attr
 from fsspec.implementations.local import LocalFileSystem
 from fsspec.utils import infer_compression
 from lamin_utils import logger
-from lamindb_setup.core.upath import infer_filesystem
+from lamindb_setup.core.upath import S3FSMap, infer_filesystem
 from packaging import version
 from upath import UPath
 
@@ -28,6 +33,8 @@ if TYPE_CHECKING:
     from fsspec.core import OpenFile
     from lamindb_setup.types import UPathStr
 
+    from lamindb import Artifact
+
 
 anndata_version_parse = version.parse(anndata_version)
 
@@ -300,6 +307,11 @@ if ZARR_INSTALLED:
 
         store = get_zarr_store(filepath)
         storage = zarr.open(store, mode=mode)
+        # zarr v2 re-initializes the mapper
+        # we need to put back the correct one
+        # S3FSMap is returned from get_zarr_store only for zarr v2
+        if isinstance(store, S3FSMap):
+            storage.store.map = store
         conn = None
         return conn, storage
 
@@ -438,9 +450,15 @@ def _try_backed_full(elem):
     return read_elem(elem)
 
 
+def _to_index(elem: np.ndarray):
+    if elem.dtype in (np.float64, np.int64):
+        elem = elem.astype(str)
+    return pd.Index(elem)
+
+
 def _safer_read_index(elem):
     if isinstance(elem, GroupTypes):
-        return pd.Index(read_elem(elem[_read_attr(elem.attrs, "_index")]))
+        return _to_index(read_elem(elem[_read_attr(elem.attrs, "_index")]))
     elif isinstance(elem, ArrayTypes):
         indices = None
         for index_name in ("index", "_index"):
@@ -450,7 +468,7 @@ def _safer_read_index(elem):
         if indices is not None and len(indices) > 0:
             if isinstance(indices[0], bytes):
                 indices = np.frompyfunc(lambda x: x.decode("utf-8"), 1, 1)(indices)
-            return pd.Index(indices)
+            return _to_index(indices)
         else:
             raise ValueError("Indices not found.")
     else:
@@ -479,33 +497,40 @@ class _MapAccessor:
         return descr
 
 
+def _safer_read_df(elem, indices=None):
+    if indices is not None:
+        obj = registry.safer_read_partial(elem, indices=indices)
+        df = _records_to_df(obj)
+    else:
+        df = registry.read_dataframe(elem)
+    if df.index.dtype in (np.float64, np.int64):
+        df.index = df.index.astype(str)
+    return df
+
+
 class _AnnDataAttrsMixin:
     storage: StorageType
     _attrs_keys: Mapping[str, list]
 
     @cached_property
-    def obs(self) -> pd.DataFrame:
+    def obs(self) -> pd.DataFrame | None:
         if "obs" not in self._attrs_keys:
             return None
         indices = getattr(self, "indices", None)
-        if indices is not None:
-            indices = (indices[0], slice(None))
-            obj = registry.safer_read_partial(self.storage["obs"], indices=indices)  # type: ignore
-            return _records_to_df(obj)
-        else:
-            return registry.read_dataframe(self.storage["obs"])  # type: ignore
+        return _safer_read_df(
+            self.storage["obs"],  # type: ignore
+            indices=(indices[0], slice(None)) if indices is not None else None,
+        )
 
     @cached_property
-    def var(self) -> pd.DataFrame:
+    def var(self) -> pd.DataFrame | None:
         if "var" not in self._attrs_keys:
             return None
         indices = getattr(self, "indices", None)
-        if indices is not None:
-            indices = (indices[1], slice(None))
-            obj = registry.safer_read_partial(self.storage["var"], indices=indices)  # type: ignore
-            return _records_to_df(obj)
-        else:
-            return registry.read_dataframe(self.storage["var"])  # type: ignore
+        return _safer_read_df(
+            self.storage["var"],  # type: ignore
+            indices=(indices[1], slice(None)) if indices is not None else None,
+        )
 
     @cached_property
     def uns(self):
@@ -702,6 +727,7 @@ class AnnDataAccessor(_AnnDataAttrsMixin):
         connection: OpenFile | None,
         storage: StorageType,
         filename: str,
+        artifact: Artifact | None = None,
     ):
         self._conn = connection
         self.storage = storage
@@ -713,6 +739,11 @@ class AnnDataAccessor(_AnnDataAttrsMixin):
         self._obs_names = _safer_read_index(self.storage["obs"])  # type: ignore
         self._var_names = _safer_read_index(self.storage["var"])  # type: ignore
 
+        self._artifact = artifact  # save artifact to update in write mode
+
+        self._updated = False  # track updates in r+ mode for zarr
+
+        self._entered = False  # check that the context manager is used
         self._closed = False
 
     def close(self):
@@ -723,11 +754,23 @@ class AnnDataAccessor(_AnnDataAttrsMixin):
             self._conn.close()
         self._closed = True
 
+        if self._updated and (artifact := self._artifact) is not None:
+            from lamindb.models.artifact import Artifact
+            from lamindb.models.sqlrecord import init_self_from_db
+
+            new_version = Artifact(
+                artifact.path, revises=artifact, _is_internal_call=True
+            ).save()
+            # note: sets _state.db = "default"
+            init_self_from_db(artifact, new_version)
+
     @property
     def closed(self):
         return self._closed
 
     def __enter__(self):
+        self._entered = True
+
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
@@ -763,6 +806,35 @@ class AnnDataAccessor(_AnnDataAttrsMixin):
             self.storage["raw"], None, None, self._obs_names, None, self.shape[0]
         )
 
+    def add_column(
+        self,
+        where: Literal["obs", "var"],
+        col_name: str,
+        col: np.ndarray | pd.Categorical,
+    ):
+        """Add a new column to .obs or .var of the underlying AnnData object."""
+        df_store = self.storage[where]  # type: ignore
+        if getattr(df_store, "read_only", True):
+            raise ValueError(
+                "You can use .add_column(...) only with zarr in a writable mode."
+            )
+        write_elem(df_store, col_name, col)
+        df_store.attrs["column-order"] = df_store.attrs["column-order"] + [col_name]
+        # remind only once if this wasn't updated before and not in the context manager
+        if not self._updated and not self._entered and self._artifact is not None:
+            logger.important(
+                "Do not forget to call .close() after you finish "
+                f"working with this accessor for {self._name} "
+                "to automatically update the corresponding artifact."
+            )
+
+        self._updated = True
+        # reset the cached property
+        # todo: maybe just append the column if the df was already loaded
+        self.__dict__.pop(where, None)
+        # update the cached columns
+        self._attrs_keys[where].append(col_name)
+
 
 # get the number of observations in an anndata object or file fast and safely
 def _anndata_n_observations(object: UPathStr | AnnData) -> int | None:

{lamindb-1.10.1 → lamindb-1.10.2}/lamindb/core/storage/_backed_access.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Literal
 
+import h5py
 from anndata._io.specs.registry import get_spec
 
 from ._anndata_accessor import AnnDataAccessor, StorageType, registry
@@ -92,10 +93,10 @@ def backed_access(
     from lamindb.models import Artifact
 
     if isinstance(artifact_or_filepath, Artifact):
-        objectpath, _ = filepath_from_artifact(
-            artifact_or_filepath, using_key=using_key
-        )
+        artifact = artifact_or_filepath
+        objectpath, _ = filepath_from_artifact(artifact, using_key=using_key)
     else:
+        artifact = None
         objectpath = artifact_or_filepath
     name = objectpath.name
     # ignore .gz, only check the real suffix
@@ -111,9 +112,11 @@ def backed_access(
     elif suffix in {".h5", ".hdf5", ".h5ad"}:
         conn, storage = registry.open("h5py", objectpath, mode=mode, **kwargs)
     elif suffix == ".zarr":
+        if mode not in {"r", "r+"}:
+            raise ValueError("`mode` should be either 'r' or 'r+' for zarr.")
         conn, storage = registry.open("zarr", objectpath, mode=mode, **kwargs)
         if "spatialdata_attrs" in storage.attrs:
-            return SpatialDataAccessor(storage, name)
+            return SpatialDataAccessor(storage, name, artifact)
     elif len(df_suffixes := _flat_suffixes(objectpath)) == 1 and (
         df_suffix := df_suffixes.pop()
     ) in set(PYARROW_SUFFIXES).union(POLARS_SUFFIXES):
@@ -127,9 +130,9 @@ def backed_access(
 
     is_anndata = suffix == ".h5ad" or get_spec(storage).encoding_type == "anndata"
     if is_anndata:
-        if mode != "r":
-            raise ValueError("Can only access `AnnData` with mode='r'.")
-        return AnnDataAccessor(conn, storage, name)
+        if mode != "r" and isinstance(storage, h5py.Group):
+            raise ValueError("Can only access `hdf5` `AnnData` with mode='r'.")
+        return AnnDataAccessor(conn, storage, name, artifact)
     else:
         return BackedAccessor(conn, storage)
 

{lamindb-1.10.1 → lamindb-1.10.2}/lamindb/core/storage/_spatialdata_accessor.py

@@ -8,13 +8,22 @@ from ._anndata_accessor import AnnDataAccessor
 if TYPE_CHECKING:
     from zarr import Group
 
+    from lamindb import Artifact
+
 
 class _TablesAccessor:
-    def __init__(self, tables: Group):
+    def __init__(self, tables: Group, artifact: Artifact | None = None):
         self._tables = tables
 
+        self._artifact = artifact
+
     def __getitem__(self, key: str) -> AnnDataAccessor:
-        return AnnDataAccessor(connection=None, storage=self._tables[key], filename=key)
+        return AnnDataAccessor(
+            connection=None,
+            storage=self._tables[key],
+            filename=key,
+            artifact=self._artifact,
+        )
 
     def keys(self) -> list[str]:
         return list(self._tables.keys())
@@ -33,14 +42,16 @@ class SpatialDataAccessor:
     For now only allows to access `tables`.
     """
 
-    def __init__(self, storage: Group, name: str):
+    def __init__(self, storage: Group, name: str, artifact: Artifact | None = None):
         self.storage = storage
         self._name = name
 
+        self._artifact = artifact
+
     @cached_property
     def tables(self) -> _TablesAccessor:
         """tables of the underlying SpatialData object."""
-        return _TablesAccessor(self.storage["tables"])
+        return _TablesAccessor(self.storage["tables"], self._artifact)
 
     def __repr__(self):
         """Description of the SpatialDataAccessor object."""

{lamindb-1.10.1 → lamindb-1.10.2}/lamindb/core/storage/_zarr.py

@@ -37,6 +37,9 @@ def get_zarr_store(
     if isinstance(storepath, LocalPathClasses):
         store = storepath_str
     elif IS_ZARR_V3:
+        # todo: also check how to treat non-asynchronous filesystems
+        # zarr has something for this, using fsspec async wrapper
+        # check FsspecStore code
         store = zarr.storage.FsspecStore.from_upath(UPath(storepath, asynchronous=True))
     else:
         store = create_mapper(storepath.fs, storepath_str, check=check, create=create)

{lamindb-1.10.1 → lamindb-1.10.2}/lamindb/curators/core.py

@@ -364,12 +364,13 @@ class SlotsCurator(Curator):
                     )
                     break
 
-            self._artifact.schema = self._schema
-            self._artifact.save()
         cat_vectors = {}
         for curator in self._slots.values():
             for key, cat_vector in curator.cat._cat_vectors.items():
                 cat_vectors[key] = cat_vector
+
+        self._artifact.schema = self._schema
+        self._artifact.save()
         return annotate_artifact(  # type: ignore
             self._artifact,
             curator=self,
@@ -685,10 +686,11 @@ class DataFrameCurator(Curator):
                 description=description,
                 revises=revises,
                 run=run,
-                format=".csv" if key.endswith(".csv") else None,
+                format=".csv" if key is not None and key.endswith(".csv") else None,
             )
-            self._artifact.schema = self._schema
-            self._artifact.save()
+
+        self._artifact.schema = self._schema
+        self._artifact.save()
         return annotate_artifact(  # type: ignore
             self._artifact,
             cat_vectors=self.cat._cat_vectors,

lamindb-1.10.2/lamindb/migrations/0118_alter_recordproject_value_projectrecord.py

@@ -0,0 +1,99 @@
+# Generated by Django 5.2 on 2025-08-07 15:28
+
+import django.db.models.deletion
+import django.db.models.functions.datetime
+from django.db import migrations, models
+
+import lamindb.base.fields
+import lamindb.base.users
+import lamindb.models.run
+import lamindb.models.sqlrecord
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lamindb", "0117_fix_artifact_storage_hash_unique_constraints"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="recordproject",
+            name="value",
+            field=lamindb.base.fields.ForeignKey(
+                blank=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="links_in_record",
+                to="lamindb.project",
+            ),
+        ),
+        migrations.CreateModel(
+            name="ProjectRecord",
+            fields=[
+                (
+                    "created_at",
+                    lamindb.base.fields.DateTimeField(
+                        blank=True,
+                        db_default=django.db.models.functions.datetime.Now(),
+                        db_index=True,
+                        editable=False,
+                    ),
+                ),
+                ("id", models.BigAutoField(primary_key=True, serialize=False)),
+                (
+                    "created_by",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        default=lamindb.base.users.current_user_id,
+                        editable=False,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="+",
+                        to="lamindb.user",
+                    ),
+                ),
+                (
+                    "feature",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        default=None,
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="links_projectrecord",
+                        to="lamindb.feature",
+                    ),
+                ),
+                (
+                    "project",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="links_record",
+                        to="lamindb.project",
+                    ),
+                ),
+                (
+                    "record",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="links_project",
+                        to="lamindb.record",
+                    ),
+                ),
+                (
+                    "run",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        default=lamindb.models.run.current_run,
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="+",
+                        to="lamindb.run",
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": {("record", "project", "feature")},
+            },
+            bases=(lamindb.models.sqlrecord.IsLink, models.Model),
+        ),
+    ]

lamindb-1.10.2/lamindb/migrations/0119_rename_records_project_linked_in_records.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.2 on 2025-08-09 13:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lamindb", "0118_alter_recordproject_value_projectrecord"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="project",
+            old_name="records",
+            new_name="linked_in_records",
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="records",
+            field=models.ManyToManyField(
+                related_name="projects",
+                through="lamindb.ProjectRecord",
+                to="lamindb.record",
+            ),
+        ),
+    ]