lamindb 1.10.1__py3-none-any.whl → 1.10.2__py3-none-any.whl

lamindb/__init__.py

@@ -108,7 +108,7 @@ Backwards compatibility.
 
 # ruff: noqa: I001
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "1.10.1"
+__version__ = "1.10.2"
 
 import warnings
 

lamindb/core/_settings.py

@@ -10,6 +10,7 @@ from lamindb_setup import settings as setup_settings
 from lamindb_setup._set_managed_storage import set_managed_storage
 from lamindb_setup.core import deprecated
 from lamindb_setup.core._settings_instance import sanitize_git_repo_url
+from lamindb_setup.core._settings_storage import StorageSettings
 
 from .subsettings._annotation_settings import AnnotationSettings, annotation_settings
 from .subsettings._creation_settings import CreationSettings, creation_settings
@@ -18,7 +19,6 @@ if TYPE_CHECKING:
     from collections.abc import Mapping
     from pathlib import Path
 
-    from lamindb_setup.core._settings_storage import StorageSettings
     from upath import UPath
 
 
@@ -193,13 +193,39 @@ class Settings:
 
     @storage.setter
     def storage(self, path_kwargs: str | Path | UPath | tuple[str | UPath, Mapping]):
+        import lamindb as ln
+
         if isinstance(path_kwargs, tuple):
             path, kwargs = path_kwargs
+            # we should ultimately deprecate passing host here, I think
             if isinstance(kwargs, str):
                 kwargs = {"host": kwargs}
         else:
             path, kwargs = path_kwargs, {}
-        set_managed_storage(path, **kwargs)
+        ssettings = StorageSettings(root=path)  # there is no need to pass kwargs here!
+        exists = ln.Storage.filter(root=ssettings.root_as_str).one_or_none()
+        if exists is None:
+            response = input(
+                f"Storage location {ssettings.root_as_str} does not yet exist. Do you want to continue with creating it? (y/n)"
+            )
+            # logger.warning(f"deprecated call because storage location does **not yet** exist; going forward, please create through ln.Storage(root={path}).save() going forward")
+            if response != "y":
+                return None
+            set_managed_storage(path, **kwargs)
+        else:
+            if exists.instance_uid != ln_setup.settings.instance.uid:
+                raise ValueError(
+                    f"Storage {ssettings.root_as_str} exists in another instance ({exists.instance_uid}), cannot write to it from here."
+                )
+            ssettings = StorageSettings(
+                root=exists.root,
+                region=exists.region,
+                uid=exists.uid,
+                instance_id=ln_setup.settings.instance._id,
+            )
+            ln_setup.settings.instance._storage = ssettings
+            kwargs.pop("host", None)  # host is not needed for existing storage
+            settings.storage._set_fs_kwargs(**kwargs)
 
     @property
     def instance_uid(self) -> str:
@@ -223,6 +249,23 @@ class Settings:
 
     @local_storage.setter
     def local_storage(self, local_root: Path):
+        import lamindb as ln
+
+        # note duplication with storage setter!
+        ssettings = StorageSettings(root=local_root)
+        exists = ln.Storage.filter(root=ssettings.root_as_str).one_or_none()
+        if exists is None:
+            response = input(
+                f"Storage location {ssettings.root_as_str} does not yet exist. Do you want to continue with creating it? (y/n)"
+            )
+            # logger.warning(f"deprecated call because storage location does **not yet** exist; going forward, please create through ln.Storage(root={path}).save() going forward")
+            if response != "y":
+                return None
+        else:
+            if exists.instance_uid != ln_setup.settings.instance.uid:
+                raise ValueError(
+                    f"Storage {ssettings.root_as_str} exists in another instance ({exists.instance_uid}), cannot write to it from here."
+                )
         ln_setup.settings.instance.local_storage = local_root
 
     @property

lamindb/core/storage/_anndata_accessor.py

@@ -13,12 +13,17 @@ from anndata import __version__ as anndata_version
 from anndata._core.index import _normalize_indices
 from anndata._core.views import _resolve_idx
 from anndata._io.h5ad import read_dataframe_legacy as read_dataframe_legacy_h5
-from anndata._io.specs.registry import get_spec, read_elem, read_elem_partial
+from anndata._io.specs.registry import (
+    get_spec,
+    read_elem,
+    read_elem_partial,
+    write_elem,
+)
 from anndata.compat import _read_attr
 from fsspec.implementations.local import LocalFileSystem
 from fsspec.utils import infer_compression
 from lamin_utils import logger
-from lamindb_setup.core.upath import infer_filesystem
+from lamindb_setup.core.upath import S3FSMap, infer_filesystem
 from packaging import version
 from upath import UPath
 
@@ -28,6 +33,8 @@ if TYPE_CHECKING:
     from fsspec.core import OpenFile
     from lamindb_setup.types import UPathStr
 
+    from lamindb import Artifact
+
 
 anndata_version_parse = version.parse(anndata_version)
 
@@ -300,6 +307,11 @@ if ZARR_INSTALLED:
 
         store = get_zarr_store(filepath)
         storage = zarr.open(store, mode=mode)
+        # zarr v2 re-initializes the mapper
+        # we need to put back the correct one
+        # S3FSMap is returned from get_zarr_store only for zarr v2
+        if isinstance(store, S3FSMap):
+            storage.store.map = store
         conn = None
         return conn, storage
 
@@ -438,9 +450,15 @@ def _try_backed_full(elem):
     return read_elem(elem)
 
 
+def _to_index(elem: np.ndarray):
+    if elem.dtype in (np.float64, np.int64):
+        elem = elem.astype(str)
+    return pd.Index(elem)
+
+
 def _safer_read_index(elem):
     if isinstance(elem, GroupTypes):
-        return pd.Index(read_elem(elem[_read_attr(elem.attrs, "_index")]))
+        return _to_index(read_elem(elem[_read_attr(elem.attrs, "_index")]))
     elif isinstance(elem, ArrayTypes):
         indices = None
         for index_name in ("index", "_index"):
@@ -450,7 +468,7 @@ def _safer_read_index(elem):
         if indices is not None and len(indices) > 0:
             if isinstance(indices[0], bytes):
                 indices = np.frompyfunc(lambda x: x.decode("utf-8"), 1, 1)(indices)
-            return pd.Index(indices)
+            return _to_index(indices)
         else:
             raise ValueError("Indices not found.")
     else:
@@ -479,33 +497,40 @@ class _MapAccessor:
         return descr
 
 
+def _safer_read_df(elem, indices=None):
+    if indices is not None:
+        obj = registry.safer_read_partial(elem, indices=indices)
+        df = _records_to_df(obj)
+    else:
+        df = registry.read_dataframe(elem)
+    if df.index.dtype in (np.float64, np.int64):
+        df.index = df.index.astype(str)
+    return df
+
+
 class _AnnDataAttrsMixin:
     storage: StorageType
     _attrs_keys: Mapping[str, list]
 
     @cached_property
-    def obs(self) -> pd.DataFrame:
+    def obs(self) -> pd.DataFrame | None:
         if "obs" not in self._attrs_keys:
             return None
         indices = getattr(self, "indices", None)
-        if indices is not None:
-            indices = (indices[0], slice(None))
-            obj = registry.safer_read_partial(self.storage["obs"], indices=indices)  # type: ignore
-            return _records_to_df(obj)
-        else:
-            return registry.read_dataframe(self.storage["obs"])  # type: ignore
+        return _safer_read_df(
+            self.storage["obs"],  # type: ignore
+            indices=(indices[0], slice(None)) if indices is not None else None,
+        )
 
     @cached_property
-    def var(self) -> pd.DataFrame:
+    def var(self) -> pd.DataFrame | None:
         if "var" not in self._attrs_keys:
             return None
         indices = getattr(self, "indices", None)
-        if indices is not None:
-            indices = (indices[1], slice(None))
-            obj = registry.safer_read_partial(self.storage["var"], indices=indices)  # type: ignore
-            return _records_to_df(obj)
-        else:
-            return registry.read_dataframe(self.storage["var"])  # type: ignore
+        return _safer_read_df(
+            self.storage["var"],  # type: ignore
+            indices=(indices[1], slice(None)) if indices is not None else None,
+        )
 
     @cached_property
     def uns(self):
@@ -702,6 +727,7 @@ class AnnDataAccessor(_AnnDataAttrsMixin):
         connection: OpenFile | None,
         storage: StorageType,
         filename: str,
+        artifact: Artifact | None = None,
     ):
         self._conn = connection
         self.storage = storage
@@ -713,6 +739,11 @@ class AnnDataAccessor(_AnnDataAttrsMixin):
         self._obs_names = _safer_read_index(self.storage["obs"])  # type: ignore
         self._var_names = _safer_read_index(self.storage["var"])  # type: ignore
 
+        self._artifact = artifact  # save artifact to update in write mode
+
+        self._updated = False  # track updates in r+ mode for zarr
+
+        self._entered = False  # check that the context manager is used
         self._closed = False
 
     def close(self):
@@ -723,11 +754,23 @@ class AnnDataAccessor(_AnnDataAttrsMixin):
             self._conn.close()
         self._closed = True
 
+        if self._updated and (artifact := self._artifact) is not None:
+            from lamindb.models.artifact import Artifact
+            from lamindb.models.sqlrecord import init_self_from_db
+
+            new_version = Artifact(
+                artifact.path, revises=artifact, _is_internal_call=True
+            ).save()
+            # note: sets _state.db = "default"
+            init_self_from_db(artifact, new_version)
+
     @property
     def closed(self):
         return self._closed
 
     def __enter__(self):
+        self._entered = True
+
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
@@ -763,6 +806,35 @@ class AnnDataAccessor(_AnnDataAttrsMixin):
             self.storage["raw"], None, None, self._obs_names, None, self.shape[0]
         )
 
+    def add_column(
+        self,
+        where: Literal["obs", "var"],
+        col_name: str,
+        col: np.ndarray | pd.Categorical,
+    ):
+        """Add a new column to .obs or .var of the underlying AnnData object."""
+        df_store = self.storage[where]  # type: ignore
+        if getattr(df_store, "read_only", True):
+            raise ValueError(
+                "You can use .add_column(...) only with zarr in a writable mode."
+            )
+        write_elem(df_store, col_name, col)
+        df_store.attrs["column-order"] = df_store.attrs["column-order"] + [col_name]
+        # remind only once if this wasn't updated before and not in the context manager
+        if not self._updated and not self._entered and self._artifact is not None:
+            logger.important(
+                "Do not forget to call .close() after you finish "
+                f"working with this accessor for {self._name} "
+                "to automatically update the corresponding artifact."
+            )
+
+        self._updated = True
+        # reset the cached property
+        # todo: maybe just append the column if the df was already loaded
+        self.__dict__.pop(where, None)
+        # update the cached columns
+        self._attrs_keys[where].append(col_name)
+
 
 # get the number of observations in an anndata object or file fast and safely
 def _anndata_n_observations(object: UPathStr | AnnData) -> int | None:

lamindb/core/storage/_backed_access.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Literal
 
+import h5py
 from anndata._io.specs.registry import get_spec
 
 from ._anndata_accessor import AnnDataAccessor, StorageType, registry
@@ -92,10 +93,10 @@ def backed_access(
     from lamindb.models import Artifact
 
     if isinstance(artifact_or_filepath, Artifact):
-        objectpath, _ = filepath_from_artifact(
-            artifact_or_filepath, using_key=using_key
-        )
+        artifact = artifact_or_filepath
+        objectpath, _ = filepath_from_artifact(artifact, using_key=using_key)
     else:
+        artifact = None
         objectpath = artifact_or_filepath
     name = objectpath.name
     # ignore .gz, only check the real suffix
@@ -111,9 +112,11 @@ def backed_access(
     elif suffix in {".h5", ".hdf5", ".h5ad"}:
         conn, storage = registry.open("h5py", objectpath, mode=mode, **kwargs)
     elif suffix == ".zarr":
+        if mode not in {"r", "r+"}:
+            raise ValueError("`mode` should be either 'r' or 'r+' for zarr.")
         conn, storage = registry.open("zarr", objectpath, mode=mode, **kwargs)
         if "spatialdata_attrs" in storage.attrs:
-            return SpatialDataAccessor(storage, name)
+            return SpatialDataAccessor(storage, name, artifact)
     elif len(df_suffixes := _flat_suffixes(objectpath)) == 1 and (
         df_suffix := df_suffixes.pop()
     ) in set(PYARROW_SUFFIXES).union(POLARS_SUFFIXES):
@@ -127,9 +130,9 @@ def backed_access(
 
     is_anndata = suffix == ".h5ad" or get_spec(storage).encoding_type == "anndata"
     if is_anndata:
-        if mode != "r":
-            raise ValueError("Can only access `AnnData` with mode='r'.")
-        return AnnDataAccessor(conn, storage, name)
+        if mode != "r" and isinstance(storage, h5py.Group):
+            raise ValueError("Can only access `hdf5` `AnnData` with mode='r'.")
+        return AnnDataAccessor(conn, storage, name, artifact)
     else:
         return BackedAccessor(conn, storage)
 

lamindb/core/storage/_spatialdata_accessor.py

@@ -8,13 +8,22 @@ from ._anndata_accessor import AnnDataAccessor
 if TYPE_CHECKING:
     from zarr import Group
 
+    from lamindb import Artifact
+
 
 class _TablesAccessor:
-    def __init__(self, tables: Group):
+    def __init__(self, tables: Group, artifact: Artifact | None = None):
         self._tables = tables
 
+        self._artifact = artifact
+
     def __getitem__(self, key: str) -> AnnDataAccessor:
-        return AnnDataAccessor(connection=None, storage=self._tables[key], filename=key)
+        return AnnDataAccessor(
+            connection=None,
+            storage=self._tables[key],
+            filename=key,
+            artifact=self._artifact,
+        )
 
     def keys(self) -> list[str]:
         return list(self._tables.keys())
@@ -33,14 +42,16 @@ class SpatialDataAccessor:
     For now only allows to access `tables`.
     """
 
-    def __init__(self, storage: Group, name: str):
+    def __init__(self, storage: Group, name: str, artifact: Artifact | None = None):
         self.storage = storage
         self._name = name
 
+        self._artifact = artifact
+
     @cached_property
     def tables(self) -> _TablesAccessor:
         """tables of the underlying SpatialData object."""
-        return _TablesAccessor(self.storage["tables"])
+        return _TablesAccessor(self.storage["tables"], self._artifact)
 
     def __repr__(self):
         """Description of the SpatialDataAccessor object."""

lamindb/core/storage/_zarr.py

@@ -37,6 +37,9 @@ def get_zarr_store(
     if isinstance(storepath, LocalPathClasses):
         store = storepath_str
     elif IS_ZARR_V3:
+        # todo: also check how to treat non-asynchronous filesystems
+        # zarr has something for this, using fsspec async wrapper
+        # check FsspecStore code
         store = zarr.storage.FsspecStore.from_upath(UPath(storepath, asynchronous=True))
     else:
         store = create_mapper(storepath.fs, storepath_str, check=check, create=create)

lamindb/curators/core.py

@@ -364,12 +364,13 @@ class SlotsCurator(Curator):
                     )
                     break
 
-            self._artifact.schema = self._schema
-            self._artifact.save()
         cat_vectors = {}
         for curator in self._slots.values():
             for key, cat_vector in curator.cat._cat_vectors.items():
                 cat_vectors[key] = cat_vector
+
+        self._artifact.schema = self._schema
+        self._artifact.save()
         return annotate_artifact(  # type: ignore
             self._artifact,
             curator=self,
@@ -685,10 +686,11 @@ class DataFrameCurator(Curator):
                 description=description,
                 revises=revises,
                 run=run,
-                format=".csv" if key.endswith(".csv") else None,
+                format=".csv" if key is not None and key.endswith(".csv") else None,
             )
-            self._artifact.schema = self._schema
-            self._artifact.save()
+
+        self._artifact.schema = self._schema
+        self._artifact.save()
         return annotate_artifact(  # type: ignore
             self._artifact,
             cat_vectors=self.cat._cat_vectors,

lamindb/migrations/0118_alter_recordproject_value_projectrecord.py

@@ -0,0 +1,99 @@
+# Generated by Django 5.2 on 2025-08-07 15:28
+
+import django.db.models.deletion
+import django.db.models.functions.datetime
+from django.db import migrations, models
+
+import lamindb.base.fields
+import lamindb.base.users
+import lamindb.models.run
+import lamindb.models.sqlrecord
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lamindb", "0117_fix_artifact_storage_hash_unique_constraints"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="recordproject",
+            name="value",
+            field=lamindb.base.fields.ForeignKey(
+                blank=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="links_in_record",
+                to="lamindb.project",
+            ),
+        ),
+        migrations.CreateModel(
+            name="ProjectRecord",
+            fields=[
+                (
+                    "created_at",
+                    lamindb.base.fields.DateTimeField(
+                        blank=True,
+                        db_default=django.db.models.functions.datetime.Now(),
+                        db_index=True,
+                        editable=False,
+                    ),
+                ),
+                ("id", models.BigAutoField(primary_key=True, serialize=False)),
+                (
+                    "created_by",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        default=lamindb.base.users.current_user_id,
+                        editable=False,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="+",
+                        to="lamindb.user",
+                    ),
+                ),
+                (
+                    "feature",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        default=None,
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="links_projectrecord",
+                        to="lamindb.feature",
+                    ),
+                ),
+                (
+                    "project",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="links_record",
+                        to="lamindb.project",
+                    ),
+                ),
+                (
+                    "record",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="links_project",
+                        to="lamindb.record",
+                    ),
+                ),
+                (
+                    "run",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        default=lamindb.models.run.current_run,
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="+",
+                        to="lamindb.run",
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": {("record", "project", "feature")},
+            },
+            bases=(lamindb.models.sqlrecord.IsLink, models.Model),
+        ),
+    ]

lamindb/migrations/0119_rename_records_project_linked_in_records.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.2 on 2025-08-09 13:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lamindb", "0118_alter_recordproject_value_projectrecord"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="project",
+            old_name="records",
+            new_name="linked_in_records",
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="records",
+            field=models.ManyToManyField(
+                related_name="projects",
+                through="lamindb.ProjectRecord",
+                to="lamindb.record",
+            ),
+        ),
+    ]

lamindb/migrations/{0117_squashed.py → 0119_squashed.py}

@@ -1,4 +1,4 @@
-# Generated by Django 5.2 on 2025-07-26 18:58
+# Generated by Django 5.2 on 2025-08-09 13:33
 
 import django.core.validators
 import django.db.models.deletion
@@ -139,6 +139,8 @@ class Migration(migrations.Migration):
         ("lamindb", "0115_alter_space_uid"),
         ("lamindb", "0116_remove_artifact_unique_artifact_storage_key_hash_and_more"),
         ("lamindb", "0117_fix_artifact_storage_hash_unique_constraints"),
+        ("lamindb", "0118_alter_recordproject_value_projectrecord"),
+        ("lamindb", "0119_rename_records_project_linked_in_records"),
     ]
 
     dependencies = []  # type: ignore
@@ -1522,6 +1524,60 @@ class Migration(migrations.Migration):
             },
             bases=(lamindb.models.can_curate.CanCurate, models.Model),
         ),
+        migrations.CreateModel(
+            name="ProjectRecord",
+            fields=[
+                (
+                    "created_at",
+                    lamindb.base.fields.DateTimeField(
+                        blank=True,
+                        db_default=django.db.models.functions.datetime.Now(),
+                        db_index=True,
+                        editable=False,
+                    ),
+                ),
+                ("id", models.BigAutoField(primary_key=True, serialize=False)),
+                (
+                    "feature",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        default=None,
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="links_projectrecord",
+                        to="lamindb.feature",
+                    ),
+                ),
+                (
+                    "project",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="links_record",
+                        to="lamindb.project",
+                    ),
+                ),
+                (
+                    "record",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="links_project",
+                        to="lamindb.record",
+                    ),
+                ),
+            ],
+            bases=(lamindb.models.sqlrecord.IsLink, models.Model),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="records",
+            field=models.ManyToManyField(
+                related_name="projects",
+                through="lamindb.ProjectRecord",
+                to="lamindb.record",
+            ),
+        ),
         migrations.AddField(
             model_name="artifactrecord",
             name="record",
@@ -1651,7 +1707,7 @@ class Migration(migrations.Migration):
                     lamindb.base.fields.ForeignKey(
                         blank=True,
                         on_delete=django.db.models.deletion.PROTECT,
-                        related_name="links_record",
+                        related_name="links_in_record",
                         to="lamindb.project",
                     ),
                 ),
@@ -1663,7 +1719,7 @@ class Migration(migrations.Migration):
         ),
         migrations.AddField(
             model_name="project",
-            name="records",
+            name="linked_in_records",
             field=models.ManyToManyField(
                 related_name="linked_projects",
                 through="lamindb.RecordProject",
@@ -2198,6 +2254,18 @@ class Migration(migrations.Migration):
                 to="lamindb.run",
             ),
         ),
+        migrations.AddField(
+            model_name="projectrecord",
+            name="run",
+            field=lamindb.base.fields.ForeignKey(
+                blank=True,
+                default=lamindb.models.run.current_run,
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="+",
+                to="lamindb.run",
+            ),
+        ),
         migrations.AddField(
             model_name="project",
             name="run",
@@ -4174,6 +4242,18 @@ class Migration(migrations.Migration):
                 to="lamindb.user",
             ),
         ),
+        migrations.AddField(
+            model_name="projectrecord",
+            name="created_by",
+            field=lamindb.base.fields.ForeignKey(
+                blank=True,
+                default=lamindb.base.users.current_user_id,
+                editable=False,
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="+",
+                to="lamindb.user",
+            ),
+        ),
         migrations.AddField(
             model_name="project",
             name="created_by",
@@ -4427,6 +4507,10 @@ class Migration(migrations.Migration):
             name="runfeaturevalue",
             unique_together={("run", "featurevalue")},
         ),
+        migrations.AlterUniqueTogether(
+            name="projectrecord",
+            unique_together={("record", "project", "feature")},
+        ),
         migrations.AlterUniqueTogether(
             name="personproject",
             unique_together={("person", "project")},

lamindb/models/artifact.py

@@ -542,6 +542,7 @@ def log_storage_hint(
 def data_is_scversedatastructure(
     data: ScverseDataStructures | UPathStr,
     structure_type: Literal["AnnData", "MuData", "SpatialData"] | None = None,
+    cloud_warning: bool = True,
 ) -> bool:
     """Determine whether a specific in-memory object or a UPathstr is any or a specific scverse data structure."""
     file_suffix = None
@@ -580,7 +581,7 @@ def data_is_scversedatastructure(
                     )
                     == data_type
                 )
-            else:
+            elif cloud_warning:
                 logger.warning(
                     f"we do not check whether cloud zarr is {structure_type}"
                 )
@@ -600,6 +601,7 @@ def data_is_soma_experiment(data: SOMAExperiment | UPathStr) -> bool:
 def _check_otype_artifact(
     data: UPathStr | pd.DataFrame | ScverseDataStructures,
     otype: str | None = None,
+    cloud_warning: bool = True,
 ) -> str:
     if otype is None:
         if isinstance(data, pd.DataFrame):
@@ -608,15 +610,15 @@ def _check_otype_artifact(
             return otype
 
         data_is_path = isinstance(data, (str, Path))
-        if data_is_scversedatastructure(data, "AnnData"):
+        if data_is_scversedatastructure(data, "AnnData", cloud_warning):
             if not data_is_path:
                 logger.warning("data is an AnnData, please use .from_anndata()")
             otype = "AnnData"
-        elif data_is_scversedatastructure(data, "MuData"):
+        elif data_is_scversedatastructure(data, "MuData", cloud_warning):
             if not data_is_path:
                 logger.warning("data is a MuData, please use .from_mudata()")
             otype = "MuData"
-        elif data_is_scversedatastructure(data, "SpatialData"):
+        elif data_is_scversedatastructure(data, "SpatialData", cloud_warning):
             if not data_is_path:
                 logger.warning("data is a SpatialData, please use .from_spatialdata()")
             otype = "SpatialData"
@@ -1424,7 +1426,9 @@ class Artifact(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
             # issue in Groovy / nf-lamin producing malformed S3 paths
             # https://laminlabs.slack.com/archives/C08J590666Q/p1751315027830849?thread_ts=1751039961.479259&cid=C08J590666Q
             data = data.replace("s3:///", "s3://")
-        otype = _check_otype_artifact(data=data, otype=otype)
+        otype = _check_otype_artifact(
+            data=data, otype=otype, cloud_warning=not _is_internal_call
+        )
         if "type" in kwargs:
             logger.warning("`type` will be removed soon, please use `kind`")
             kind = kwargs.pop("type")
@@ -2285,17 +2289,19 @@ class Artifact(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
     ):
         """Open a dataset for streaming.
 
-        Works for `AnnData` (`.h5ad` and `.zarr`), generic `hdf5` and `zarr`,
-        `tiledbsoma` objects (`.tiledbsoma`), `pyarrow` or `polars` compatible formats
+        Works for `AnnData` (`.h5ad` and `.zarr`), `SpatialData` (`.zarr`),
+        generic `hdf5` and `zarr`, `tiledbsoma` objects (`.tiledbsoma`),
+        `pyarrow` or `polars` compatible formats
         (`.parquet`, `.csv`, `.ipc` etc. files or directories with such files).
 
         Args:
-            mode: can only be `"w"` (write mode) for `tiledbsoma` stores,
+            mode: can be `"r"` or `"w"` (write mode) for `tiledbsoma` stores,
+                `"r"` or `"r+"` for `AnnData` or `SpatialData` `zarr` stores,
                 otherwise should be always `"r"` (read-only mode).
             engine: Which module to use for lazy loading of a dataframe
                 from `pyarrow` or `polars` compatible formats.
                 This has no effect if the artifact is not a dataframe, i.e.
-                if it is an `AnnData,` `hdf5`, `zarr` or `tiledbsoma` object.
+                if it is an `AnnData,` `hdf5`, `zarr`, `tiledbsoma` object etc.
             is_run_input: Whether to track this artifact as run input.
             **kwargs: Keyword arguments for the accessor, i.e. `h5py` or `zarr` connection,
                 `pyarrow.dataset.dataset`, `polars.scan_*` function.
@@ -2339,7 +2345,8 @@ class Artifact(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
                 s + ".gz" for s in PYARROW_SUFFIXES
             )  # this doesn't work for externally gzipped files, REMOVE LATER
         )
-        if self.suffix not in suffixes:
+        suffix = self.suffix
+        if suffix not in suffixes:
             raise ValueError(
                 "Artifact should have a zarr, h5, tiledbsoma object"
                 " or a compatible `pyarrow.dataset.dataset` or `polars.scan_*` directory"
@@ -2348,23 +2355,28 @@ class Artifact(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
                 f" Or no suffix for a folder with {', '.join(df_suffixes)} files"
                 " (no mixing allowed)."
             )
-        if self.suffix != ".tiledbsoma" and self.key != "soma" and mode != "r":
-            raise ValueError(
-                "Only a tiledbsoma store can be openened with `mode!='r'`."
-            )
-
         using_key = settings._using_key
         filepath, cache_key = filepath_cache_key_from_artifact(
             self, using_key=using_key
         )
+
         is_tiledbsoma_w = (
-            filepath.name == "soma" or self.suffix == ".tiledbsoma"
+            filepath.name == "soma" or suffix == ".tiledbsoma"
         ) and mode == "w"
+        is_zarr_w = suffix == ".zarr" and mode == "r+"
+
+        if mode != "r" and not (is_tiledbsoma_w or is_zarr_w):
+            raise ValueError(
+                f"It is not allowed to open a {suffix} object with mode='{mode}'. "
+                "You can open all supported formats with mode='r', "
+                "a tiledbsoma store with mode='w', "
+                "AnnData or SpatialData zarr store with mode='r+'."
+            )
         # consider the case where an object is already locally cached
         localpath = setup_settings.paths.cloud_to_local_no_update(
             filepath, cache_key=cache_key
         )
-        if is_tiledbsoma_w:
+        if is_tiledbsoma_w or is_zarr_w:
             open_cache = False
         else:
             open_cache = not isinstance(
@@ -2395,9 +2407,7 @@ class Artifact(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
                 else:
                     localpath.unlink(missing_ok=True)
         else:
-            access = backed_access(
-                filepath, mode, engine, using_key=using_key, **kwargs
-            )
+            access = backed_access(self, mode, engine, using_key=using_key, **kwargs)
             if is_tiledbsoma_w:
 
                 def finalize():
@@ -2413,6 +2423,7 @@ class Artifact(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
                         new_version = Artifact(
                             filepath, revises=self, _is_internal_call=True
                         ).save()
+                        # note: sets _state.db = "default"
                         init_self_from_db(self, new_version)
 
                         if localpath != filepath and localpath.exists():

lamindb/models/project.py

@@ -286,10 +286,14 @@ class Project(SQLRecord, CanCurate, TracksRun, TracksUpdates, ValidateFields):
         Schema, through="SchemaProject", related_name="projects"
     )
     """Linked schemas."""
-    records: Record = models.ManyToManyField(
+    linked_in_records: Record = models.ManyToManyField(
         Record, through="RecordProject", related_name="linked_projects"
     )
     """Linked records."""
+    records: Record = models.ManyToManyField(
+        Record, through="ProjectRecord", related_name="projects"
+    )
+    """Annotated record."""
     collections: Collection = models.ManyToManyField(
         Collection, through="CollectionProject", related_name="projects"
     )
@@ -440,11 +444,29 @@ class RecordReference(BaseSQLRecord, IsLink):
         unique_together = ("record", "feature", "value")
 
 
+# for annotation of records with projects, RecordProject is for storing project values
+class ProjectRecord(BaseSQLRecord, IsLink, TracksRun):
+    id: int = models.BigAutoField(primary_key=True)
+    record: Record = ForeignKey(Record, CASCADE, related_name="links_project")
+    project: Project = ForeignKey(Project, PROTECT, related_name="links_record")
+    feature: Feature | None = ForeignKey(
+        Feature,
+        PROTECT,
+        null=True,
+        default=None,
+        related_name="links_projectrecord",
+    )
+
+    class Meta:
+        # can have the same label linked to the same artifact if the feature is different
+        unique_together = ("record", "project", "feature")
+
+
 class RecordProject(BaseSQLRecord, IsLink):
     id: int = models.BigAutoField(primary_key=True)
     record: Record = ForeignKey(Record, CASCADE, related_name="values_project")
     feature: Feature = ForeignKey(Feature, PROTECT, related_name="links_recordproject")
-    value: Project = ForeignKey(Project, PROTECT, related_name="links_record")
+    value: Project = ForeignKey(Project, PROTECT, related_name="links_in_record")
 
     class Meta:
         unique_together = ("record", "feature", "value")

{lamindb-1.10.1.dist-info → lamindb-1.10.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: lamindb
-Version: 1.10.1
+Version: 1.10.2
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10,<3.14
@@ -9,6 +9,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+License-File: LICENSE
 Requires-Dist: lamin_utils==0.15.0
 Requires-Dist: lamin_cli==1.6.1
 Requires-Dist: lamindb_setup[aws]==1.9.1

{lamindb-1.10.1.dist-info → lamindb-1.10.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-lamindb/__init__.py,sha256=cWTeR7mqoNI5I8jjkRldYjJOqmaCsy7n8wCvbbWbeNI,2905
+lamindb/__init__.py,sha256=pY7V83g2bBifDvpSqVwvAWDmkIBDXUh_fBMIG70o7Zo,2905
 lamindb/_finish.py,sha256=3HVKRw27rQs_S2ior-JX2IU0vUkeG5a6p8XgEgdT8-U,21219
 lamindb/_tracked.py,sha256=-wK7BJv30nf4v2_nH5qDCyxHvug7ih6duQNGxDrj3UE,4447
 lamindb/_view.py,sha256=cod1RnZoLyzMVJcjWjytg78Sf4qsR8IAdqpwzsi8FTw,4950
@@ -14,21 +14,21 @@ lamindb/core/__init__.py,sha256=I9F-GugBMZwFLpUPb1MXyLfccIVAj021Gb_00h_18MY,633
 lamindb/core/_compat.py,sha256=NLnKk1qk4xdgMV-QwFDnBnbio02ujjlF86icvhpdv4c,2029
 lamindb/core/_context.py,sha256=5edP0M5_m8zgNXu7-1XaRLOv20i5LaWHBkEvMHsSiB0,40376
 lamindb/core/_mapped_collection.py,sha256=osquwC6ee0wJ_I6O-8AZwnQUa_r9zqa0MN82Q-nBI3Y,25746
-lamindb/core/_settings.py,sha256=Dj44Xh5x3OEpTPvG0twREcIv6X6Er5NzDZO86i8Cq9g,8678
+lamindb/core/_settings.py,sha256=urrw4PtH6XR5MXyziggkzdCHMG2qUI51ZF6ISFOtA7w,10990
 lamindb/core/_sync_git.py,sha256=Z7keuyS5X7CAj285sEbZIFExZF9mtjGH8DzKwz3xhHw,5881
 lamindb/core/_track_environment.py,sha256=fa0-qKEe0BpL79_nsDUDtbg1iA3VpJTh0RCOGdc2XOA,974
 lamindb/core/exceptions.py,sha256=FMEoSvT3FvtLkxQAt2oDXPeaPem8V5x5UBbTsPFYU5w,53
 lamindb/core/loaders.py,sha256=QH3r3Q_aPrbkgiWaV30TmxcLCs6zZFQLRW442DsTrNU,5456
 lamindb/core/types.py,sha256=_u8amXATAZN-nNGNWYGmyqYDcHbT_i0NZeLRhm_-ygI,378
 lamindb/core/storage/__init__.py,sha256=2gJyn9w6rzv3oPHjY756OfQYrLEXb37YuKMqh6ZjbF8,542
-lamindb/core/storage/_anndata_accessor.py,sha256=NOAqI-SHS8IwM0MwsZmRUtqG8r4tBTJ9T8888lhvQkw,26611
-lamindb/core/storage/_backed_access.py,sha256=5y3C_D1g89d831FAXr8-WashmmL3cdNfTR5s61vrMdA,7858
+lamindb/core/storage/_anndata_accessor.py,sha256=0u6U7wNUc__yb2addm1Isp_ITEABuy-YHrGSF3oFz7g,29092
+lamindb/core/storage/_backed_access.py,sha256=f5BOAEFSfiFQcZQiok-vTpSUqF6BbSyF6UKkYLC_chc,8077
 lamindb/core/storage/_polars_lazy_df.py,sha256=_JhnU8RmoTzU2kp7kyHRSUTAkFhpCJo6NNnHxvBXQv4,2929
 lamindb/core/storage/_pyarrow_dataset.py,sha256=lRYYt7edUtwauhxd7RwFud6YPDbz2PFvYYgqLhfapfk,1398
-lamindb/core/storage/_spatialdata_accessor.py,sha256=e5jqRMWAb2zA0srHOTpgmUdFZk54g6gVEbkEBqq6Ndk,1446
+lamindb/core/storage/_spatialdata_accessor.py,sha256=mn3iiNIBixGpBJoMmkHCvcoUut7Ttwy9QxGK7P5ywNo,1718
 lamindb/core/storage/_tiledbsoma.py,sha256=EdzgGVTamB7Ef31doGFWwIOhTd9TNrBfNUplDWSp6wo,11386
 lamindb/core/storage/_valid_suffixes.py,sha256=vUSeQ4s01rdhD_vSd6wKmFBsgMJAKkBMnL_T9Y1znMg,501
-lamindb/core/storage/_zarr.py,sha256=PncgrnA3XX2D01AQsTatCImEpbJUriEo00LHmARCkIE,4265
+lamindb/core/storage/_zarr.py,sha256=VHT01D_QktmZB4r92XSe8FYveoPkFtn_8XAd3hV1GSA,4433
 lamindb/core/storage/objects.py,sha256=ISVjBuXPQENZ2XVQDvfX-HZSyDjQi_OGnoJXQmI---Y,3282
 lamindb/core/storage/paths.py,sha256=C6qAFtWFvFA2RtA9M4KGxT87wmZWqB9b1V5jOCY0ALc,7141
 lamindb/core/subsettings/__init__.py,sha256=f_vOqZOjVGez8pLmtrUuc_ayDGXl07t_ZY-P2Cedxbo,201
@@ -36,7 +36,7 @@ lamindb/core/subsettings/_annotation_settings.py,sha256=o-yTYw-NmjFmtehbKU8qnf7t
 lamindb/core/subsettings/_creation_settings.py,sha256=NGHWKqCFSzVNBxAr2VnmdYguiFdW29XUK7T9wRsVshg,906
 lamindb/curators/__init__.py,sha256=WLnaVxrhQGZxGB3pjg-SM4oUu6DaKA78S_J3BfVKLEg,496
 lamindb/curators/_legacy.py,sha256=V2zF1J7jN2Ry7ascH06JytrAQzeOiirg7e4a9d0hpvM,54768
-lamindb/curators/core.py,sha256=OfCSLw6Ngu_HmCSjwRUZrsVwcdq-ge9JL1Wfl0djiyM,68832
+lamindb/curators/core.py,sha256=2ZzZG8Xyg3N7UdKrQwhgFjtrd2MSz2fKJ0D3WsInD2s,68838
 lamindb/examples/__init__.py,sha256=f0pBxijIA26ULUBnsP2sa1e4CLqMTeUUEqNeINJIf9o,179
 lamindb/examples/cellxgene/__init__.py,sha256=Xzxfi_NQcWdK-RrbNFdlIFQFVPG8Qy18ekYx3sOQZeM,161
 lamindb/examples/cellxgene/_cellxgene.py,sha256=HP_skQJbNYDEYhiLUpaP9kfmtPivmgTuGyoWNLs07nw,8844
@@ -105,7 +105,9 @@ lamindb/migrations/0114_alter_run__status_code.py,sha256=KkGecSBJElA3LBnhSK5_rFp
 lamindb/migrations/0115_alter_space_uid.py,sha256=18fCP8d31Ox1KxSSmfzU-W3lSpS3xtiaBNbPeHQiuTM,1332
 lamindb/migrations/0116_remove_artifact_unique_artifact_storage_key_hash_and_more.py,sha256=vCAeIg8TdtVaC2kZIq3KTY5hcz_RiXh_44Sczv75RA4,1562
 lamindb/migrations/0117_fix_artifact_storage_hash_unique_constraints.py,sha256=OqgUmf9_TeNwTddwcwZdJYfpN6cpFYBaw5_KitYumNM,1033
-lamindb/migrations/0117_squashed.py,sha256=i9rdXIVWkgNaZTaaFs-Ozi_0CC-pKHOdF2ZM37M4fOY,163772
+lamindb/migrations/0118_alter_recordproject_value_projectrecord.py,sha256=aNC_o3xfH9wk1BTbOWKbXeV4r_IPXgytYBRTFi_U-MM,3493
+lamindb/migrations/0119_rename_records_project_linked_in_records.py,sha256=Feh4rCfoGD4kiGG-sk_IQx7cplDn-yVIlzI5FzE8utI,688
+lamindb/migrations/0119_squashed.py,sha256=s4qKQa6EMR3fontNSSd18LsamK0HkfqPJtolwqWcAVI,166916
 lamindb/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lamindb/models/__init__.py,sha256=mtjZH0x31aV463YaUef8ZvdQHlGa6SZr_thsrlbdkCg,2419
 lamindb/models/_describe.py,sha256=OD-MokYrB6mfzkmFdTkRj0J93LyqxFOgJi_5LvOXr1A,10915
@@ -115,13 +117,13 @@ lamindb/models/_from_values.py,sha256=cCGPMDlAbBrdhFW-XrIQVZ10q1LNg4MycYPLOkF0fT
 lamindb/models/_is_versioned.py,sha256=Th2_cBf9UWh27E6ANxg6LGmjBOumXFy7AjH0GG4FoXA,7601
 lamindb/models/_label_manager.py,sha256=O3KaaTEVId5ky3h0aCGg1kDoFFfrovFYlV82YsZZyIs,12127
 lamindb/models/_relations.py,sha256=zHYLujy9vkuB9jVq5844TpzLSP__iYNCQcsl-FzK1Jw,3700
-lamindb/models/artifact.py,sha256=yBNgOJTokGKy5EUDzugzs1bXBObGKSqDLJ2nZLk6M3s,117610
+lamindb/models/artifact.py,sha256=NZtF9x7z3O7pSaCGQwA9V-lnoSvaHfusEMkuWjxora4,118203
 lamindb/models/artifact_set.py,sha256=VOZEGDo3m_9Yg_ftx3I2fwdydjHN61X_qV18N6xG4kM,4117
 lamindb/models/can_curate.py,sha256=_6ymHhtc9zpU8O6ozqNDqn7jh87C9JisU_dl2gH2Hpo,29329
 lamindb/models/collection.py,sha256=ZMBaDqQtil9yWqDC_twKbavGkdAulcu9j2WlfHADxn0,28358
 lamindb/models/feature.py,sha256=eIskKsSBkwW6ftfOZ_ngeqD9Pi6Y70cY_m3WHTCXNU8,38031
 lamindb/models/has_parents.py,sha256=NRNshrWCX7G3nnM3lnnHQ3Ho216T3EJfgakY6KlTvt8,20301
-lamindb/models/project.py,sha256=4w2rMZMPrLmvirvJQanhKfqwGqEnpY42iaOWi99YPw0,17345
+lamindb/models/project.py,sha256=AzfkZsMIUtTM2QGz_jhS4F5zt53V-j2l4Sz-b6NvW34,18192
 lamindb/models/query_manager.py,sha256=EzbyNA5zWUbLYH5yJ7dIC90j1teVoQHrXpRLjCfBEao,11036
 lamindb/models/query_set.py,sha256=d27m8UF8QZAzHZBEVE1QdJRtx9wCzYgkDdHGCATBM48,34815
 lamindb/models/record.py,sha256=WW6iuQT8M4KHcpZoYGSTjND10hvDUIff7DOkyD5d2Fc,12070
@@ -137,7 +139,7 @@ lamindb/setup/_switch.py,sha256=njZJN__JOhVrBFGClQG1wobdhJJp6l_XzPGKtKSCrfU,434
 lamindb/setup/core/__init__.py,sha256=SevlVrc2AZWL3uALbE5sopxBnIZPWZ1IB0NBDudiAL8,167
 lamindb/setup/errors/__init__.py,sha256=bAHTxOUJW1rm4zpF0Pvqkftn8W6iMGnQ-uyNBu13Nfg,171
 lamindb/setup/types/__init__.py,sha256=ATaosOi6q-cDWB52T69_sRmLMqj8cHfc-vljzZsrJNw,169
-lamindb-1.10.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamindb-1.10.1.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-lamindb-1.10.1.dist-info/METADATA,sha256=UYl8nK7xlSZg0L4GaKYNkMjXuNs36LvfgSJFq-StSho,5199
-lamindb-1.10.1.dist-info/RECORD,,
+lamindb-1.10.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamindb-1.10.2.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+lamindb-1.10.2.dist-info/METADATA,sha256=n3KlfwBGbuxGLNq-qpfSWxHaU7ZmEhjsHwfrvHrBchk,5221
+lamindb-1.10.2.dist-info/RECORD,,

{lamindb-1.10.1.dist-info → lamindb-1.10.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: flit 3.10.1
+Generator: flit 3.12.0
 Root-Is-Purelib: true
 Tag: py3-none-any