lamindb 0.70.2__tar.gz → 0.70.3__tar.gz

{lamindb-0.70.2 → lamindb-0.70.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.70.2
+Version: 0.70.3
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Requires-Dist: lnschema_core==0.65.1
-Requires-Dist: lamindb_setup==0.69.4
+Requires-Dist: lamindb_setup==0.69.5
 Requires-Dist: lamin_utils==0.13.2
 Requires-Dist: lamin_cli==0.12.3
 Requires-Dist: rapidfuzz

{lamindb-0.70.2 → lamindb-0.70.3}/docs/changelog.md

@@ -14,6 +14,12 @@ When upgrading your LaminDB installation to a new `minor` version in `major.mino
 
 ## 0.70
 
+### 0.70.3 {small}`2024-04-22`
+
+- 🚸 Update metadata like description upon re-running [PR](https://github.com/laminlabs/lamindb/pull/1588) [@falexwolf](https://github.com/falexwolf)
+- 🐛 Fix detection of AnnData in zarr and h5ad, refactor directory upload [PR](https://github.com/laminlabs/lamindb/pull/1587) [@Koncopd](https://github.com/Koncopd)
+- 🚸 Raise error if transforms of type notebook or script are passed manually [PR](https://github.com/laminlabs/lamindb/pull/1584) [@falexwolf](https://github.com/falexwolf)
+
 ### 0.70.2 {small}`2024-04-19`
 
 - ♻️ In Vitessce integration, separate `VitessceConfig` from its referenced artifacts [PR](https://github.com/laminlabs/lamindb/pull/1582) [@falexwolf](https://github.com/falexwolf)

{lamindb-0.70.2 → lamindb-0.70.3}/lamindb/__init__.py

@@ -40,7 +40,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.70.2"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.70.3"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 

{lamindb-0.70.2 → lamindb-0.70.3}/lamindb/_artifact.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from pathlib import Path, PurePath, PurePosixPath
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Mapping
 
 import fsspec
 import lamindb_setup as ln_setup
@@ -26,7 +26,7 @@ from lnschema_core.types import (
 )
 
 from lamindb._utils import attach_func_to_class_method
-from lamindb.core._data import _track_run_input
+from lamindb.core._data import Data, _track_run_input
 from lamindb.core._settings import settings
 from lamindb.core.storage import (
     LocalPathClasses,
@@ -53,6 +53,14 @@ from .core._data import (
 from .core.storage.objects import _mudata_is_installed
 from .core.storage.paths import AUTO_KEY_PREFIX
 
+try:
+    from .core.storage._zarr import zarr_is_adata
+except ImportError:
+
+    def zarr_is_adata(storepath):  # type: ignore
+        raise ImportError("Please install zarr: pip install zarr")
+
+
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
     from mudata import MuData
@@ -329,16 +337,17 @@ def get_artifact_kwargs_from_data(
         using_key=using_key,
     )
     if isinstance(stat_or_artifact, Artifact):
+        artifact = stat_or_artifact
         # update the run of the existing artifact
         if run is not None:
             # save the information that this artifact was previously
             # produced by another run
-            if stat_or_artifact.run is not None:
-                stat_or_artifact.run.replicated_output_artifacts.add(stat_or_artifact)
+            if artifact.run is not None:
+                artifact.run.replicated_output_artifacts.add(artifact)
             # update the run of the artifact with the latest run
             stat_or_artifact.run = run
             stat_or_artifact.transform = run.transform
-        return stat_or_artifact, None
+        return artifact, None
     else:
         size, hash, hash_type, n_objects = stat_or_artifact
 
@@ -446,12 +455,19 @@ def data_is_anndata(data: AnnData | UPathStr):
     if isinstance(data, AnnData):
         return True
     if isinstance(data, (str, Path, UPath)):
-        if Path(data).suffix == ".h5ad":
+        data_path = UPath(data)
+        if data_path.suffix == ".h5ad":
             return True
-        elif Path(data).suffix == ".zarr":
-            raise NotImplementedError(
-                "auto-detecting AnnData from Zarr is not yet supported"
-            )
+        elif data_path.suffix == ".zarr":
+            # ".anndata.zarr" is a valid suffix (core.storage._valid_suffixes)
+            if ".anndata" in data_path.suffixes:
+                return True
+            # check only for local, expensive for cloud
+            if fsspec.utils.get_protocol(data_path) == "file":
+                return zarr_is_adata(data_path)
+            else:
+                logger.warning("We do not check if cloud zarr is AnnData or not.")
+                return False
     return False
 
 
@@ -461,27 +477,39 @@ def data_is_mudata(data: MuData | UPathStr):
 
         if isinstance(data, MuData):
             return True
-    if isinstance(data, (str, Path, UPath)):
-        return Path(data).suffix in {".h5mu"}
+    if isinstance(data, (str, Path)):
+        return UPath(data).suffix in {".h5mu"}
     return False
 
 
 def _check_accessor_artifact(data: Any, accessor: str | None = None):
-    if accessor is None and not isinstance(data, (str, Path, UPath)):
+    if accessor is None:
         if isinstance(data, pd.DataFrame):
             logger.warning("data is a DataFrame, please use .from_df()")
             accessor = "DataFrame"
-        elif data_is_anndata(data):
-            logger.warning("data is an AnnData, please use .from_anndata()")
+            return accessor
+
+        data_is_path = isinstance(data, (str, Path))
+        if data_is_anndata(data):
+            if not data_is_path:
+                logger.warning("data is an AnnData, please use .from_anndata()")
             accessor = "AnnData"
         elif data_is_mudata(data):
-            logger.warning("data is a MuData, please use .from_mudata()")
+            if not data_is_path:
+                logger.warning("data is a MuData, please use .from_mudata()")
             accessor = "MuData"
-        else:
+        elif not data_is_path:  # UPath is a subclass of Path
             raise TypeError("data has to be a string, Path, UPath")
     return accessor
 
 
+def update_attributes(data: Data, attributes: Mapping[str, str]):
+    for key, value in attributes.items():
+        if getattr(data, key) != value:
+            logger.warning(f"updated {key} from {getattr(data, key)} to {value}")
+            setattr(data, key, value)
+
+
 def __init__(artifact: Artifact, *args, **kwargs):
     # Below checks for the Django-internal call in from_db()
     # it'd be better if we could avoid this, but not being able to create a Artifact
@@ -558,8 +586,13 @@ def __init__(artifact: Artifact, *args, **kwargs):
     if isinstance(kwargs_or_artifact, Artifact):
         from ._registry import init_self_from_db
 
-        # kwargs_or_artifact is an existing file
         init_self_from_db(artifact, kwargs_or_artifact)
+        # adding "key" here is dangerous because key might be auto-populated
+        update_attributes(artifact, {"description": description})
+        if artifact.key != key:
+            logger.warning(
+                f"key {artifact.key} on existing artifact differs from passed key {key}"
+            )
         return None
     else:
         kwargs = kwargs_or_artifact

{lamindb-0.70.2 → lamindb-0.70.3}/lamindb/_collection.py

@@ -18,13 +18,13 @@ from lamindb_setup.core.hashing import hash_set
 from lnschema_core.models import Collection, CollectionArtifact, FeatureSet
 from lnschema_core.types import VisibilityChoice
 
+from lamindb._artifact import update_attributes
 from lamindb._utils import attach_func_to_class_method
 from lamindb.core._data import _track_run_input
 from lamindb.core._mapped_collection import MappedCollection
 from lamindb.core.versioning import get_uid_from_old_version, init_uid
 
 from . import Artifact, Run
-from ._artifact import data_is_anndata
 from ._registry import init_self_from_db
 from .core._data import (
     add_transform_to_kwargs,
@@ -128,6 +128,7 @@ def __init__(
             existing_collection.run = run
             existing_collection.transform = run.transform
         init_self_from_db(collection, existing_collection)
+        update_attributes(collection, {"description": description, "name": name})
         for slot, feature_set in collection.features._feature_set_by_slot.items():
             if slot in feature_sets:
                 if not feature_sets[slot] == feature_set:

{lamindb-0.70.2 → lamindb-0.70.3}/lamindb/core/_run_context.py

@@ -306,6 +306,11 @@ class run_context:
             if not is_tracked:
                 raise_transform_settings_error()
         else:
+            if transform.type in {"notebook", "script"}:
+                raise ValueError(
+                    "Use ln.track() without passing transform in a notebook or script"
+                    " - metadata is automatically parsed"
+                )
             transform_exists = None
             if transform.id is not None:
                 # transform has an id but unclear whether already saved

{lamindb-0.70.2 → lamindb-0.70.3}/lamindb/core/storage/_zarr.py

@@ -7,28 +7,44 @@ import scipy.sparse as sparse
 import zarr
 from anndata._io import read_zarr
 from anndata._io.specs import write_elem
+from anndata._io.specs.registry import get_spec
+from fsspec.implementations.local import LocalFileSystem
 from lamindb_setup.core.upath import create_mapper, infer_filesystem
 
 from ._anndata_sizes import _size_elem, _size_raw, size_adata
 
 if TYPE_CHECKING:
     from anndata import AnnData
-
-
-def read_adata_zarr(storepath) -> AnnData:
-    fs, storepath = infer_filesystem(storepath)
-    store = create_mapper(fs, storepath, check=True)
-
-    adata = read_zarr(store)
-
+    from lamindb_setup.core.types import UPathStr
+
+
+def zarr_is_adata(storepath: UPathStr) -> bool:
+    fs, storepath_str = infer_filesystem(storepath)
+    if isinstance(fs, LocalFileSystem):
+        # this is faster than through an fsspec mapper for local
+        open_obj = storepath_str
+    else:
+        open_obj = create_mapper(fs, storepath_str, check=True)
+    storage = zarr.open(open_obj, mode="r")
+    return get_spec(storage).encoding_type == "anndata"
+
+
+def read_adata_zarr(storepath: UPathStr) -> AnnData:
+    fs, storepath_str = infer_filesystem(storepath)
+    if isinstance(fs, LocalFileSystem):
+        # this is faster than through an fsspec mapper for local
+        open_obj = storepath_str
+    else:
+        open_obj = create_mapper(fs, storepath_str, check=True)
+    adata = read_zarr(open_obj)
     return adata
 
 
 def write_adata_zarr(
-    adata: AnnData, storepath, callback=None, chunks=None, **dataset_kwargs
+    adata: AnnData, storepath: UPathStr, callback=None, chunks=None, **dataset_kwargs
 ):
-    fs, storepath = infer_filesystem(storepath)
-    store = create_mapper(fs, storepath, create=True)
+    fs, storepath_str = infer_filesystem(storepath)
+    store = create_mapper(fs, storepath_str, create=True)
 
     f = zarr.open(store, mode="w")
 

{lamindb-0.70.2 → lamindb-0.70.3}/lamindb/core/storage/paths.py

@@ -29,7 +29,7 @@ try:
     from ._zarr import read_adata_zarr
 except ImportError:
 
-    def read_adata_zarr(filepath):  # type: ignore
+    def read_adata_zarr(storepath):  # type: ignore
         raise ImportError("Please install zarr: pip install zarr")
 
 
@@ -114,7 +114,7 @@ def store_file_or_folder(local_path: UPathStr, storage_path: UPath) -> None:
     local_path = Path(local_path)
     if not isinstance(storage_path, LocalPathClasses):
         # this uploads files and directories
-        storage_path.upload_from(local_path, recursive=True, print_progress=True)
+        storage_path.upload_from(local_path, dir_inplace=True, print_progress=True)
     else:  # storage path is local
         storage_path.parent.mkdir(parents=True, exist_ok=True)
         if local_path.is_file():

{lamindb-0.70.2 → lamindb-0.70.3}/pyproject.toml

@@ -17,7 +17,7 @@ classifiers = [
 dependencies = [
     # Lamin PINNED packages
     "lnschema_core==0.65.1",
-    "lamindb_setup==0.69.4",
+    "lamindb_setup==0.69.5",
     "lamin_utils==0.13.2",
     "lamin_cli==0.12.3",
     # others

{lamindb-0.70.2 → lamindb-0.70.3}/sub/lamindb-setup/docs/changelog.md

@@ -3,6 +3,7 @@
 <!-- prettier-ignore -->
 Name | PR | Developer | Date | Version
 --- | --- | --- | --- | ---
+✨ Upload dirs inplace | [722](https://github.com/laminlabs/lamindb-setup/pull/722) | [Koncopd](https://github.com/Koncopd) | 2024-04-20 | 0.69.5
 ♻️ Do not always read from local cache | [720](https://github.com/laminlabs/lamindb-setup/pull/720) | [falexwolf](https://github.com/falexwolf) | 2024-04-19 | 0.69.4
 ♻️ Improve suffix handling | [718](https://github.com/laminlabs/lamindb-setup/pull/718) | [falexwolf](https://github.com/falexwolf) | 2024-04-19 | 0.69.3
 ✨ Enable hybrid storage mode | [717](https://github.com/laminlabs/lamindb-setup/pull/717) | [falexwolf](https://github.com/falexwolf) | 2024-04-16 | 0.69.1

{lamindb-0.70.2 → lamindb-0.70.3}/sub/lamindb-setup/docs/hub-cloud/06-connect-hosted-instance.ipynb

@@ -88,7 +88,23 @@
     "target_dir = root / \"test-dir-upload\"\n",
     "target_dir.upload_from(test_dir, recursive=True)\n",
     "\n",
-    "assert target_dir.is_dir()"
+    "assert target_dir.is_dir()\n",
+    "assert (target_dir / \"test-dir-upload\").exists()\n",
+    "\n",
+    "target_dir.rmdir()\n",
+    "assert not target_dir.exists()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "target_dir.upload_from(test_dir, dir_inplace=True)\n",
+    "\n",
+    "assert target_dir.is_dir()\n",
+    "assert not (target_dir / \"test-dir-upload\").exists()"
    ]
   },
   {

{lamindb-0.70.2 → lamindb-0.70.3}/sub/lamindb-setup/lamindb_setup/__init__.py

@@ -36,7 +36,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.69.4"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.69.5"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

{lamindb-0.70.2 → lamindb-0.70.3}/sub/lamindb-setup/lamindb_setup/core/upath.py

@@ -165,7 +165,7 @@ class ProgressCallback(fsspec.callbacks.Callback):
         return None
 
 
-def download_to(self, path, print_progress: bool = False, **kwargs):
+def download_to(self, path: UPathStr, print_progress: bool = False, **kwargs):
     """Download to a path."""
     if print_progress:
         # can't do path.is_dir() because path doesn't exist
@@ -180,20 +180,38 @@ def download_to(self, path, print_progress: bool = False, **kwargs):
     self.fs.download(str(self), str(path), **kwargs)
 
 
-def upload_from(self, path, print_progress: bool = False, **kwargs):
+def upload_from(
+    self,
+    path: UPathStr,
+    dir_inplace: bool = False,
+    print_progress: bool = False,
+    **kwargs,
+):
     """Upload from a local path."""
+    path_is_dir = os.path.isdir(path)
+    if not path_is_dir:
+        dir_inplace = False
+
     if print_progress:
-        if not os.path.isdir(path):
+        if not path_is_dir:
             cb = ProgressCallback("uploading")
         else:
             # todo: make proper progress bar for directories
             cb = fsspec.callbacks.NoOpCallback()
         kwargs["callback"] = cb
+
+    if dir_inplace:
+        path = Path(path)
+        source = [f for f in path.rglob("*") if f.is_file()]
+        destination = [str(self / f.relative_to(path)) for f in source]
+        source = [str(f) for f in source]  # type: ignore
+    else:
+        source = str(path)  # type: ignore
+        destination = str(self)  # type: ignore
     # this weird thing is to avoid s3fs triggering create_bucket in upload
     # if dirs are present
     # it allows to avoid permission error
-    destination = str(self)
-    if self.protocol != "s3" or os.path.isfile(path):
+    if self.protocol != "s3" or not path_is_dir or dir_inplace:
         cleanup_cache = False
     else:
         bucket = self._url.netloc
@@ -205,7 +223,7 @@ def upload_from(self, path, print_progress: bool = False, **kwargs):
         else:
             cleanup_cache = False
 
-    self.fs.upload(str(path), destination, **kwargs)
+    self.fs.upload(source, destination, **kwargs)
 
     if cleanup_cache:
         # normally this is invalidated after the upload but still better to check

{lamindb-0.70.2 → lamindb-0.70.3}/tests/test_artifact.py

@@ -12,11 +12,12 @@ import pytest
 from lamindb import _artifact
 from lamindb._artifact import (
     check_path_is_child_of_root,
+    data_is_anndata,
     get_relative_path_to_directory,
     process_data,
 )
 from lamindb.core._settings import settings
-from lamindb.core.storage._zarr import write_adata_zarr
+from lamindb.core.storage._zarr import write_adata_zarr, zarr_is_adata
 from lamindb.core.storage.paths import (
     AUTO_KEY_PREFIX,
     auto_storage_key_from_artifact_uid,
@@ -155,6 +156,13 @@ def get_test_filepaths(request):  # -> Tuple[bool, Path, Path, Path, str]
     shutil.rmtree(test_dir)
 
 
+def test_data_is_anndata_paths():
+    assert data_is_anndata("something.h5ad")
+    assert data_is_anndata("something.anndata.zarr")
+    assert data_is_anndata("s3://somewhere/something.anndata.zarr")
+    assert not data_is_anndata("s3://somewhere/something.zarr")
+
+
 def test_is_new_version_of_versioned_file(df, adata):
     # attempt to create a file with an invalid version
     with pytest.raises(ValueError) as error:
@@ -492,11 +500,18 @@ def test_from_dir_single_artifact(get_test_filepaths, key):
     assert artifact.n_objects == 3
     assert artifact.hash == hash_test_dir
     assert artifact._state.adding
+    assert artifact.description is None
+    assert artifact.path.exists()
     artifact.save()
     # now run again, because now we'll have hash-based lookup!
-    artifact = ln.Artifact(test_dirpath, key=key)
-    assert not artifact._state.adding
-    artifact.delete(permanent=True, storage=False)
+    artifact2 = ln.Artifact(test_dirpath, key=key, description="something")
+    assert not artifact2._state.adding
+    assert artifact.id == artifact2.id
+    assert artifact.uid == artifact2.uid
+    assert artifact.storage == artifact2.storage
+    assert artifact2.path.exists()
+    assert artifact2.description == "something"
+    artifact2.delete(permanent=True, storage=False)
 
 
 def test_from_dir_s3():
@@ -746,9 +761,13 @@ def test_folder_upload_cache(adata):
     write_adata_zarr(adata, zarr_path, callback)
 
     artifact = ln.Artifact(zarr_path, key="test_adata.zarr")
+    assert artifact.accessor == "AnnData"
     artifact.save()
 
     assert isinstance(artifact.path, CloudPath) and artifact.path.exists()
+    assert zarr_is_adata(artifact.path)
+
+    shutil.rmtree(artifact.cache())
 
     cache_path = settings._storage_settings.cloud_to_local_no_update(artifact.path)
     assert isinstance(artifact.load(), ad.AnnData)

{lamindb-0.70.2 → lamindb-0.70.3}/tests/test_collection.py

@@ -156,11 +156,19 @@ def test_from_consistent_artifacts(adata, adata2):
     run = ln.Run(transform)
     run.save()
     collection = ln.Collection([artifact1, artifact2], name="My test", run=run)
+    assert collection._state.adding
     collection.save()
     assert set(collection.run.input_artifacts.all()) == {artifact1, artifact2}
     adata_joined = collection.load()
     assert "artifact_uid" in adata_joined.obs.columns
     assert artifact1.uid in adata_joined.obs.artifact_uid.cat.categories
+
+    # re-run with hash-based lookup
+    collection2 = ln.Collection([artifact1, artifact2], name="My test 1", run=run)
+    assert not collection2._state.adding
+    assert collection2.id == collection.id
+    assert collection2.name == "My test 1"
+
     artifact1.delete(permanent=True, storage=True)
     artifact2.delete(permanent=True, storage=True)
     collection.delete(permanent=True)

{lamindb-0.70.2 → lamindb-0.70.3}/tests/test_run_context.py

@@ -98,3 +98,14 @@ def test_sync_git_repo():
         "https://github.com/laminlabs/lamin-cli/blob/"
     )
     assert transform.reference_type == "url"
+
+
+@pytest.mark.parametrize("type", ["notebook", "script"])
+def test_track_notebook_or_script_manually(type):
+    transform = ln.Transform(name="My notebook", type=type)
+    with pytest.raises(ValueError) as error:
+        ln.track(transform=transform)
+    assert (
+        error.exconly()
+        == "ValueError: Use ln.track() without passing transform in a notebook or script - metadata is automatically parsed"
+    )