lamindb 0.68.0__py3-none-any.whl → 0.68.2__py3-none-any.whl

lamindb/_validate.py

@@ -1,18 +1,18 @@
 from typing import Dict, Iterable, List, Literal, Optional, Set, Union
 
+import lamindb_setup as ln_setup
 import numpy as np
 import pandas as pd
 from django.core.exceptions import FieldDoesNotExist
 from django.db.models import QuerySet
 from lamin_utils import colors, logger
 from lamin_utils._inspect import InspectResult
-from lamindb_setup.dev._docs import doc_args
+from lamindb_setup.core._docs import doc_args
 from lnschema_core import CanValidate, Registry
 from lnschema_core.types import ListLike, StrField
 
 from lamindb._utils import attach_func_to_class_method
 
-from . import _TESTING
 from ._from_values import _has_organism_field, _print_values
 from ._registry import _queryset, get_default_str_field
 
@@ -468,7 +468,7 @@ METHOD_NAMES = [
     "set_abbr",
 ]
 
-if _TESTING:  # type: ignore
+if ln_setup._TESTING:  # type: ignore
     from inspect import signature
 
     SIGS = {

lamindb/core/__init__.py

@@ -43,7 +43,6 @@ Auxiliary tools:
 
    run_context
    datasets
-   hashing
    storage
    fields
    Settings

lamindb/core/_data.py

@@ -2,7 +2,7 @@ from collections import defaultdict
 from typing import Any, Dict, Iterable, List, Optional, Union
 
 from lamin_utils import colors, logger
-from lamindb_setup.dev._docs import doc_args
+from lamindb_setup.core._docs import doc_args
 from lnschema_core.models import (
     Artifact,
     Collection,

lamindb/core/_feature_manager.py

@@ -4,7 +4,7 @@ from typing import Dict, Iterable, Optional, Union
 import anndata as ad
 from anndata import AnnData
 from lamin_utils import colors, logger
-from lamindb_setup.dev.upath import create_path
+from lamindb_setup.core.upath import create_path
 from lnschema_core.models import Artifact, Collection, Data, Feature, Registry
 from lnschema_core.types import AnnDataLike, FieldAttr
 
@@ -325,7 +325,11 @@ class FeatureManager:
                 for feature in new_members:
                     # not calling save=True here as in labels, because want to
                     # bulk save below
-                    transfer_to_default_db(feature, using_key, mute=mute)
+                    # transfer_fk is set to False because they are already transferred
+                    # in the previous step transfer_fk_to_default_db_bulk
+                    transfer_to_default_db(
+                        feature, using_key, mute=mute, transfer_fk=False
+                    )
                 logger.info(
                     f"saving {new_members.count()} new {registry.__name__} records"
                 )

lamindb/core/_label_manager.py

@@ -58,7 +58,9 @@ def transfer_add_labels(labels, features_lookup_self, self, row, parents: bool =
         if len(new_labels) > 0:
             transfer_fk_to_default_db_bulk(new_labels, using_key=None)
             for label in new_labels:
-                transfer_to_default_db(label, using_key=None, mute=True)
+                transfer_to_default_db(
+                    label, using_key=None, mute=True, transfer_fk=False
+                )
             # not saving parents for Organism during transfer
             registry = new_labels[0].__class__
             logger.info(f"saving {len(new_labels)} new {registry.__name__} records")
@@ -219,7 +221,9 @@ class LabelManager:
             if len(new_labels) > 0:
                 transfer_fk_to_default_db_bulk(new_labels, using_key)
                 for label in new_labels:
-                    transfer_to_default_db(label, using_key, mute=True)
+                    transfer_to_default_db(
+                        label, using_key, mute=True, transfer_fk=False
+                    )
                 save(new_labels, parents=parents)
             # this should not occur as file and collection should have the same attributes
             # but this might not be true for custom schema

lamindb/core/_mapped_collection.py

@@ -1,12 +1,13 @@
 from collections import Counter
 from functools import reduce
-from os import PathLike
+from pathlib import Path
 from typing import Dict, List, Literal, Optional, Union
 
 import numpy as np
 import pandas as pd
 from lamin_utils import logger
-from lamindb_setup.dev.upath import UPath
+from lamindb_setup.core.types import UPathStr
+from lamindb_setup.core.upath import UPath
 
 from .storage._backed_access import (
     ArrayTypes,
@@ -52,11 +53,26 @@ class MappedCollection:
 
         A similar data loader exists `here
         <https://github.com/Genentech/scimilarity>`__.
+
+    Args:
+        path_list: A list of paths to `AnnData` objects stored in `h5ad` or `zrad` formats.
+        label_keys: Columns of the ``.obs`` slot - the names of the metadata
+            features storing labels.
+        join: `"inner"` or `"outer"` virtual joins. If ``None`` is passed,
+            does not join.
+        encode_labels: Encode labels into integers.
+            Can be a list with elements from ``label_keys```.
+        unknown_label: Encode this label to -1.
+            Can be a dictionary with keys from ``label_keys`` if ``encode_labels=True```
+            or from ``encode_labels`` if it is a list.
+        cache_categories: Enable caching categories of ``label_keys`` for faster access.
+        parallel: Enable sampling with multiple processes.
+        dtype: Convert numpy arrays from ``.X`` to this dtype on selection.
     """
 
     def __init__(
         self,
-        path_list: List[Union[str, PathLike]],
+        path_list: List[UPathStr],
         label_keys: Optional[Union[str, List[str]]] = None,
         join: Optional[Literal["inner", "outer"]] = "inner",
         encode_labels: Union[bool, List[str]] = True,
@@ -147,7 +163,7 @@ class MappedCollection:
             self._cache_cats[label] = []
             for storage in self.storages:
                 with _Connect(storage) as store:
-                    cats = self.get_categories(store, label)
+                    cats = self._get_categories(store, label)
                     if cats is not None:
                         cats = decode(cats) if isinstance(cats[0], bytes) else cats[...]
                     self._cache_cats[label].append(cats)
@@ -203,7 +219,7 @@ class MappedCollection:
             var_idxs_join = None
 
         with _Connect(self.storages[storage_idx]) as store:
-            out = {"x": self.get_data_idx(store, obs_idx, var_idxs_join)}
+            out = {"x": self._get_data_idx(store, obs_idx, var_idxs_join)}
             if self.label_keys is not None:
                 for label in self.label_keys:
                     if label in self._cache_cats:
@@ -212,13 +228,13 @@ class MappedCollection:
                             cats = []
                     else:
                         cats = None
-                    label_idx = self.get_label_idx(store, obs_idx, label, cats)
+                    label_idx = self._get_label_idx(store, obs_idx, label, cats)
                     if label in self.encoders:
                         label_idx = self.encoders[label][label_idx]
                     out[label] = label_idx
         return out
 
-    def get_data_idx(
+    def _get_data_idx(
         self,
         storage: StorageType,  # type: ignore
         idx: int,
@@ -259,7 +275,7 @@ class MappedCollection:
                     layer_idx = layer_idx[var_idxs_join]
             return layer_idx
 
-    def get_label_idx(
+    def _get_label_idx(
         self,
         storage: StorageType,
         idx: int,
@@ -280,7 +296,7 @@ class MappedCollection:
         if categories is not None:
             cats = categories
         else:
-            cats = self.get_categories(storage, label_key)
+            cats = self._get_categories(storage, label_key)
         if cats is not None and len(cats) > 0:
             label = cats[label]
         if isinstance(label, bytes):
@@ -305,17 +321,17 @@ class MappedCollection:
         return weights
 
     def get_merged_labels(self, label_key: str):
-        """Get merged labels."""
+        """Get merged labels for `label_key` from all `.obs`."""
         labels_merge = []
         decode = np.frompyfunc(lambda x: x.decode("utf-8"), 1, 1)
         for i, storage in enumerate(self.storages):
             with _Connect(storage) as store:
-                codes = self.get_codes(store, label_key)
+                codes = self._get_codes(store, label_key)
                 labels = decode(codes) if isinstance(codes[0], bytes) else codes
                 if label_key in self._cache_cats:
                     cats = self._cache_cats[label_key][i]
                 else:
-                    cats = self.get_categories(store, label_key)
+                    cats = self._get_categories(store, label_key)
                 if cats is not None:
                     cats = decode(cats) if isinstance(cats[0], bytes) else cats
                     labels = cats[labels]
@@ -323,7 +339,7 @@ class MappedCollection:
         return np.hstack(labels_merge)
 
     def get_merged_categories(self, label_key: str):
-        """Get merged categories."""
+        """Get merged categories for `label_key` from all `.obs`."""
         cats_merge = set()
         decode = np.frompyfunc(lambda x: x.decode("utf-8"), 1, 1)
         for i, storage in enumerate(self.storages):
@@ -331,17 +347,17 @@ class MappedCollection:
                 if label_key in self._cache_cats:
                     cats = self._cache_cats[label_key][i]
                 else:
-                    cats = self.get_categories(store, label_key)
+                    cats = self._get_categories(store, label_key)
                 if cats is not None:
                     cats = decode(cats) if isinstance(cats[0], bytes) else cats
                     cats_merge.update(cats)
                 else:
-                    codes = self.get_codes(store, label_key)
+                    codes = self._get_codes(store, label_key)
                     codes = decode(codes) if isinstance(codes[0], bytes) else codes
                     cats_merge.update(codes)
         return cats_merge
 
-    def get_categories(self, storage: StorageType, label_key: str):  # type: ignore
+    def _get_categories(self, storage: StorageType, label_key: str):  # type: ignore
         """Get categories."""
         obs = storage["obs"]  # type: ignore
         if isinstance(obs, ArrayTypes):  # type: ignore
@@ -370,7 +386,7 @@ class MappedCollection:
                     return None
         return None
 
-    def get_codes(self, storage: StorageType, label_key: str):  # type: ignore
+    def _get_codes(self, storage: StorageType, label_key: str):  # type: ignore
         """Get codes."""
         obs = storage["obs"]  # type: ignore
         if isinstance(obs, ArrayTypes):  # type: ignore
@@ -383,7 +399,10 @@ class MappedCollection:
                 return label["codes"][...]
 
     def close(self):
-        """Close connection to array streaming backend."""
+        """Close connections to array streaming backend.
+
+        No effect if `parallel=True`.
+        """
         for storage in self.storages:
             if hasattr(storage, "close"):
                 storage.close()
@@ -394,6 +413,10 @@ class MappedCollection:
 
     @property
     def closed(self):
+        """Check if connections to array streaming backend are closed.
+
+        Does not matter if `parallel=True`.
+        """
         return self._closed
 
     def __enter__(self):
@@ -404,6 +427,10 @@ class MappedCollection:
 
     @staticmethod
     def torch_worker_init_fn(worker_id):
+        """`worker_init_fn` for `torch.utils.data.DataLoader`.
+
+        Improves performance for `num_workers > 1`.
+        """
         from torch.utils.data import get_worker_info
 
         mapped = get_worker_info().dataset

lamindb/core/_run_context.py

@@ -9,7 +9,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
 
 from lamin_utils import logger
 from lamindb_setup import settings
-from lamindb_setup.dev import InstanceSettings
+from lamindb_setup.core import InstanceSettings
 from lnschema_core import Run, Transform, ids
 from lnschema_core.types import TransformType
 from lnschema_core.users import current_user_id
@@ -25,12 +25,6 @@ msg_path_failed = (
 )
 
 
-# we don't want a real error here, as this is so frequent
-# in VSCode
-class UpdateNbWithNonInteractiveEditor(SystemExit):
-    pass
-
-
 class NotebookNotSavedError(Exception):
     pass
 
@@ -48,19 +42,6 @@ def get_uid_ext(version: str) -> str:
     return encodebytes(hashlib.md5(version.encode()).digest())[:4]
 
 
-def get_transform_kwargs_from_stem_uid(
-    stem_uid: str,
-    version: str,
-) -> Tuple[Optional[Transform], str, str]:
-    uid_ext = get_uid_ext(version)
-    new_uid = stem_uid + uid_ext
-    assert len(new_uid) == 16
-    transform = Transform.filter(
-        uid__startswith=stem_uid, version=version
-    ).one_or_none()
-    return transform, new_uid, version
-
-
 def get_stem_uid_and_version_from_file(file_path: str) -> Tuple[str, str]:
     # line-by-line matching might be faster, but let's go with this for now
     with open(file_path) as file:
@@ -90,33 +71,26 @@ def get_stem_uid_and_version_from_file(file_path: str) -> Tuple[str, str]:
     version = version_match.group(1) if version_match else None
 
     if stem_uid is None or version is None:
-        logger.error(
+        raise SystemExit(
             f"ln.transform.stem_uid and ln.transform.version aren't set in {file_path}\n"
             "Call ln.track() and copy/paste the output into the notebook"
         )
-        # we're not using `raise SystemExit` here to have the right return code on the CLI
-        sys.exit(1)
     return stem_uid, version
 
 
-# also see lamindb.core._run_context.reinitialize_notebook for related code
-def update_transform_source(
-    filepath: str,
+def update_stem_uid_or_version(
+    stem_uid: str,
+    version: str,
     bump_version: bool = False,
 ) -> (bool, str, str):  # type:ignore
-    stem_uid, version = get_stem_uid_and_version_from_file(filepath)
-
-    uid_ext = get_uid_ext(version)
+    get_uid_ext(version)
     updated = False
-    # ask for generating a new stem uid
-    response = "bump"
-    if not bump_version:
+    if bump_version:
+        response = "bump"
+    else:
+        # ask for generating a new stem uid
         # it simply looks better here to not use the logger because we won't have an
         # emoji also for the subsequent input question
-        print(
-            f"Transform is tracked with stem_uid='{stem_uid}' & version='{version}'"
-            f" (uid='{stem_uid}{uid_ext}')"
-        )
         if os.getenv("LAMIN_TESTING") is None:
             response = input(
                 "To create a new stem uid, type 'new'. To bump the version, type 'bump'"
@@ -291,16 +265,15 @@ class run_context:
                 and transform_settings.version is not None
             )
             if transform_settings_are_set:
-                (
-                    transform,
-                    uid,
-                    version,
-                ) = get_transform_kwargs_from_stem_uid(
+                stem_uid, version = (
                     transform_settings.stem_uid,
                     transform_settings.version,
                 )
+                transform = Transform.filter(
+                    uid__startswith=stem_uid, version=version
+                ).one_or_none()
                 if is_run_from_ipython:
-                    short_name, name, filepath = cls._track_notebook(path=path)
+                    short_name, name, _ = cls._track_notebook(path=path)
                 else:
                     import inspect
 
@@ -308,19 +281,22 @@ class run_context:
                     module = inspect.getmodule(frame[0])
                     name = Path(module.__file__).name  # type: ignore
                     short_name = name
-                    filepath = module.__file__
-                is_tracked = cls._create_or_load_transform(
-                    uid=uid,
+                transform_type = (
+                    TransformType.notebook
+                    if is_run_from_ipython
+                    else TransformType.script
+                )
+                cls._create_or_load_transform(
+                    stem_uid=stem_uid,
                     version=version,
                     name=name,
                     reference=reference,
-                    transform_type=TransformType.notebook
-                    if is_run_from_ipython
-                    else TransformType.pipeline,
+                    transform_type=transform_type,
                     short_name=short_name,
-                    filepath=filepath,
                     transform=transform,
                 )
+                # if no error is raised, the transform is tracked
+                is_tracked = True
             if not is_tracked:
                 raise_transform_settings_error()
         else:
@@ -437,17 +413,17 @@ class run_context:
     def _create_or_load_transform(
         cls,
         *,
-        uid: str,
+        stem_uid: str,
         version: Optional[str],
         name: str,
-        reference: Optional[str],
-        short_name: Optional[str],
-        transform_type: TransformType,
-        filepath: str,
+        reference: Optional[str] = None,
+        short_name: Optional[str] = None,
+        transform_type: TransformType = None,
         transform: Optional[Transform] = None,
-    ) -> bool:
+    ):
         # make a new transform record
         if transform is None:
+            uid = f"{stem_uid}{get_uid_ext(version)}"
             transform = Transform(
                 uid=uid,
                 version=version,
@@ -459,7 +435,25 @@ class run_context:
             transform.save()
             logger.important(f"saved: {transform}")
         else:
-            # check whether there was an update
+            # check whether there was an update to the transform, like
+            # renaming the file or updating the title
+            if transform.name != name or transform.short_name != short_name:
+                if os.getenv("LAMIN_TESTING") is None:
+                    response = input(
+                        "Updated transform filename and/or title: Do you want to assign a"
+                        " new stem_uid or version? (y/n)"
+                    )
+                else:
+                    response = "y"
+                if response == "y":
+                    # will raise SystemExit
+                    update_stem_uid_or_version(stem_uid, version)
+                else:
+                    transform.name = name
+                    transform.short_name = short_name
+                    transform.save()
+                    logger.important(f"updated: {transform}")
+            # check whether the transform artifacts were already saved
             if (
                 transform.source_code_id is not None
                 or transform.latest_report_id is not None
@@ -472,28 +466,10 @@ class run_context:
                 else:
                     response = "y"
                 if response == "y":
-                    update_transform_source(filepath, bump_version=True)
+                    update_stem_uid_or_version(stem_uid, version, bump_version=True)
                 else:
-                    logger.warning(
-                        "not tracking this transform, either increase version or delete"
-                        " the saved transform.source_code and transform.latest_report"
-                    )
-                    return False
-            if transform.name != name or transform.short_name != short_name:
-                response = input(
-                    "Updated notebook name and/or title: Do you want to assign a"
-                    " new uid prefix or version? (y/n)"
-                )
-                if response == "y":
-                    update_transform_source(filepath)
-                transform.name = name
-                transform.short_name = short_name
-                transform.save()
-                if response == "y":
-                    logger.important(f"saved: {transform}")
-                else:
-                    logger.important(f"updated: {transform}")
+                    # we want a new stem_uid in this case, hence raise the error
+                    raise_transform_settings_error()
             else:
                 logger.important(f"loaded: {transform}")
         cls.transform = transform
-        return True

lamindb/core/_settings.py

@@ -3,6 +3,7 @@ from typing import Dict, Literal, Mapping, Optional, Tuple, Union
 
 import lamindb_setup as ln_setup
 from lamin_utils import logger
+from lamindb_setup._add_remote_storage import switch_default_storage
 from upath import UPath
 
 VERBOSITY_TO_INT = {
@@ -112,11 +113,14 @@ class Settings:
     def storage(
         self, path_kwargs: Union[str, Path, UPath, Tuple[Union[str, UPath], Mapping]]
     ):
+        logger.warning(
+            "you'll no longer be able to set arbitrary storage locations soon"
+        )
         if isinstance(path_kwargs, tuple):
             path, kwargs = path_kwargs
         else:
             path, kwargs = path_kwargs, {}
-        ln_setup.set.storage(path, **kwargs)
+        switch_default_storage(path, **kwargs)
 
     @property
     def verbosity(self) -> str:

lamindb/core/storage/__init__.py

@@ -6,7 +6,7 @@
    AnnDataAccessor
    BackedAccessor
 """
-from lamindb_setup.dev.upath import LocalPathClasses, UPath, infer_filesystem
+from lamindb_setup.core.upath import LocalPathClasses, UPath, infer_filesystem
 
 from ._anndata_sizes import size_adata
 from ._backed_access import AnnDataAccessor, BackedAccessor

lamindb/core/storage/_backed_access.py

@@ -18,7 +18,7 @@ from anndata.compat import _read_attr
 from fsspec.core import OpenFile
 from fsspec.implementations.local import LocalFileSystem
 from lamin_utils import logger
-from lamindb_setup.dev.upath import UPath, create_mapper, infer_filesystem
+from lamindb_setup.core.upath import UPath, create_mapper, infer_filesystem
 from lnschema_core import Artifact
 from packaging import version
 

lamindb/core/storage/_zarr.py

@@ -6,7 +6,7 @@ import zarr
 from anndata import AnnData
 from anndata._io import read_zarr
 from anndata._io.specs import write_elem
-from lamindb_setup.dev.upath import create_mapper, infer_filesystem
+from lamindb_setup.core.upath import create_mapper, infer_filesystem
 
 from ._anndata_sizes import _size_elem, _size_raw, size_adata
 

lamindb/core/storage/file.py

@@ -8,8 +8,9 @@ import anndata as ad
 import pandas as pd
 from lamin_utils import logger
 from lamindb_setup import settings as setup_settings
-from lamindb_setup.dev import StorageSettings
-from lamindb_setup.dev.upath import (
+from lamindb_setup.core import StorageSettings
+from lamindb_setup.core.types import UPathStr
+from lamindb_setup.core.upath import (
     LocalPathClasses,
     UPath,
     create_path,
@@ -103,7 +104,7 @@ def read_adata_h5ad(filepath, **kwargs) -> ad.AnnData:
         return adata
 
 
-def store_artifact(localpath: Union[str, Path, UPath], storagepath: UPath) -> None:
+def store_artifact(localpath: UPathStr, storagepath: UPath) -> None:
     """Store directory or file to configured storage location.
 
     Returns size in bytes.
@@ -132,8 +133,12 @@ def delete_storage_using_key(
     delete_storage(filepath)
 
 
-def delete_storage(storagepath: Union[Path, UPath]):
+def delete_storage(storagepath: Path):
     """Delete arbitrary artifact."""
+    if not storagepath.is_relative_to(settings.storage):
+        logger.warning("couldn't delete files outside of default storage")
+        return "did-not-delete"
+    # only delete files in the default storage
     if storagepath.is_file():
         storagepath.unlink()
     elif storagepath.is_dir():
@@ -156,12 +161,12 @@ def read_fcs(*args, **kwargs):
     return readfcs.read(*args, **kwargs)
 
 
-def read_tsv(path: Union[str, Path, UPath]) -> pd.DataFrame:
+def read_tsv(path: UPathStr) -> pd.DataFrame:
     path_sanitized = Path(path)
     return pd.read_csv(path_sanitized, sep="\t")
 
 
-def load_html(path: Union[str, Path, UPath]):
+def load_html(path: UPathStr):
     if is_run_from_ipython:
         with open(path, encoding="utf-8") as f:
             html_content = f.read()
@@ -180,7 +185,7 @@ def load_html(path: Union[str, Path, UPath]):
         return path
 
 
-def load_json(path: Union[str, Path, UPath]):
+def load_json(path: UPathStr):
     import json
 
     with open(path) as f:
@@ -188,7 +193,7 @@ def load_json(path: Union[str, Path, UPath]):
     return data
 
 
-def load_to_memory(filepath: Union[str, Path, UPath], stream: bool = False, **kwargs):
+def load_to_memory(filepath: UPathStr, stream: bool = False, **kwargs):
     """Load a file into memory.
 
     Returns the filepath if no in-memory form is found.

lamindb/core/storage/object.py

@@ -1,8 +1,7 @@
-from pathlib import Path
-from typing import Optional, Union
+from typing import Optional
 
 from anndata import AnnData
-from lamindb_setup.dev.upath import UPath
+from lamindb_setup.core.types import UPathStr
 from pandas import DataFrame
 
 
@@ -25,7 +24,7 @@ def infer_suffix(dmem, adata_format: Optional[str] = None):
         raise NotImplementedError
 
 
-def write_to_file(dmem, filepath: Union[str, Path, UPath]):
+def write_to_file(dmem, filepath: UPathStr):
     if isinstance(dmem, AnnData):
         dmem.write(filepath)
     elif isinstance(dmem, DataFrame):

lamindb/core/types.py

@@ -3,16 +3,16 @@
 .. autosummary::
    :toctree: .
 
-   PathLike
+   UPathStr
    DataLike
    StrField
    ListLike
    TransformType
 """
+from lamindb_setup.core.types import UPathStr
 from lnschema_core.types import (
     DataLike,
     ListLike,
-    PathLike,
     StrField,
     TransformType,
 )

lamindb/core/versioning.py

@@ -1,6 +1,6 @@
 from typing import Optional, Tuple
 
-from lamindb_setup.dev.upath import LocalPathClasses, UPath
+from lamindb_setup.core.upath import LocalPathClasses, UPath
 from lnschema_core import ids
 from lnschema_core.models import IsVersioned