lamindb 0.74.0__py3-none-any.whl → 0.74.2__py3-none-any.whl

lamindb/__init__.py

@@ -24,7 +24,7 @@ Key functionality:
    connect
    track
    finish
-   Annotate
+   Curate
    view
    save
 
@@ -42,7 +42,7 @@ Modules & settings:
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.74.0"
+__version__ = "0.74.2"
 
 import os as _os
 
@@ -77,30 +77,27 @@ if _check_instance_setup(from_lamindb=True):
 
     from . import core  # isort: split
     from . import (
-        _annotate,
         _artifact,
         _can_validate,
         _collection,
+        _curate,
         _feature,
         _feature_set,
         _is_versioned,
         _parents,
-        _registry,
+        _record,
         _run,
         _storage,
         _transform,
         _ulabel,
+        integrations,
     )
-
-    dev = core  # backward compat
-    from . import integrations
-    from ._annotate import Annotate
+    from ._curate import Curate
     from ._finish import finish
     from ._save import save
     from ._view import view
     from .core._run_context import run_context as _run_context
     from .core._settings import settings
-    from .core._transform_settings import transform  # backward compat
 
     # schema modules
     if not _os.environ.get("LAMINDB_MULTI_INSTANCE") == "true":
@@ -112,3 +109,6 @@ if _check_instance_setup(from_lamindb=True):
 
     track = _run_context._track
     settings.__doc__ = """Global :class:`~lamindb.core.Settings`."""
+    from django.db.models import Q
+
+    Annotate = Curate  # backward compat

lamindb/_artifact.py

@@ -1,22 +1,19 @@
 from __future__ import annotations
 
-import os
 import shutil
-from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path, PurePath, PurePosixPath
 from typing import TYPE_CHECKING, Any, Mapping
 
 import fsspec
 import lamindb_setup as ln_setup
 import pandas as pd
-import psutil
 from anndata import AnnData
 from lamin_utils import colors, logger
 from lamindb_setup import settings as setup_settings
 from lamindb_setup._init_instance import register_storage_in_instance
 from lamindb_setup.core._docs import doc_args
 from lamindb_setup.core._settings_storage import init_storage
-from lamindb_setup.core.hashing import b16_to_b64, hash_file, hash_md5s_from_dir
+from lamindb_setup.core.hashing import hash_dir, hash_file
 from lamindb_setup.core.upath import (
     create_path,
     extract_suffix_from_path,
@@ -66,6 +63,8 @@ except ImportError:
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
     from mudata import MuData
+    from tiledbsoma import Collection as SOMACollection
+    from tiledbsoma import Experiment as SOMAExperiment
 
     from lamindb.core.storage._backed_access import AnnDataAccessor, BackedAccessor
 
@@ -190,7 +189,7 @@ def get_stat_or_artifact(
     using_key: str | None = None,
 ) -> tuple[int, str | None, str | None, int | None] | Artifact:
     n_objects = None
-    if settings.upon_file_create_skip_size_hash:
+    if settings.creation.artifact_skip_size_hash:
         return None, None, None, n_objects
     stat = path.stat()  # one network request
     if not isinstance(path, LocalPathClasses):
@@ -207,26 +206,7 @@ def get_stat_or_artifact(
             return size, hash, hash_type, n_objects
     else:
         if path.is_dir():
-            files = (subpath for subpath in path.rglob("*") if subpath.is_file())
-
-            def hash_size(file):
-                file_size = file.stat().st_size
-                return hash_file(file, file_size)[0], file_size
-
-            try:
-                n_workers = len(psutil.Process().cpu_affinity())
-            except AttributeError:
-                n_workers = psutil.cpu_count()
-            if n_workers > 1:
-                with ThreadPoolExecutor(n_workers) as pool:
-                    hashes_sizes = pool.map(hash_size, files)
-            else:
-                hashes_sizes = map(hash_size, files)
-            hashes, sizes = zip(*hashes_sizes)
-
-            hash, hash_type = hash_md5s_from_dir(hashes)
-            n_objects = len(hashes)
-            size = sum(sizes)
+            size, hash, hash_type, n_objects = hash_dir(path)
         else:
             hash, hash_type = hash_file(path)
             size = stat.st_size
@@ -242,14 +222,14 @@ def get_stat_or_artifact(
             Artifact.objects.using(using_key).filter(hash=hash, visibility=None).all()
         )
     if len(result) > 0:
-        if settings.upon_artifact_create_if_hash_exists == "error":
+        if settings.creation.artifact_if_hash_exists == "error":
             msg = f"artifact with same hash exists: {result[0]}"
             hint = (
                 "💡 you can make this error a warning:\n"
-                "    ln.settings.upon_artifact_create_if_hash_exists"
+                "    ln.settings.creation.artifact_if_hash_exists"
             )
             raise FileExistsError(f"{msg}\n{hint}")
-        elif settings.upon_artifact_create_if_hash_exists == "warn_create_new":
+        elif settings.creation.artifact_if_hash_exists == "warn_create_new":
             logger.warning(
                 "creating new Artifact object despite existing artifact with same hash:"
                 f" {result[0]}"
@@ -376,7 +356,7 @@ def get_artifact_kwargs_from_data(
     )
 
     # do we use a virtual or an actual storage key?
-    key_is_virtual = settings.artifact_use_virtual_keys
+    key_is_virtual = settings.creation._artifact_use_virtual_keys
 
     # if the file is already in storage, independent of the default
     # we use an actual storage key
@@ -444,7 +424,7 @@ def log_storage_hint(
     logger.hint(hint)
 
 
-def data_is_anndata(data: AnnData | UPathStr):
+def data_is_anndata(data: AnnData | UPathStr) -> bool:
     if isinstance(data, AnnData):
         return True
     if isinstance(data, (str, Path, UPath)):
@@ -464,7 +444,7 @@ def data_is_anndata(data: AnnData | UPathStr):
     return False
 
 
-def data_is_mudata(data: MuData | UPathStr):
+def data_is_mudata(data: MuData | UPathStr) -> bool:
     if _mudata_is_installed():
         from mudata import MuData
 
@@ -582,7 +562,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
 
     # an object with the same hash already exists
     if isinstance(kwargs_or_artifact, Artifact):
-        from ._registry import init_self_from_db
+        from ._record import init_self_from_db
 
         init_self_from_db(artifact, kwargs_or_artifact)
         # adding "key" here is dangerous because key might be auto-populated
@@ -645,7 +625,7 @@ def from_df(
     is_new_version_of: Artifact | None = None,
     **kwargs,
 ) -> Artifact:
-    """{}."""
+    """{}"""  # noqa: D415
     artifact = Artifact(
         data=df,
         key=key,
@@ -672,7 +652,7 @@ def from_anndata(
     is_new_version_of: Artifact | None = None,
     **kwargs,
 ) -> Artifact:
-    """{}."""
+    """{}"""  # noqa: D415
     if not data_is_anndata(adata):
         raise ValueError("data has to be an AnnData object or a path to AnnData-like")
     artifact = Artifact(
@@ -701,7 +681,7 @@ def from_mudata(
     is_new_version_of: Artifact | None = None,
     **kwargs,
 ) -> Artifact:
-    """{}."""
+    """{}"""  # noqa: D415
     artifact = Artifact(
         data=mdata,
         key=key,
@@ -725,7 +705,7 @@ def from_dir(
     *,
     run: Run | None = None,
 ) -> list[Artifact]:
-    """{}."""
+    """{}"""  # noqa: D415
     logger.warning(
         "this creates one artifact per file in the directory - you might simply call"
         " ln.Artifact(dir) to get one artifact for the entire directory"
@@ -880,14 +860,25 @@ def replace(
     self._to_store = not check_path_in_storage
 
 
+# deprecated
+def backed(
+    self, is_run_input: bool | None = None
+) -> AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment:
+    logger.warning("`.backed()` is deprecated, use `.open()`!'")
+    return self.open(is_run_input)
+
+
 # docstring handled through attach_func_to_class_method
-def backed(self, is_run_input: bool | None = None) -> AnnDataAccessor | BackedAccessor:
-    suffixes = (".h5", ".hdf5", ".h5ad", ".zarr")
+def open(
+    self, is_run_input: bool | None = None
+) -> AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment:
+    # ignore empty suffix for now
+    suffixes = (".h5", ".hdf5", ".h5ad", ".zarr", ".tiledbsoma", "")
     if self.suffix not in suffixes:
         raise ValueError(
-            "Artifact should have a zarr or h5 object as the underlying data, please"
+            "Artifact should have a zarr, h5 or tiledbsoma object as the underlying data, please"
             " use one of the following suffixes for the object name:"
-            f" {', '.join(suffixes)}."
+            f" {', '.join(suffixes[:-1])}."
         )
 
     from lamindb.core.storage._backed_access import backed_access
@@ -965,7 +956,7 @@ def delete(
         )
         delete_record = response == "y"
     else:
-        assert permanent
+        assert permanent  # noqa: S101
         delete_record = True
 
     if delete_record:
@@ -1010,7 +1001,7 @@ def _delete_skip_storage(artifact, *args, **kwargs) -> None:
 
 
 # docstring handled through attach_func_to_class_method
-def save(self, upload: bool | None = None, **kwargs) -> None:
+def save(self, upload: bool | None = None, **kwargs) -> Artifact:
     state_was_adding = self._state.adding
     print_progress = kwargs.pop("print_progress", True)
     access_token = kwargs.pop("access_token", None)
@@ -1064,7 +1055,7 @@ def _save_skip_storage(file, **kwargs) -> None:
 @property  # type: ignore
 @doc_args(Artifact.path.__doc__)
 def path(self) -> Path | UPath:
-    """{}."""
+    """{}"""  # noqa: D415
     using_key = settings._using_key
     return filepath_from_artifact(self, using_key)
 
@@ -1080,7 +1071,7 @@ METHOD_NAMES = [
     "from_anndata",
     "from_df",
     "from_mudata",
-    "backed",
+    "open",
     "cache",
     "load",
     "delete",
@@ -1106,5 +1097,4 @@ for name in METHOD_NAMES:
 Artifact._delete_skip_storage = _delete_skip_storage
 Artifact._save_skip_storage = _save_skip_storage
 Artifact.path = path
-# this seems a Django-generated function
-delattr(Artifact, "get_visibility_display")
+Artifact.backed = backed

lamindb/_can_validate.py

@@ -8,12 +8,12 @@ import pandas as pd
 from django.core.exceptions import FieldDoesNotExist
 from lamin_utils import colors, logger
 from lamindb_setup.core._docs import doc_args
-from lnschema_core import CanValidate, Registry
+from lnschema_core import CanValidate, Record
 
 from lamindb._utils import attach_func_to_class_method
 
 from ._from_values import _has_organism_field, _print_values
-from ._registry import _queryset, get_default_str_field
+from ._record import _queryset, get_default_str_field
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
@@ -29,10 +29,10 @@ def inspect(
     field: str | StrField | None = None,
     *,
     mute: bool = False,
-    organism: str | Registry | None = None,
-    public_source: Registry | None = None,
+    organism: str | Record | None = None,
+    public_source: Record | None = None,
 ) -> InspectResult:
-    """{}."""
+    """{}"""  # noqa: D415
     return _inspect(
         cls=cls,
         values=values,
@@ -51,9 +51,9 @@ def validate(
     field: str | StrField | None = None,
     *,
     mute: bool = False,
-    organism: str | Registry | None = None,
+    organism: str | Record | None = None,
 ) -> np.ndarray:
-    """{}."""
+    """{}"""  # noqa: D415
     return _validate(cls=cls, values=values, field=field, mute=mute, organism=organism)
 
 
@@ -64,10 +64,10 @@ def _inspect(
     *,
     mute: bool = False,
     using_key: str | None = None,
-    organism: str | Registry | None = None,
-    public_source: Registry | None = None,
+    organism: str | Record | None = None,
+    public_source: Record | None = None,
 ) -> pd.DataFrame | dict[str, list[str]]:
-    """{}."""
+    """{}"""  # noqa: D415
     from lamin_utils._inspect import inspect
 
     if isinstance(values, str):
@@ -148,9 +148,9 @@ def _validate(
     *,
     mute: bool = False,
     using_key: str | None = None,
-    organism: str | Registry | None = None,
+    organism: str | Record | None = None,
 ) -> np.ndarray:
-    """{}."""
+    """{}"""  # noqa: D415
     from lamin_utils._inspect import validate
 
     return_str = True if isinstance(values, str) else False
@@ -197,9 +197,9 @@ def standardize(
     public_aware: bool = True,
     keep: Literal["first", "last", False] = "first",
     synonyms_field: str = "synonyms",
-    organism: str | Registry | None = None,
+    organism: str | Record | None = None,
 ) -> list[str] | dict[str, str]:
-    """{}."""
+    """{}"""  # noqa: D415
     return _standardize(
         cls=cls,
         values=values,
@@ -223,8 +223,10 @@ def set_abbr(self, value: str):
     else:
         try:
             self.add_synonym(value, save=False)
-        except Exception:  # pragma: no cover
-            pass
+        except Exception as e:  # pragma: no cover
+            logger.debug(
+                f"Encountered an Exception while attempting to add synonyms.\n{e}"
+            )
 
     if not self._state.adding:
         self.save()
@@ -260,9 +262,9 @@ def _standardize(
     keep: Literal["first", "last", False] = "first",
     synonyms_field: str = "synonyms",
     using_key: str | None = None,
-    organism: str | Registry | None = None,
+    organism: str | Record | None = None,
 ) -> list[str] | dict[str, str]:
-    """{}."""
+    """{}"""  # noqa: D415
     from lamin_utils._standardize import standardize as map_synonyms
 
     return_str = True if isinstance(values, str) else False
@@ -358,14 +360,14 @@ def _standardize(
 
 def _add_or_remove_synonyms(
     synonym: str | Iterable,
-    record: Registry,
+    record: Record,
     action: Literal["add", "remove"],
     force: bool = False,
     save: bool | None = None,
 ):
     """Add or remove synonyms."""
 
-    def check_synonyms_in_all_records(synonyms: set[str], record: Registry):
+    def check_synonyms_in_all_records(synonyms: set[str], record: Record):
         """Errors if input synonym is associated with other records in the DB."""
         import pandas as pd
         from IPython.display import display
@@ -433,7 +435,7 @@ def _add_or_remove_synonyms(
         record.save()
 
 
-def _check_synonyms_field_exist(record: Registry):
+def _check_synonyms_field_exist(record: Record):
     try:
         record.__getattribute__("synonyms")
     except AttributeError:
@@ -445,7 +447,7 @@ def _check_synonyms_field_exist(record: Registry):
 def _filter_query_based_on_organism(
     queryset: QuerySet,
     field: str,
-    organism: str | Registry | None = None,
+    organism: str | Record | None = None,
     values_list_field: str | None = None,
 ):
     """Filter a queryset based on organism."""

lamindb/_collection.py

@@ -30,7 +30,7 @@ from lamindb.core._mapped_collection import MappedCollection
 from lamindb.core.versioning import get_uid_from_old_version, init_uid
 
 from . import Artifact, Run
-from ._registry import init_self_from_db
+from ._record import init_self_from_db
 from .core._data import (
     add_transform_to_kwargs,
     get_run,
@@ -100,7 +100,7 @@ def __init__(
     else:
         if not hasattr(artifacts, "__getitem__"):
             raise ValueError("Artifact or List[Artifact] is allowed.")
-        assert isinstance(artifacts[0], Artifact)  # type: ignore
+        assert isinstance(artifacts[0], Artifact)  # type: ignore  # noqa: S101
     hash, feature_sets = from_artifacts(artifacts)  # type: ignore
     if meta is not None:
         if not isinstance(meta, Artifact):
@@ -328,7 +328,7 @@ def delete(self, permanent: bool | None = None) -> None:
 
 
 # docstring handled through attach_func_to_class_method
-def save(self, using: str | None = None) -> None:
+def save(self, using: str | None = None) -> Collection:
     if self.artifact is not None:
         self.artifact.save()
     # we don't need to save feature sets again
@@ -350,6 +350,7 @@ def save(self, using: str | None = None) -> None:
     save_feature_set_links(self)
     if using is not None:
         logger.warning("using argument is ignored")
+    return self
 
 
 # docstring handled through attach_func_to_class_method
@@ -364,7 +365,7 @@ def restore(self) -> None:
 @property  # type: ignore
 @doc_args(Collection.artifacts.__doc__)
 def artifacts(self) -> QuerySet:
-    """{}."""
+    """{}"""  # noqa: D415
     return self.unordered_artifacts.order_by("collection_links__id")
 
 
@@ -390,7 +391,5 @@ if ln_setup._TESTING:
 for name in METHOD_NAMES:
     attach_func_to_class_method(name, Collection, globals())
 
-# this seems a Django-generated function
-delattr(Collection, "get_visibility_display")
 Collection.artifacts = artifacts
 Collection.stage = cache