lamindb 0.76.13__py3-none-any.whl → 0.76.15__py3-none-any.whl

lamindb/__init__.py

@@ -43,7 +43,7 @@ Modules and settings.
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.76.13"
+__version__ = "0.76.15"
 
 import os as _os
 

lamindb/_artifact.py

@@ -28,39 +28,41 @@ from lnschema_core.types import (
     VisibilityChoice,
 )
 
-from lamindb._utils import attach_func_to_class_method
-from lamindb.core._data import _track_run_input, describe, view_lineage
-from lamindb.core._settings import settings
-from lamindb.core.exceptions import IntegrityError, InvalidArgument
-from lamindb.core.loaders import load_to_memory
-from lamindb.core.storage import (
+from ._utils import attach_func_to_class_method
+from .core._data import (
+    _track_run_input,
+    add_transform_to_kwargs,
+    describe,
+    get_run,
+    save_feature_set_links,
+    save_feature_sets,
+    view_lineage,
+)
+from .core._settings import settings
+from .core.exceptions import IntegrityError, InvalidArgument
+from .core.loaders import load_to_memory
+from .core.storage import (
     LocalPathClasses,
     UPath,
     delete_storage,
     infer_suffix,
     write_to_disk,
 )
-from lamindb.core.storage.paths import (
+from .core.storage._pyarrow_dataset import PYARROW_SUFFIXES
+from .core.storage.objects import _mudata_is_installed
+from .core.storage.paths import (
+    AUTO_KEY_PREFIX,
     auto_storage_key_from_artifact,
     auto_storage_key_from_artifact_uid,
     check_path_is_child_of_root,
     filepath_cache_key_from_artifact,
     filepath_from_artifact,
 )
-from lamindb.core.versioning import (
+from .core.versioning import (
     create_uid,
     message_update_key_in_version_family,
 )
 
-from .core._data import (
-    add_transform_to_kwargs,
-    get_run,
-    save_feature_set_links,
-    save_feature_sets,
-)
-from .core.storage.objects import _mudata_is_installed
-from .core.storage.paths import AUTO_KEY_PREFIX
-
 try:
     from .core.storage._zarr import zarr_is_adata
 except ImportError:
@@ -72,6 +74,7 @@ except ImportError:
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
     from mudata import MuData
+    from pyarrow.dataset import Dataset as PyArrowDataset
     from tiledbsoma import Collection as SOMACollection
     from tiledbsoma import Experiment as SOMAExperiment
 
@@ -772,19 +775,14 @@ def from_dir(
     else:
         folder_key_path = Path(key)
 
-    # always sanitize by stripping a trailing slash
-    folder_key = folder_key_path.as_posix().rstrip("/")
-
-    # TODO: (non-local) UPath doesn't list the first level artifacts and dirs with "*"
-    pattern = "" if not isinstance(folderpath, LocalPathClasses) else "*"
-
+    folder_key = folder_key_path.as_posix()
     # silence fine-grained logging
     verbosity = settings.verbosity
     verbosity_int = settings._verbosity_int
     if verbosity_int >= 1:
         settings.verbosity = "warning"
     artifacts_dict = {}
-    for filepath in folderpath.rglob(pattern):
+    for filepath in folderpath.rglob("*"):
         if filepath.is_file():
             relative_path = get_relative_path_to_directory(filepath, folderpath)
             artifact_key = folder_key + "/" + relative_path.as_posix()
@@ -802,7 +800,8 @@ def from_dir(
         if artifact.hash is not None
     ]
     uids = artifacts_dict.keys()
-    if len(set(hashes)) == len(hashes):
+    n_unique_hashes = len(set(hashes))
+    if n_unique_hashes == len(hashes):
         artifacts = list(artifacts_dict.values())
     else:
         # consider exact duplicates (same id, same hash)
@@ -811,7 +810,7 @@ def from_dir(
         #     logger.warning("dropping duplicate records in list of artifact records")
         #     artifacts = list(set(uids))
         # consider false duplicates (different id, same hash)
-        if not len(set(uids)) == len(set(hashes)):
+        if not len(set(uids)) == n_unique_hashes:
             seen_hashes = set()
             non_unique_artifacts = {
                 hash: artifact
@@ -905,14 +904,19 @@ def replace(
 # docstring handled through attach_func_to_class_method
 def open(
     self, mode: str = "r", is_run_input: bool | None = None
-) -> AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment:
+) -> (
+    AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment | PyArrowDataset
+):
     # ignore empty suffix for now
-    suffixes = (".h5", ".hdf5", ".h5ad", ".zarr", ".tiledbsoma", "")
+    suffixes = ("", ".h5", ".hdf5", ".h5ad", ".zarr", ".tiledbsoma") + PYARROW_SUFFIXES
     if self.suffix not in suffixes:
         raise ValueError(
-            "Artifact should have a zarr, h5 or tiledbsoma object as the underlying data, please"
-            " use one of the following suffixes for the object name:"
-            f" {', '.join(suffixes[:-1])}."
+            "Artifact should have a zarr, h5, tiledbsoma object"
+            " or a compatible `pyarrow.dataset.dataset` directory"
+            " as the underlying data, please use one of the following suffixes"
+            f" for the object name: {', '.join(suffixes[1:])}."
+            f" Or no suffix for a folder with {', '.join(PYARROW_SUFFIXES)} files"
+            " (no mixing allowed)."
         )
     if self.suffix != ".tiledbsoma" and self.key != "soma" and mode != "r":
         raise ValueError("Only a tiledbsoma store can be openened with `mode!='r'`.")
@@ -925,7 +929,7 @@ def open(
         filepath.name == "soma" or filepath.suffix == ".tiledbsoma"
     ) and mode == "w"
     # consider the case where an object is already locally cached
-    localpath = setup_settings.instance.storage.cloud_to_local_no_update(
+    localpath = setup_settings.paths.cloud_to_local_no_update(
         filepath, cache_key=cache_key
     )
     if not is_tiledbsoma_w and localpath.exists():
@@ -963,12 +967,12 @@ def _synchronize_cleanup_on_error(
     filepath: UPath, cache_key: str | None = None
 ) -> UPath:
     try:
-        cache_path = setup_settings.instance.storage.cloud_to_local(
+        cache_path = setup_settings.paths.cloud_to_local(
             filepath, cache_key=cache_key, print_progress=True
         )
     except Exception as e:
         if not isinstance(filepath, LocalPathClasses):
-            cache_path = setup_settings.instance.storage.cloud_to_local_no_update(
+            cache_path = setup_settings.paths.cloud_to_local_no_update(
                 filepath, cache_key=cache_key
             )
             if cache_path.is_file():
@@ -1156,9 +1160,7 @@ def _cache_path(self) -> UPath:
     )
     if isinstance(filepath, LocalPathClasses):
         return filepath
-    return setup_settings.instance.storage.cloud_to_local_no_update(
-        filepath, cache_key=cache_key
-    )
+    return setup_settings.paths.cloud_to_local_no_update(filepath, cache_key=cache_key)
 
 
 # docstring handled through attach_func_to_class_method

lamindb/_can_validate.py

@@ -10,10 +10,10 @@ from lamin_utils import colors, logger
 from lamindb_setup.core._docs import doc_args
 from lnschema_core import CanValidate, Record
 
-from lamindb._utils import attach_func_to_class_method
-
 from ._from_values import _has_organism_field, _print_values, get_or_create_records
 from ._record import _queryset, get_name_field
+from ._utils import attach_func_to_class_method
+from .core.exceptions import ValidationError
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
@@ -496,9 +496,9 @@ def _add_or_remove_synonyms(
                 " with the following records:\n"
             )
             display(records_df)
-            raise ValueError(
-                "cannot assigned a synonym that is already associated with a record to a different record.\n"
-                "Consider removing the synonym from existing records or using a different synonym."
+            raise ValidationError(
+                f"you are trying to assign a synonym to record: {record}\n"
+                "    → consider removing the synonym from existing records or using a different synonym."
             )
 
     # passed synonyms
@@ -516,7 +516,7 @@ def _add_or_remove_synonyms(
         return
     # because we use | as the separator
     if any("|" in i for i in syn_new_set):
-        raise ValueError("a synonym can't contain '|'!")
+        raise ValidationError("a synonym can't contain '|'!")
 
     # existing synonyms
     syns_exist = record.synonyms

lamindb/_collection.py

@@ -20,20 +20,21 @@ from lnschema_core.models import (
 )
 from lnschema_core.types import VisibilityChoice
 
-from lamindb._utils import attach_func_to_class_method
-from lamindb.core._data import _track_run_input, describe, view_lineage
-from lamindb.core._mapped_collection import MappedCollection
-from lamindb.core.versioning import process_revises
-
 from . import Artifact, Run
 from ._record import init_self_from_db, update_attributes
+from ._utils import attach_func_to_class_method
 from .core._data import (
+    _track_run_input,
     add_transform_to_kwargs,
+    describe,
     get_run,
     save_feature_set_links,
     save_feature_sets,
+    view_lineage,
 )
+from .core._mapped_collection import MappedCollection
 from .core._settings import settings
+from .core.versioning import process_revises
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
@@ -187,6 +188,16 @@ def __init__(
     _track_run_input(artifacts, run=run)
 
 
+# docstring handled through attach_func_to_class_method
+def append(self, artifact: Artifact, run: Run | None = None) -> Collection:
+    return Collection(
+        self.artifacts.all().list() + [artifact],
+        description=self.description,
+        revises=self,
+        run=run,
+    )
+
+
 # internal function, not exposed to user
 def from_artifacts(artifacts: Iterable[Artifact]) -> tuple[str, dict[str, str]]:
     # assert all artifacts are already saved
@@ -353,6 +364,7 @@ def restore(self) -> None:
 @doc_args(Collection.ordered_artifacts.__doc__)
 def ordered_artifacts(self) -> QuerySet:
     """{}"""  # noqa: D415
+    # tracking is done via QueryManager (_query_manager.py)
     return self.artifacts.order_by("links_collection__id")
 
 
@@ -365,6 +377,7 @@ def data_artifact(self) -> Artifact | None:
 
 METHOD_NAMES = [
     "__init__",
+    "append",
     "mapped",
     "cache",
     "load",