lamindb 0.76.14__py3-none-any.whl → 0.76.15__py3-none-any.whl

lamindb/__init__.py

@@ -43,7 +43,7 @@ Modules and settings.
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.76.14"
+__version__ = "0.76.15"
 
 import os as _os
 

lamindb/_artifact.py

@@ -28,39 +28,41 @@ from lnschema_core.types import (
     VisibilityChoice,
 )
 
-from lamindb._utils import attach_func_to_class_method
-from lamindb.core._data import _track_run_input, describe, view_lineage
-from lamindb.core._settings import settings
-from lamindb.core.exceptions import IntegrityError, InvalidArgument
-from lamindb.core.loaders import load_to_memory
-from lamindb.core.storage import (
+from ._utils import attach_func_to_class_method
+from .core._data import (
+    _track_run_input,
+    add_transform_to_kwargs,
+    describe,
+    get_run,
+    save_feature_set_links,
+    save_feature_sets,
+    view_lineage,
+)
+from .core._settings import settings
+from .core.exceptions import IntegrityError, InvalidArgument
+from .core.loaders import load_to_memory
+from .core.storage import (
     LocalPathClasses,
     UPath,
     delete_storage,
     infer_suffix,
     write_to_disk,
 )
-from lamindb.core.storage.paths import (
+from .core.storage._pyarrow_dataset import PYARROW_SUFFIXES
+from .core.storage.objects import _mudata_is_installed
+from .core.storage.paths import (
+    AUTO_KEY_PREFIX,
     auto_storage_key_from_artifact,
     auto_storage_key_from_artifact_uid,
     check_path_is_child_of_root,
     filepath_cache_key_from_artifact,
     filepath_from_artifact,
 )
-from lamindb.core.versioning import (
+from .core.versioning import (
     create_uid,
     message_update_key_in_version_family,
 )
 
-from .core._data import (
-    add_transform_to_kwargs,
-    get_run,
-    save_feature_set_links,
-    save_feature_sets,
-)
-from .core.storage.objects import _mudata_is_installed
-from .core.storage.paths import AUTO_KEY_PREFIX
-
 try:
     from .core.storage._zarr import zarr_is_adata
 except ImportError:
@@ -72,6 +74,7 @@ except ImportError:
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
     from mudata import MuData
+    from pyarrow.dataset import Dataset as PyArrowDataset
     from tiledbsoma import Collection as SOMACollection
     from tiledbsoma import Experiment as SOMAExperiment
 
@@ -772,19 +775,14 @@ def from_dir(
     else:
         folder_key_path = Path(key)
 
-    # always sanitize by stripping a trailing slash
-    folder_key = folder_key_path.as_posix().rstrip("/")
-
-    # TODO: (non-local) UPath doesn't list the first level artifacts and dirs with "*"
-    pattern = "" if not isinstance(folderpath, LocalPathClasses) else "*"
-
+    folder_key = folder_key_path.as_posix()
     # silence fine-grained logging
     verbosity = settings.verbosity
     verbosity_int = settings._verbosity_int
     if verbosity_int >= 1:
         settings.verbosity = "warning"
     artifacts_dict = {}
-    for filepath in folderpath.rglob(pattern):
+    for filepath in folderpath.rglob("*"):
         if filepath.is_file():
             relative_path = get_relative_path_to_directory(filepath, folderpath)
             artifact_key = folder_key + "/" + relative_path.as_posix()
@@ -802,7 +800,8 @@ def from_dir(
         if artifact.hash is not None
     ]
     uids = artifacts_dict.keys()
-    if len(set(hashes)) == len(hashes):
+    n_unique_hashes = len(set(hashes))
+    if n_unique_hashes == len(hashes):
         artifacts = list(artifacts_dict.values())
     else:
         # consider exact duplicates (same id, same hash)
@@ -811,7 +810,7 @@ def from_dir(
         #     logger.warning("dropping duplicate records in list of artifact records")
         #     artifacts = list(set(uids))
         # consider false duplicates (different id, same hash)
-        if not len(set(uids)) == len(set(hashes)):
+        if not len(set(uids)) == n_unique_hashes:
             seen_hashes = set()
             non_unique_artifacts = {
                 hash: artifact
@@ -905,14 +904,19 @@ def replace(
 # docstring handled through attach_func_to_class_method
 def open(
     self, mode: str = "r", is_run_input: bool | None = None
-) -> AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment:
+) -> (
+    AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment | PyArrowDataset
+):
     # ignore empty suffix for now
-    suffixes = (".h5", ".hdf5", ".h5ad", ".zarr", ".tiledbsoma", "")
+    suffixes = ("", ".h5", ".hdf5", ".h5ad", ".zarr", ".tiledbsoma") + PYARROW_SUFFIXES
     if self.suffix not in suffixes:
         raise ValueError(
-            "Artifact should have a zarr, h5 or tiledbsoma object as the underlying data, please"
-            " use one of the following suffixes for the object name:"
-            f" {', '.join(suffixes[:-1])}."
+            "Artifact should have a zarr, h5, tiledbsoma object"
+            " or a compatible `pyarrow.dataset.dataset` directory"
+            " as the underlying data, please use one of the following suffixes"
+            f" for the object name: {', '.join(suffixes[1:])}."
+            f" Or no suffix for a folder with {', '.join(PYARROW_SUFFIXES)} files"
+            " (no mixing allowed)."
         )
     if self.suffix != ".tiledbsoma" and self.key != "soma" and mode != "r":
         raise ValueError("Only a tiledbsoma store can be openened with `mode!='r'`.")

lamindb/_can_validate.py

@@ -10,10 +10,10 @@ from lamin_utils import colors, logger
 from lamindb_setup.core._docs import doc_args
 from lnschema_core import CanValidate, Record
 
-from lamindb._utils import attach_func_to_class_method
-
 from ._from_values import _has_organism_field, _print_values, get_or_create_records
 from ._record import _queryset, get_name_field
+from ._utils import attach_func_to_class_method
+from .core.exceptions import ValidationError
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
@@ -496,9 +496,9 @@ def _add_or_remove_synonyms(
                 " with the following records:\n"
             )
             display(records_df)
-            raise ValueError(
-                "cannot assigned a synonym that is already associated with a record to a different record.\n"
-                "Consider removing the synonym from existing records or using a different synonym."
+            raise ValidationError(
+                f"you are trying to assign a synonym to record: {record}\n"
+                "    → consider removing the synonym from existing records or using a different synonym."
             )
 
     # passed synonyms
@@ -516,7 +516,7 @@ def _add_or_remove_synonyms(
         return
     # because we use | as the separator
     if any("|" in i for i in syn_new_set):
-        raise ValueError("a synonym can't contain '|'!")
+        raise ValidationError("a synonym can't contain '|'!")
 
     # existing synonyms
     syns_exist = record.synonyms

lamindb/_collection.py

@@ -20,20 +20,21 @@ from lnschema_core.models import (
 )
 from lnschema_core.types import VisibilityChoice
 
-from lamindb._utils import attach_func_to_class_method
-from lamindb.core._data import _track_run_input, describe, view_lineage
-from lamindb.core._mapped_collection import MappedCollection
-from lamindb.core.versioning import process_revises
-
 from . import Artifact, Run
 from ._record import init_self_from_db, update_attributes
+from ._utils import attach_func_to_class_method
 from .core._data import (
+    _track_run_input,
     add_transform_to_kwargs,
+    describe,
     get_run,
     save_feature_set_links,
     save_feature_sets,
+    view_lineage,
 )
+from .core._mapped_collection import MappedCollection
 from .core._settings import settings
+from .core.versioning import process_revises
 
 if TYPE_CHECKING:
     from collections.abc import Iterable

lamindb/_curate.py

@@ -184,7 +184,7 @@ class DataFrameCurator(BaseCurator):
     def non_validated(self) -> list:
         """Return the non-validated features and labels."""
         if self._non_validated is None:
-            raise ValueError("Please run validate() first!")
+            raise ValidationError("Please run validate() first!")
         return self._non_validated
 
     @property
@@ -222,7 +222,7 @@ class DataFrameCurator(BaseCurator):
             valid_keys = set(self._df.columns) | {"columns"} | extra
             nonval_keys = [key for key in d.keys() if key not in valid_keys]
             if len(nonval_keys) > 0:
-                raise ValueError(
+                raise ValidationError(
                     f"the following keys passed to {name} are not allowed: {nonval_keys}"
                 )
 
@@ -288,9 +288,11 @@ class DataFrameCurator(BaseCurator):
             self._save_columns(validated_only=validated_only, **kwargs)
         else:
             if categorical not in self.fields:
-                raise ValueError(f"Feature {categorical} is not part of the fields!")
+                raise ValidationError(
+                    f"Feature {categorical} is not part of the fields!"
+                )
             update_registry(
-                values=self._df[categorical].unique().tolist(),
+                values=flatten_unique(self._df[categorical]),
                 field=self.fields[categorical],
                 key=categorical,
                 using_key=self._using_key,
@@ -508,13 +510,13 @@ class AnnDataCurator(DataFrameCurator):
             exclude=self._exclude.get("var_index"),
         )
 
-    def _update_registry_all(self):
+    def _update_registry_all(self, validated_only: bool = True, **kwargs):
         """Save labels for all features."""
         logger.info("saving validated records of 'var_index'")
-        self._save_from_var_index(validated_only=True, **self._kwargs)
+        self._save_from_var_index(validated_only=validated_only, **self._kwargs)
         for name in self._obs_fields.keys():
             logger.info(f"saving validated terms of '{name}'")
-            self._update_registry(name, validated_only=True, **self._kwargs)
+            self._update_registry(name, validated_only=validated_only, **self._kwargs)
 
     def add_new_from_var_index(self, organism: str | None = None, **kwargs):
         """Update variable records.
@@ -704,7 +706,7 @@ class MuDataCurator:
         """Verify the modality exists."""
         for modality in modalities:
             if modality not in self._mdata.mod.keys():
-                raise ValueError(f"modality '{modality}' does not exist!")
+                raise ValidationError(f"modality '{modality}' does not exist!")
 
     def _save_from_var_index_modality(
         self, modality: str, validated_only: bool = True, **kwargs
@@ -729,7 +731,7 @@ class MuDataCurator:
         obs_fields: dict[str, dict[str, FieldAttr]] = {}
         for k, v in categoricals.items():
             if k not in self._mdata.obs.columns:
-                raise ValueError(f"column '{k}' does not exist in mdata.obs!")
+                raise ValidationError(f"column '{k}' does not exist in mdata.obs!")
             if any(k.startswith(prefix) for prefix in prefixes):
                 modality, col = k.split(":")[0], k.split(":")[1]
                 if modality not in obs_fields.keys():
@@ -1120,7 +1122,7 @@ def check_registry_organism(registry: Record, organism: str | None = None) -> di
         import bionty as bt
 
         if organism is None and bt.settings.organism is None:
-            raise ValueError(
+            raise ValidationError(
                 f"{registry.__name__} registry requires an organism!\n"
                 "      → please pass an organism name via organism="
             )
@@ -1148,8 +1150,8 @@ def validate_categories(
         using_key: A reference LaminDB instance.
         organism: The organism name.
         source: The source record.
-        exclude: Exclude specific values.
-        standardize: Standardize the values.
+        exclude: Exclude specific values from validation.
+        standardize: Whether to standardize the values.
         validated_hint_print: The hint to print for validated values.
     """
     from lamindb._from_values import _print_values
@@ -1210,12 +1212,15 @@ def validate_categories(
 
     validated_hint_print = validated_hint_print or f".add_validated_from('{key}')"
     n_validated = len(values_validated)
+
     if n_validated > 0:
         _log_mapping_info()
+        terms_str = f"{', '.join([f'{chr(39)}{v}{chr(39)}' for v in values_validated[:10]])}{', ...' if len(values_validated) > 10 else ''}"
+        val_numerous = "" if n_validated == 1 else "s"
         logger.warning(
-            f"found {colors.yellow(n_validated)} validated terms: "
-            f"{colors.yellow(values_validated)}\n      → save terms via "
-            f"{colors.yellow(validated_hint_print)}"
+            f"found {colors.yellow(n_validated)} validated term{val_numerous}: "
+            f"{colors.yellow(terms_str)}\n"
+            f"→ save term{val_numerous} via {colors.yellow(validated_hint_print)}"
         )
 
     non_validated_hint_print = validated_hint_print.replace("_validated_", "_new_")
@@ -1230,13 +1235,15 @@ def validate_categories(
             # validated values still need to be saved to the current instance
             return False, []
     else:
-        are = "are" if n_non_validated > 1 else "is"
+        non_val_numerous = ("", "is") if n_non_validated == 1 else ("s", "are")
         print_values = _print_values(non_validated)
         warning_message = (
-            f"{colors.red(f'{n_non_validated} terms')} {are} not validated: "
-            f"{colors.red(print_values)}\n      → fix typos, remove non-existent values, or save terms via "
+            f"{colors.red(f'{n_non_validated} term{non_val_numerous[0]}')} {non_val_numerous[1]} not validated: "
+            f"{colors.red(', '.join(print_values.split(', ')[:10]) + ', ...' if len(print_values.split(', ')) > 10 else print_values)}\n"
+            f"→ fix typo{non_val_numerous[0]}, remove non-existent value{non_val_numerous[0]}, or save term{non_val_numerous[0]} via "
             f"{colors.red(non_validated_hint_print)}"
         )
+
         if logger.indent == "":
             _log_mapping_info()
         logger.warning(warning_message)
@@ -1427,6 +1434,19 @@ def save_artifact(
     return artifact
 
 
+def flatten_unique(series):
+    """Flatten a pandas series if it contains lists."""
+    result = set()
+
+    for item in series:
+        if isinstance(item, list):
+            result.update(item)
+        else:
+            result.add(item)
+
+    return list(result)
+
+
 def update_registry(
     values: list[str],
     field: FieldAttr,
@@ -1596,24 +1616,25 @@ def log_saved_labels(
             continue
 
         if k == "without reference" and validated_only:
-            msg = colors.yellow(
-                f"{len(labels)} non-validated values are not saved in {model_field}: {labels}!"
-            )
-            lookup_print = (
-                f"lookup().{key}" if key.isidentifier() else f".lookup()['{key}']"
-            )
-
-            hint = f".add_new_from('{key}')"
-            msg += f"\n      → to lookup values, use {lookup_print}"
-            msg += (
-                f"\n      → to save, run {colors.yellow(hint)}"
-                if save_function == "add_new_from"
-                else f"\n      → to save, run {colors.yellow(save_function)}"
-            )
-            if warning:
-                logger.warning(msg)
-            else:
-                logger.info(msg)
+            continue
+            # msg = colors.yellow(
+            #     f"{len(labels)} non-validated values are not saved in {model_field}: {labels}!"
+            # )
+            # lookup_print = (
+            #     f"lookup().{key}" if key.isidentifier() else f".lookup()['{key}']"
+            # )
+
+            # hint = f".add_new_from('{key}')"
+            # msg += f"\n      → to lookup values, use {lookup_print}"
+            # msg += (
+            #     f"\n      → to save, run {colors.yellow(hint)}"
+            #     if save_function == "add_new_from"
+            #     else f"\n      → to save, run {colors.yellow(save_function)}"
+            # )
+            # if warning:
+            #     logger.warning(msg)
+            # else:
+            #     logger.info(msg)
         else:
             k = "" if k == "without reference" else f"{colors.green(k)} "
             # the term "transferred" stresses that this is always in the context of transferring
@@ -1631,8 +1652,8 @@ def save_ulabels_with_parent(values: list[str], field: FieldAttr, key: str) -> N
     all_records = registry.from_values(list(values), field=field)
     is_feature = registry.filter(name=f"is_{key}").one_or_none()
     if is_feature is None:
-        is_feature = registry(name=f"is_{key}")
-        is_feature.save()
+        is_feature = registry(name=f"is_{key}").save()
+        logger.important(f"Created a parent ULabel: {is_feature}")
     is_feature.children.add(*all_records)
 
 
@@ -1689,7 +1710,7 @@ def _save_organism(name: str):  # pragma: no cover
     if organism is None:
         organism = bt.Organism.from_source(name=name)
         if organism is None:
-            raise ValueError(
+            raise ValidationError(
                 f"Organism '{name}' not found\n"
                 f"      → please save it: bt.Organism(name='{name}').save()"
             )

lamindb/_feature.py

@@ -8,10 +8,9 @@ from lamindb_setup.core._docs import doc_args
 from lnschema_core.models import Artifact, Feature
 from pandas.api.types import CategoricalDtype, is_string_dtype
 
-from lamindb._utils import attach_func_to_class_method
-from lamindb.core._settings import settings
-
 from ._query_set import RecordsList
+from ._utils import attach_func_to_class_method
+from .core._settings import settings
 from .core.schema import dict_schema_name_to_model_name
 
 if TYPE_CHECKING:

lamindb/_feature_set.py

@@ -10,10 +10,9 @@ from lamindb_setup.core.hashing import hash_set
 from lnschema_core import Feature, FeatureSet, Record, ids
 from lnschema_core.types import FieldAttr, ListLike
 
-from lamindb._utils import attach_func_to_class_method
-
 from ._feature import convert_numpy_dtype_to_lamin_feature_type
 from ._record import init_self_from_db
+from ._utils import attach_func_to_class_method
 from .core.exceptions import ValidationError
 from .core.schema import (
     dict_related_model_to_related_name,

lamindb/_is_versioned.py

@@ -5,8 +5,7 @@ from lamin_utils import logger
 from lamindb_setup.core.upath import UPath
 from lnschema_core.models import IsVersioned
 
-from lamindb._utils import attach_func_to_class_method
-
+from ._utils import attach_func_to_class_method
 from .core.versioning import create_uid, get_new_path_from_uid
 
 

lamindb/_parents.py

@@ -8,13 +8,14 @@ from lamin_utils import logger
 from lnschema_core import Artifact, Collection, Record, Run, Transform
 from lnschema_core.models import HasParents, format_field_value
 
-from lamindb._utils import attach_func_to_class_method
-
 from ._record import get_name_field
+from ._utils import attach_func_to_class_method
 
 if TYPE_CHECKING:
     from lnschema_core.types import StrField
 
+    from lamindb.core import QuerySet
+
 LAMIN_GREEN_LIGHTER = "#10b981"
 LAMIN_GREEN_DARKER = "#065f46"
 GREEN_FILL = "honeydew"
@@ -22,6 +23,30 @@ TRANSFORM_EMOJIS = {"notebook": "📔", "app": "🖥️", "pipeline": "🧩"}
 is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
 
 
+# this is optimized to have fewer recursive calls
+# also len of QuerySet can be costly at times
+def _query_relatives(
+    records: QuerySet | list[Record],
+    kind: Literal["parents", "children"],
+    cls: type[HasParents],
+) -> QuerySet:
+    relatives = cls.objects.none()
+    if len(records) == 0:
+        return relatives
+    for record in records:
+        relatives = relatives.union(getattr(record, kind).all())
+    relatives = relatives.union(_query_relatives(relatives, kind, cls))
+    return relatives
+
+
+def query_parents(self) -> QuerySet:
+    return _query_relatives([self], "parents", self.__class__)
+
+
+def query_children(self) -> QuerySet:
+    return _query_relatives([self], "children", self.__class__)
+
+
 def _transform_emoji(transform: Transform):
     if transform is not None:
         return TRANSFORM_EMOJIS.get(transform.type, "💫")
@@ -474,9 +499,7 @@ def _df_edges_from_runs(df_values: list):
     return df
 
 
-METHOD_NAMES = [
-    "view_parents",
-]
+METHOD_NAMES = ["view_parents", "query_parents", "query_children"]
 
 if ln_setup._TESTING:  # type: ignore
     from inspect import signature

lamindb/_query_manager.py

@@ -7,9 +7,8 @@ from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
 from lnschema_core.models import Record
 
-from lamindb.core._settings import settings
-
 from .core._feature_manager import get_feature_set_by_slot_
+from .core._settings import settings
 
 if TYPE_CHECKING:
     from lnschema_core.types import StrField

lamindb/_query_set.py

@@ -20,7 +20,7 @@ from lnschema_core.models import (
     VisibilityChoice,
 )
 
-from lamindb.core.exceptions import DoesNotExist
+from .core.exceptions import DoesNotExist
 
 if TYPE_CHECKING:
     from collections.abc import Iterable

lamindb/_record.py

@@ -19,9 +19,9 @@ from lamindb_setup.core._hub_core import connect_instance_hub
 from lamindb_setup.core._settings_store import instance_settings_file
 from lnschema_core.models import Artifact, Feature, IsVersioned, Record, Run, Transform
 
-from lamindb._utils import attach_func_to_class_method
-from lamindb.core._settings import settings
-from lamindb.core.exceptions import RecordNameChangeIntegrityError
+from ._utils import attach_func_to_class_method
+from .core._settings import settings
+from .core.exceptions import RecordNameChangeIntegrityError
 
 if TYPE_CHECKING:
     import pandas as pd

lamindb/_save.py

@@ -15,8 +15,8 @@ from lamin_utils import logger
 from lamindb_setup.core.upath import LocalPathClasses
 from lnschema_core.models import Artifact, Record
 
-from lamindb.core._settings import settings
-from lamindb.core.storage.paths import (
+from .core._settings import settings
+from .core.storage.paths import (
     _cache_key_from_artifact_storage,
     attempt_accessing_path,
     auto_storage_key_from_artifact,

lamindb/_transform.py

@@ -6,10 +6,9 @@ from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
 from lnschema_core.models import Run, Transform
 
-from lamindb.core.exceptions import InconsistentKey
-
 from ._parents import _view_parents
 from ._run import delete_run_artifacts
+from .core.exceptions import InconsistentKey
 from .core.versioning import message_update_key_in_version_family, process_revises
 
 if TYPE_CHECKING:

lamindb/_ulabel.py

@@ -6,7 +6,7 @@ import lamindb_setup as ln_setup
 from lamindb_setup.core._docs import doc_args
 from lnschema_core import ULabel
 
-from lamindb._utils import attach_func_to_class_method
+from ._utils import attach_func_to_class_method
 
 if TYPE_CHECKING:
     from lnschema_core.types import ListLike

lamindb/core/_mapped_collection.py

@@ -69,7 +69,7 @@ class MappedCollection:
 
     .. note::
 
-        For a guide, see :doc:`docs:scrna5`.
+        For a guide, see :doc:`docs:scrna-mappedcollection`.
 
         For more convenient use within :class:`~lamindb.core.MappedCollection`,
         see :meth:`~lamindb.Collection.mapped`.

lamindb/core/storage/_backed_access.py

@@ -7,11 +7,13 @@ from anndata._io.specs.registry import get_spec
 from lnschema_core import Artifact
 
 from ._anndata_accessor import AnnDataAccessor, StorageType, registry
+from ._pyarrow_dataset import _is_pyarrow_dataset, _open_pyarrow_dataset
 from ._tiledbsoma import _open_tiledbsoma
 from .paths import filepath_from_artifact
 
 if TYPE_CHECKING:
     from fsspec.core import OpenFile
+    from pyarrow.dataset import Dataset as PyArrowDataset
     from tiledbsoma import Collection as SOMACollection
     from tiledbsoma import Experiment as SOMAExperiment
     from upath import UPath
@@ -67,22 +69,28 @@ def backed_access(
     artifact_or_filepath: Artifact | UPath,
     mode: str = "r",
     using_key: str | None = None,
-) -> AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment:
+) -> (
+    AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment | PyArrowDataset
+):
     if isinstance(artifact_or_filepath, Artifact):
-        filepath, _ = filepath_from_artifact(artifact_or_filepath, using_key=using_key)
+        objectpath, _ = filepath_from_artifact(
+            artifact_or_filepath, using_key=using_key
+        )
     else:
-        filepath = artifact_or_filepath
-    name = filepath.name
-    suffix = filepath.suffix
+        objectpath = artifact_or_filepath
+    name = objectpath.name
+    suffix = objectpath.suffix
 
     if name == "soma" or suffix == ".tiledbsoma":
         if mode not in {"r", "w"}:
             raise ValueError("`mode` should be either 'r' or 'w' for tiledbsoma.")
-        return _open_tiledbsoma(filepath, mode=mode)  # type: ignore
+        return _open_tiledbsoma(objectpath, mode=mode)  # type: ignore
     elif suffix in {".h5", ".hdf5", ".h5ad"}:
-        conn, storage = registry.open("h5py", filepath, mode=mode)
+        conn, storage = registry.open("h5py", objectpath, mode=mode)
     elif suffix == ".zarr":
-        conn, storage = registry.open("zarr", filepath, mode=mode)
+        conn, storage = registry.open("zarr", objectpath, mode=mode)
+    elif _is_pyarrow_dataset(objectpath):
+        return _open_pyarrow_dataset(objectpath)
     else:
         raise ValueError(
             "object should have .h5, .hdf5, .h5ad, .zarr, .tiledbsoma suffix, not"

lamindb/core/storage/_pyarrow_dataset.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pyarrow.dataset
+from lamindb_setup.core.upath import LocalPathClasses
+
+if TYPE_CHECKING:
+    from upath import UPath
+
+
+PYARROW_SUFFIXES = (".parquet", ".csv", ".json", ".orc", ".arrow", ".feather")
+
+
+def _is_pyarrow_dataset(path: UPath) -> bool:
+    # it is assumed here that path exists
+    if path.is_file():
+        return path.suffix in PYARROW_SUFFIXES
+    else:
+        objects = path.rglob("*")
+        suffixes = {object.suffix for object in objects if object.suffix != ""}
+        return len(suffixes) == 1 and suffixes.pop() in PYARROW_SUFFIXES
+
+
+def _open_pyarrow_dataset(path: UPath) -> pyarrow.dataset.Dataset:
+    if isinstance(path, LocalPathClasses):
+        path_str, filesystem = path.as_posix(), None
+    else:
+        path_str, filesystem = path.path, path.fs
+
+    return pyarrow.dataset.dataset(path_str, filesystem=filesystem)

{lamindb-0.76.14.dist-info → lamindb-0.76.15.dist-info}/METADATA

@@ -1,14 +1,15 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.14
+Version: 0.76.15
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
-Requires-Python: >=3.9
+Requires-Python: >=3.9,<3.13
 Description-Content-Type: text/markdown
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.76.0
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: lnschema_core==0.76.1
 Requires-Dist: lamin_utils==0.13.7
 Requires-Dist: lamin_cli==0.20.1
 Requires-Dist: lamindb_setup
@@ -17,9 +18,7 @@ Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
 Requires-Dist: python-dateutil
 Requires-Dist: anndata>=0.8.0,<=0.10.9
-Requires-Dist: scipy<1.13.0rc1
 Requires-Dist: fsspec
-Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
@@ -35,7 +34,7 @@ Requires-Dist: mudata ; extra == "dev"
 Requires-Dist: nbproject_test>=0.5.1 ; extra == "dev"
 Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
-Requires-Dist: readfcs>=1.1.8 ; extra == "fcs"
+Requires-Dist: readfcs>=1.1.9 ; extra == "fcs"
 Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
 Requires-Dist: nbproject==0.10.5 ; extra == "jupyter"
 Requires-Dist: jupytext ; extra == "jupyter"

{lamindb-0.76.14.dist-info → lamindb-0.76.15.dist-info}/RECORD

@@ -1,23 +1,23 @@
-lamindb/__init__.py,sha256=PelFCbUaNuz_1rnpOqj-mDCMqCfT2hJ27M3C-VkGJl0,2278
-lamindb/_artifact.py,sha256=9soXXT3g9tG1BTwX4DCRPurbFFcpHKOT8W7SsoLiNbo,44847
-lamindb/_can_validate.py,sha256=1pUavLwZ_yPAtbVYKOGYUHaPxlJGZ246qZ0e-4ZUDSc,19552
-lamindb/_collection.py,sha256=vt604fUjkmOYCGR4Sq_NTwnPywATfjUAdkQjuJJ17y0,14613
-lamindb/_curate.py,sha256=rFbPEoD-E-5s3QPIcUuedUO6a2c8QfpTrBfVX9gUVpE,63120
-lamindb/_feature.py,sha256=nZhtrH0ssoNls-hV-dkwfK9sKypg2El59R9qfarxfUE,5340
-lamindb/_feature_set.py,sha256=JQSP-YLam1KW-rDzly5Dm4IYVL2A6ec7ufIf6iCc2W8,8169
+lamindb/__init__.py,sha256=ZoRboX4PQc0sbVajeZ1fd4GoLN3YDj5U74zVUpcSd9I,2278
+lamindb/_artifact.py,sha256=WNdKAJFu3sFgQ_Qe1JflDHiTP4EGhAIkzaHFwTthAjY,44903
+lamindb/_can_validate.py,sha256=TKfkHgkPl1bwuJrQCp6pHgQbi7m2pc_zjFhZfUAUU20,19573
+lamindb/_collection.py,sha256=MLOEoOgTu7rTlRD7zkm1k0YIk_gVhQDO17JbmZCptOs,14573
+lamindb/_curate.py,sha256=pLsdtnnTn0qQbsUB4hrU3yuoVnVNh-BQe10EpMNZ4Ns,64083
+lamindb/_feature.py,sha256=9cgrcHoyOa1jpON-9KiUfFSHcxiGECiefUAqAx4cVvU,5325
+lamindb/_feature_set.py,sha256=WdXw_YGlMXCs8l0WVHOrqvvrH2hsQLqCiho8LFDYwhI,8161
 lamindb/_filter.py,sha256=Pf9NHV4gm7NOC0Frtvx4W7nvwt2EowOP74DwppyXAZs,635
 lamindb/_finish.py,sha256=VMAmxCUFmTKIMSCx7LEh4QAnWDeue6MeUAAzkMVEYMU,9546
 lamindb/_from_values.py,sha256=uRtZLaMWKoANMMXm1hrADHfckRCTiK8_d02Yp07nLkw,14119
-lamindb/_is_versioned.py,sha256=5lAnhTboltFkZCKVRV1uxkm0OCjJz_HKi3yQq_vEuMs,1306
-lamindb/_parents.py,sha256=KMBUfCLNqjmFzOdZIXaUFqDPeEpWP28MCkHHPq887h8,16341
-lamindb/_query_manager.py,sha256=pmPhJQ85-7XeAU9TFv6LPGi9F7dBgztZgZcXz33HYJM,3710
-lamindb/_query_set.py,sha256=AyWvFZ-Vnd_1dhbDLkiyEh2-2XiIR_OpEk72xoQ2JVg,12980
-lamindb/_record.py,sha256=FkU7G1OUl0HPQO6wh8EkPh4T_ogxcy6QGkrVz_I4WUw,26840
+lamindb/_is_versioned.py,sha256=GWZk-usV6aB33Cl9AlrnEGE5nxUkZic7QJzOW_DrwQA,1298
+lamindb/_parents.py,sha256=INhbqh6IaUjuYVUOp-6rnOGN-8kGZirNqqW9XQ1qz_M,17119
+lamindb/_query_manager.py,sha256=noc05Ad-aADxckOVBVDAiErFB7gL8XTgckELvI4rGmM,3702
+lamindb/_query_set.py,sha256=6vHOvB_uXzKVVIw8AAVw7EYOIAGuw3TYcUzkpNlFLdE,12973
+lamindb/_record.py,sha256=isR9GMQFwUUwVSmPNABvEzcJS38TbjhD7Cc6kygPsTA,26819
 lamindb/_run.py,sha256=K_5drpLn3D7y3XtZ3vtAw35rG5RCSvB4bXQZx4ESSI0,1964
-lamindb/_save.py,sha256=BCaSFnANYPxTqL5gw7Hrh_9kz7SDyOxrJV2KW6rXqts,11366
+lamindb/_save.py,sha256=OD052Qr_hiMyAonHTktKETe_Bhnp1RY810y0rwZqpBQ,11352
 lamindb/_storage.py,sha256=GBVChv-DHVMNEBJL5l_JT6B4RDhZ6NnwgzmUICphYKk,413
-lamindb/_transform.py,sha256=wZDkY8lp4d_OsO5a7rLs1RamkDzBXZSLaWJU34zRnmA,4728
-lamindb/_ulabel.py,sha256=XDSdZBXX_ki5s1vOths3MjF2x5DPggBR_PV_KF4SGyg,1611
+lamindb/_transform.py,sha256=HpqRCk0ZTmqxSV4nRbyvDq8fAQEE9wTj31d-CusiL6A,4720
+lamindb/_ulabel.py,sha256=DQQzAPkrOg8W9I77BJ5umajR8MQcFSvXYUy53YNN2HA,1604
 lamindb/_utils.py,sha256=LGdiW4k3GClLz65vKAVRkL6Tw-Gkx9DWAdez1jyA5bE,428
 lamindb/_view.py,sha256=4Ln2ItTb3857PAI-70O8eJYqoTJ_NNFc7E_wds6OGns,2412
 lamindb/core/__init__.py,sha256=y87MCP1BEC2qHNVDIOwqVteIP_2hPCdIoa9JXr0EG8U,1524
@@ -26,7 +26,7 @@ lamindb/core/_data.py,sha256=BVZkxK8aloSecH25LivbwnjcG1fz7Gs2UDceO5pWd3I,20049
 lamindb/core/_django.py,sha256=yeMPp1n9WrFmEjVRdavfpVqAolPLd24RseTQlvsK67w,7157
 lamindb/core/_feature_manager.py,sha256=q4WmzJvFLL_fAs-vNRgV2klanAoU6Wu8_g0O2dyIjVg,40027
 lamindb/core/_label_manager.py,sha256=yh-r4KbtOArMUKPJL75yOxJc8HUKqsik8pExBVKyDlA,10949
-lamindb/core/_mapped_collection.py,sha256=M50haewVAFONeF71QQbzD09L8lVZCL1hyf0W87jKE5U,24575
+lamindb/core/_mapped_collection.py,sha256=EDS0xzOdCc_iGE_Iqv5COTVHNm4jWue7Jtcd8DdXkJU,24591
 lamindb/core/_settings.py,sha256=6jNadlQdimxCsKR2ZyUD0YJYzOdubTnKktki-MqEWqQ,6137
 lamindb/core/_sync_git.py,sha256=lIgl6YfpH4rCFT1WILAp7zlemZfxog1d0zp3cX0KIZw,4531
 lamindb/core/_track_environment.py,sha256=Ywzg_sJ7guI1dcsN7h5orce9VdYl8VGVE3OLITlHBXQ,820
@@ -42,7 +42,8 @@ lamindb/core/datasets/_fake.py,sha256=BZF9R_1iF0HDnvtZNqL2FtsjSMuqDIfuFxnw_LJYIh
 lamindb/core/storage/__init__.py,sha256=JOIMu_7unbyhndtH1j0Q-9AvY8knSuc1IJO9sQnyBAQ,498
 lamindb/core/storage/_anndata_accessor.py,sha256=2p1HjoatmZjQ1u94tjgmXgiv8MKowrQH5xInDmiLCw4,24191
 lamindb/core/storage/_anndata_sizes.py,sha256=aXO3OB--tF5MChenSsigW6Q-RuE8YJJOUTVukkLrv9A,1029
-lamindb/core/storage/_backed_access.py,sha256=O0zazsDlW0PKa52WHV5HooHGGI81FxLT6VEvNONdiEc,3234
+lamindb/core/storage/_backed_access.py,sha256=t9iS9mlZQBy1FfIS-Twt-96npYiShbPwEo2y9_3b6jY,3517
+lamindb/core/storage/_pyarrow_dataset.py,sha256=wuLsEvdblqMdUdDfMtis8AWrE3igzvFWTSTbxuD1Oc8,926
 lamindb/core/storage/_tiledbsoma.py,sha256=0NPLS5m1icEhzWPfXAv4U2SNiLGqGQd7FM6xCm5wYEc,7269
 lamindb/core/storage/_valid_suffixes.py,sha256=vUSeQ4s01rdhD_vSd6wKmFBsgMJAKkBMnL_T9Y1znMg,501
 lamindb/core/storage/_zarr.py,sha256=TODQD3p1eykoPwP-c-YRP_UDmsbMeBGMGvkBxxOMeYc,3663
@@ -55,7 +56,7 @@ lamindb/integrations/__init__.py,sha256=RWGMYYIzr8zvmNPyVB4m-p4gMDhxdRbjES2Ed23O
 lamindb/integrations/_vitessce.py,sha256=uPl45_w4Uu9_BhpBDDVonC1nDOuAnB7DAnzi5w5bZAE,4032
 lamindb/setup/__init__.py,sha256=OwZpZzPDv5lPPGXZP7-zK6UdO4FHvvuBh439yZvIp3A,410
 lamindb/setup/core/__init__.py,sha256=SevlVrc2AZWL3uALbE5sopxBnIZPWZ1IB0NBDudiAL8,167
-lamindb-0.76.14.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamindb-0.76.14.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb-0.76.14.dist-info/METADATA,sha256=j7r4goc9s3ANslQ7-pT7WIct1uDLJK3DGuN3kPsixYs,2361
-lamindb-0.76.14.dist-info/RECORD,,
+lamindb-0.76.15.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamindb-0.76.15.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb-0.76.15.dist-info/METADATA,sha256=31K0SoWfAxBpWR3x_7PLFfwpu-mnsQ8IHFqsvjdg_cw,2365
+lamindb-0.76.15.dist-info/RECORD,,