lamindb 1.0.4__py3-none-any.whl → 1.1.0__py3-none-any.whl

lamindb/core/storage/_tiledbsoma.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, Literal
+from urllib.parse import urlparse
 
 import pandas as pd
 import pyarrow as pa
@@ -17,6 +18,7 @@ if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
     from tiledbsoma import Collection as SOMACollection
     from tiledbsoma import Experiment as SOMAExperiment
+    from tiledbsoma import Measurement as SOMAMeasurement
     from upath import UPath
 
 
@@ -36,9 +38,21 @@ def _load_h5ad_zarr(objpath: UPath):
 
 
 def _tiledb_config_s3(storepath: UPath) -> dict:
-    region = get_storage_region(storepath)
-    tiledb_config = {"vfs.s3.region": region}
     storage_options = storepath.storage_options
+    tiledb_config = {}
+
+    endpoint_url = storage_options.get("endpoint_url", None)
+    if endpoint_url is not None:
+        tiledb_config["vfs.s3.region"] = ""
+        tiledb_config["vfs.s3.use_virtual_addressing"] = "false"
+        parsed = urlparse(endpoint_url)
+        tiledb_config["vfs.s3.scheme"] = parsed.scheme
+        tiledb_config["vfs.s3.endpoint_override"] = (
+            parsed._replace(scheme="").geturl().lstrip("/")
+        )
+    else:
+        tiledb_config["vfs.s3.region"] = get_storage_region(storepath)
+
     if "key" in storage_options:
         tiledb_config["vfs.s3.aws_access_key_id"] = storage_options["key"]
     if "secret" in storage_options:
@@ -51,7 +65,7 @@ def _tiledb_config_s3(storepath: UPath) -> dict:
 
 def _open_tiledbsoma(
     storepath: UPath, mode: Literal["r", "w"] = "r"
-) -> SOMACollection | SOMAExperiment:
+) -> SOMACollection | SOMAExperiment | SOMAMeasurement:
     try:
         import tiledbsoma as soma
     except ImportError as e:
@@ -71,6 +85,8 @@ def _open_tiledbsoma(
     soma_objects = [obj.name for obj in storepath.iterdir()]
     if "obs" in soma_objects and "ms" in soma_objects:
         SOMAType = soma.Experiment
+    elif "var" in soma_objects:
+        SOMAType = soma.Measurement
     else:
         SOMAType = soma.Collection
     return SOMAType.open(storepath_str, mode=mode, context=ctx)
@@ -134,17 +150,17 @@ def save_tiledbsoma_experiment(
         )
         storepath = setup_settings.storage.root / storage_key
 
-    if storepath.protocol == "s3":
+    if storepath.protocol == "s3":  # type: ignore
         ctx = soma.SOMATileDBContext(tiledb_config=_tiledb_config_s3(storepath))
     else:
         ctx = None
 
-    storepath = storepath.as_posix()
+    storepath_str = storepath.as_posix()
 
     add_run_uid = True
     run_uid_dtype = "category"
     if appending:
-        with soma.Experiment.open(storepath, mode="r", context=ctx) as store:
+        with soma.Experiment.open(storepath_str, mode="r", context=ctx) as store:
             obs_schema = store["obs"].schema
             add_run_uid = "lamin_run_uid" in obs_schema.names
             # this is needed to enable backwards compatibility with tiledbsoma stores
@@ -175,7 +191,7 @@ def save_tiledbsoma_experiment(
     registration_mapping = kwargs.get("registration_mapping", None)
     if registration_mapping is None and (appending or len(adata_objects) > 1):
         registration_mapping = soma_io.register_anndatas(
-            experiment_uri=storepath if appending else None,
+            experiment_uri=storepath_str if appending else None,
             adatas=adata_objects,
             measurement_name=measurement_name,
             obs_field_name=obs_id_name,
@@ -195,19 +211,19 @@ def save_tiledbsoma_experiment(
         assert len(adata_objects) == 1  # noqa: S101
         n_observations = adata_objects[0].n_obs
 
-    logger.important(f"Writing the tiledbsoma store to {storepath}")
+    logger.important(f"Writing the tiledbsoma store to {storepath_str}")
     for adata_obj in adata_objects:
-        if resize_experiment and soma.Experiment.exists(storepath, context=ctx):
+        if resize_experiment and soma.Experiment.exists(storepath_str, context=ctx):
             # can only happen if registration_mapping is not None
             soma_io.resize_experiment(
-                storepath,
+                storepath_str,
                 nobs=n_observations,
                 nvars=registration_mapping.get_var_shapes(),
                 context=ctx,
             )
             resize_experiment = False
         soma_io.from_anndata(
-            storepath,
+            storepath_str,
             adata_obj,
             measurement_name,
             context=ctx,
@@ -217,7 +233,7 @@ def save_tiledbsoma_experiment(
             **kwargs,
         )
 
-    artifact = Artifact(
+    artifact = Artifact(  # type: ignore
         storepath,
         key=key,
         description=description,
@@ -229,3 +245,31 @@ def save_tiledbsoma_experiment(
     artifact.otype = "tiledbsoma"
 
     return artifact.save()
+
+
+# this is less defensive than _anndata_n_observations
+# this doesn't really catches errors
+# assumes that the tiledbsoma object is well-formed
+def _soma_store_n_observations(obj) -> int:
+    if obj.soma_type in {"SOMADataFrame", "SOMASparseNDArray", "SOMADenseNDArray"}:
+        return obj.non_empty_domain()[0][1] + 1
+    elif obj.soma_type == "SOMAExperiment":
+        return _soma_store_n_observations(obj["obs"])
+    elif obj.soma_type == "SOMAMeasurement":
+        keys = obj.keys()
+        for slot in ("X", "obsm", "obsp"):
+            if slot in keys:
+                return _soma_store_n_observations(next(iter(obj[slot].values())))
+    elif obj.soma_type == "SOMACollection":
+        n_obs = 0
+        for value in obj.values():
+            n_obs += _soma_store_n_observations(value)
+        return n_obs
+    raise ValueError(
+        "Could not infer the number of observations from the tiledbsoma object."
+    )
+
+
+def _soma_n_observations(objectpath: UPath) -> int:
+    with _open_tiledbsoma(objectpath, mode="r") as store:
+        return _soma_store_n_observations(store)

lamindb/core/storage/paths.py

@@ -4,7 +4,6 @@ import shutil
 from typing import TYPE_CHECKING
 
 import fsspec
-from lamin_utils import logger
 from lamindb_setup.core import StorageSettings
 from lamindb_setup.core.upath import (
     LocalPathClasses,
@@ -42,25 +41,27 @@ def auto_storage_key_from_artifact_uid(uid: str, suffix: str, is_dir: bool) -> s
     return storage_key
 
 
-def check_path_is_child_of_root(path: UPathStr, root: UPathStr) -> bool:
-    # str is needed to eliminate UPath storage_options
-    # from the equality checks below
-    # and for fsspec.utils.get_protocol
-    path_str = str(path)
-    root_str = str(root)
-    root_protocol = fsspec.utils.get_protocol(root_str)
-    # check that the protocols are the same first
-    if fsspec.utils.get_protocol(path_str) != root_protocol:
-        return False
-    if root_protocol in {"http", "https"}:
-        # in this case it is a base url, not a file
-        # so formally does not exist
+def _safely_resolve(upath: UPath) -> UPath:
+    if upath.protocol in {"http", "https"}:
         resolve_kwargs = {"follow_redirects": False}
     else:
         resolve_kwargs = {}
-    return (
-        UPath(root_str).resolve(**resolve_kwargs) in UPath(path_str).resolve().parents
-    )
+    return upath.resolve(**resolve_kwargs)
+
+
+def check_path_is_child_of_root(path: UPathStr, root: UPathStr) -> bool:
+    if fsspec.utils.get_protocol(str(path)) != fsspec.utils.get_protocol(str(root)):
+        return False
+    path_upath = _safely_resolve(UPath(path))
+    root_upath = _safely_resolve(UPath(root))
+    if path_upath.protocol == "s3":
+        endpoint_path = path_upath.storage_options.get("endpoint_url", "")
+        endpoint_root = root_upath.storage_options.get("endpoint_url", "")
+        if endpoint_path != endpoint_root:
+            return False
+    # str is needed to eliminate UPath storage_options
+    # which affect equality checks
+    return UPath(str(root_upath)) in UPath(str(path_upath)).parents
 
 
 # returns filepath and root of the storage
@@ -139,7 +140,20 @@ def store_file_or_folder(
     local_path = UPath(local_path)
     if not isinstance(storage_path, LocalPathClasses):
         # this uploads files and directories
-        create_folder = False if local_path.is_dir() else None
+        if local_path.is_dir():
+            create_folder = False
+            try:
+                # if storage_path already exists we need to delete it
+                # if local_path is a directory
+                # to replace storage_path correctly
+                if storage_path.stat().as_info()["type"] == "directory":
+                    storage_path.rmdir()
+                else:
+                    storage_path.unlink()
+            except (FileNotFoundError, PermissionError):
+                pass
+        else:
+            create_folder = None
         storage_path.upload_from(
             local_path, create_folder=create_folder, print_progress=print_progress
         )
@@ -156,10 +170,15 @@ def store_file_or_folder(
 
 
 def delete_storage_using_key(
-    artifact: Artifact, storage_key: str, using_key: str | None
-):
+    artifact: Artifact,
+    storage_key: str,
+    raise_file_not_found_error: bool = True,
+    using_key: str | None = None,
+) -> None | str:
     filepath, _ = attempt_accessing_path(artifact, storage_key, using_key=using_key)
-    delete_storage(filepath)
+    return delete_storage(
+        filepath, raise_file_not_found_error=raise_file_not_found_error
+    )
 
 
 def delete_storage(
@@ -178,5 +197,5 @@ def delete_storage(
     elif raise_file_not_found_error:
         raise FileNotFoundError(f"{storagepath} is not an existing path!")
     else:
-        logger.warning(f"{storagepath} is not an existing path!")
+        return "did-not-delete"
     return None

lamindb/core/subsettings/_creation_settings.py

@@ -1,13 +1,8 @@
-from typing import Literal
-
-
 class CreationSettings:
-    artifact_if_hash_exists: Literal[
-        "warn_return_existing", "error", "warn_create_new"
-    ] = "warn_return_existing"
-    """Behavior if file hash exists (default `"warn_return_existing"`).
+    search_names: bool = True
+    """Switch off to speed up creating records (default `True`).
 
-    One of `["warn_return_existing", "error", "warn_create_new"]`.
+    If `True`, search for alternative names and avoids duplicates.
 
     FAQ: :doc:`/faq/idempotency`
     """
@@ -18,15 +13,8 @@ class CreationSettings:
 
     It speeds up file creation by about a factor 100.
     """
-    search_names: bool = True
-    """To speed up creating records (default `True`).
-
-    If `True`, search for alternative names.
-
-    FAQ: :doc:`/faq/idempotency`
-    """
     artifact_silence_missing_run_warning: bool = False
-    """Silence warning about missing run & transform during artifact creation."""
+    """Silence warning about missing run & transform during artifact creation (default `False`)."""
     _artifact_use_virtual_keys: bool = True
     """Treat `key` parameter in :class:`~lamindb.Artifact` as virtual.