lamindb 0.76.8__py3-none-any.whl → 0.76.10__py3-none-any.whl

lamindb/core/storage/objects.py

@@ -1,62 +1,62 @@
-from __future__ import annotations
-
-from pathlib import PurePosixPath
-from typing import TYPE_CHECKING
-
-from anndata import AnnData
-from pandas import DataFrame
-
-if TYPE_CHECKING:
-    from lamindb_setup.core.types import UPathStr
-
-
-def _mudata_is_installed():
-    try:
-        import mudata
-    except ImportError:
-        return False
-    return True
-
-
-def infer_suffix(dmem, adata_format: str | None = None):
-    """Infer LaminDB storage file suffix from a data object."""
-    if isinstance(dmem, AnnData):
-        if adata_format is not None:
-            if adata_format not in {"h5ad", "zarr", "anndata.zarr"}:
-                raise ValueError(
-                    "Error when specifying AnnData storage format, it should be"
-                    f" 'h5ad', 'zarr', not '{adata_format}'. Check 'format'"
-                    " or the suffix of 'key'."
-                )
-            return "." + adata_format
-        return ".h5ad"
-    elif isinstance(dmem, DataFrame):
-        return ".parquet"
-    else:
-        if _mudata_is_installed():
-            from mudata import MuData
-
-            if isinstance(dmem, MuData):
-                return ".h5mu"
-        raise NotImplementedError
-
-
-def write_to_disk(dmem, filepath: UPathStr):
-    if isinstance(dmem, AnnData):
-        suffix = PurePosixPath(filepath).suffix
-        if suffix == ".h5ad":
-            dmem.write_h5ad(filepath)
-        elif suffix == ".zarr":
-            dmem.write_zarr(filepath)
-        else:
-            raise NotImplementedError
-    elif isinstance(dmem, DataFrame):
-        dmem.to_parquet(filepath)
-    else:
-        if _mudata_is_installed():
-            from mudata import MuData
-
-            if isinstance(dmem, MuData):
-                dmem.write(filepath)
-                return
-        raise NotImplementedError
+from __future__ import annotations
+
+from pathlib import PurePosixPath
+from typing import TYPE_CHECKING
+
+from anndata import AnnData
+from pandas import DataFrame
+
+if TYPE_CHECKING:
+    from lamindb_setup.core.types import UPathStr
+
+
+def _mudata_is_installed():
+    try:
+        import mudata
+    except ImportError:
+        return False
+    return True
+
+
+def infer_suffix(dmem, adata_format: str | None = None):
+    """Infer LaminDB storage file suffix from a data object."""
+    if isinstance(dmem, AnnData):
+        if adata_format is not None:
+            if adata_format not in {"h5ad", "zarr", "anndata.zarr"}:
+                raise ValueError(
+                    "Error when specifying AnnData storage format, it should be"
+                    f" 'h5ad', 'zarr', not '{adata_format}'. Check 'format'"
+                    " or the suffix of 'key'."
+                )
+            return "." + adata_format
+        return ".h5ad"
+    elif isinstance(dmem, DataFrame):
+        return ".parquet"
+    else:
+        if _mudata_is_installed():
+            from mudata import MuData
+
+            if isinstance(dmem, MuData):
+                return ".h5mu"
+        raise NotImplementedError
+
+
+def write_to_disk(dmem, filepath: UPathStr):
+    if isinstance(dmem, AnnData):
+        suffix = PurePosixPath(filepath).suffix
+        if suffix == ".h5ad":
+            dmem.write_h5ad(filepath)
+        elif suffix == ".zarr":
+            dmem.write_zarr(filepath)
+        else:
+            raise NotImplementedError
+    elif isinstance(dmem, DataFrame):
+        dmem.to_parquet(filepath)
+    else:
+        if _mudata_is_installed():
+            from mudata import MuData
+
+            if isinstance(dmem, MuData):
+                dmem.write(filepath)
+                return
+        raise NotImplementedError

lamindb/core/storage/paths.py

@@ -1,172 +1,172 @@
-from __future__ import annotations
-
-import shutil
-from typing import TYPE_CHECKING
-
-import anndata as ad
-import pandas as pd
-from lamin_utils import logger
-from lamindb_setup.core import StorageSettings
-from lamindb_setup.core.upath import (
-    LocalPathClasses,
-    UPath,
-    create_path,
-    infer_filesystem,
-)
-from lnschema_core.models import Artifact, Storage
-
-from lamindb.core._settings import settings
-
-if TYPE_CHECKING:
-    from pathlib import Path
-
-    from lamindb_setup.core.types import UPathStr
-
-
-AUTO_KEY_PREFIX = ".lamindb/"
-
-
-# add type annotations back asap when re-organizing the module
-def auto_storage_key_from_artifact(artifact: Artifact):
-    if artifact.key is None or artifact._key_is_virtual:
-        is_dir = artifact.n_objects is not None
-        return auto_storage_key_from_artifact_uid(artifact.uid, artifact.suffix, is_dir)
-    else:
-        return artifact.key
-
-
-def auto_storage_key_from_artifact_uid(uid: str, suffix: str, is_dir: bool) -> str:
-    assert isinstance(suffix, str)  # noqa: S101 Suffix cannot be None.
-    if is_dir:
-        uid_storage = uid[:16]  # 16 chars, leave 4 chars for versioning
-    else:
-        uid_storage = uid
-    storage_key = f"{AUTO_KEY_PREFIX}{uid_storage}{suffix}"
-    return storage_key
-
-
-def check_path_is_child_of_root(path: Path | UPath, root: Path | UPath | None) -> bool:
-    # str is needed to eliminate UPath storage_options
-    # from the equality checks below
-    path = UPath(str(path))
-    root = UPath(str(root))
-    return root.resolve() in path.resolve().parents
-
-
-# returns filepath and root of the storage
-def attempt_accessing_path(
-    artifact: Artifact,
-    storage_key: str,
-    using_key: str | None = None,
-    access_token: str | None = None,
-) -> tuple[UPath, StorageSettings]:
-    # check whether the file is in the default db and whether storage
-    # matches default storage
-    if (
-        artifact._state.db in ("default", None)
-        and artifact.storage_id == settings._storage_settings.id
-    ):
-        if access_token is None:
-            storage_settings = settings._storage_settings
-        else:
-            storage_settings = StorageSettings(
-                settings.storage.root, access_token=access_token
-            )
-    else:
-        if artifact._state.db not in ("default", None) and using_key is None:
-            storage = Storage.using(artifact._state.db).get(id=artifact.storage_id)
-        else:
-            storage = Storage.objects.using(using_key).get(id=artifact.storage_id)
-        # find a better way than passing None to instance_settings in the future!
-        storage_settings = StorageSettings(storage.root, access_token=access_token)
-    path = storage_settings.key_to_filepath(storage_key)
-    return path, storage_settings
-
-
-def filepath_from_artifact(
-    artifact: Artifact, using_key: str | None = None
-) -> tuple[UPath, StorageSettings | None]:
-    if hasattr(artifact, "_local_filepath") and artifact._local_filepath is not None:
-        return artifact._local_filepath.resolve(), None
-    storage_key = auto_storage_key_from_artifact(artifact)
-    path, storage_settings = attempt_accessing_path(
-        artifact, storage_key, using_key=using_key
-    )
-    return path, storage_settings
-
-
-# virtual key is taken into consideration
-# only if the version is latest
-def _cache_key_from_artifact_storage(
-    artifact: Artifact, storage_settings: StorageSettings | None
-):
-    cache_key = None
-    if (
-        artifact._key_is_virtual
-        and artifact.key is not None
-        and storage_settings is not None
-        and artifact.is_latest
-    ):
-        cache_key = (storage_settings.root / artifact.key).path
-    return cache_key
-
-
-# return filepath and cache_key if needed
-def filepath_cache_key_from_artifact(
-    artifact: Artifact, using_key: str | None = None
-) -> tuple[UPath, str | None]:
-    filepath, storage_settings = filepath_from_artifact(artifact, using_key)
-    if isinstance(filepath, LocalPathClasses):
-        return filepath, None
-    cache_key = _cache_key_from_artifact_storage(artifact, storage_settings)
-    return filepath, cache_key
-
-
-def store_file_or_folder(
-    local_path: UPathStr, storage_path: UPath, print_progress: bool = True
-) -> None:
-    """Store file or folder (localpath) at storagepath."""
-    local_path = UPath(local_path)
-    if not isinstance(storage_path, LocalPathClasses):
-        # this uploads files and directories
-        create_folder = False if local_path.is_dir() else None
-        storage_path.upload_from(
-            local_path, create_folder=create_folder, print_progress=print_progress
-        )
-    else:  # storage path is local
-        if local_path.resolve().as_posix() == storage_path.resolve().as_posix():
-            return None
-        storage_path.parent.mkdir(parents=True, exist_ok=True)
-        if local_path.is_file():
-            shutil.copyfile(local_path, storage_path)
-        else:
-            if storage_path.exists():
-                shutil.rmtree(storage_path)
-            shutil.copytree(local_path, storage_path)
-
-
-def delete_storage_using_key(
-    artifact: Artifact, storage_key: str, using_key: str | None
-):
-    filepath, _ = attempt_accessing_path(artifact, storage_key, using_key=using_key)
-    delete_storage(filepath)
-
-
-def delete_storage(
-    storagepath: Path, raise_file_not_found_error: bool = True
-) -> None | str:
-    """Delete arbitrary artifact."""
-    if storagepath.is_file():
-        storagepath.unlink()
-    elif storagepath.is_dir():
-        if isinstance(storagepath, LocalPathClasses) or not isinstance(
-            storagepath, UPath
-        ):
-            shutil.rmtree(storagepath)
-        else:
-            storagepath.rmdir()
-    elif raise_file_not_found_error:
-        raise FileNotFoundError(f"{storagepath} is not an existing path!")
-    else:
-        logger.warning(f"{storagepath} is not an existing path!")
-    return None
+from __future__ import annotations
+
+import shutil
+from typing import TYPE_CHECKING
+
+import anndata as ad
+import pandas as pd
+from lamin_utils import logger
+from lamindb_setup.core import StorageSettings
+from lamindb_setup.core.upath import (
+    LocalPathClasses,
+    UPath,
+    create_path,
+    infer_filesystem,
+)
+from lnschema_core.models import Artifact, Storage
+
+from lamindb.core._settings import settings
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from lamindb_setup.core.types import UPathStr
+
+
+AUTO_KEY_PREFIX = ".lamindb/"
+
+
+# add type annotations back asap when re-organizing the module
+def auto_storage_key_from_artifact(artifact: Artifact):
+    if artifact.key is None or artifact._key_is_virtual:
+        is_dir = artifact.n_objects is not None
+        return auto_storage_key_from_artifact_uid(artifact.uid, artifact.suffix, is_dir)
+    else:
+        return artifact.key
+
+
+def auto_storage_key_from_artifact_uid(uid: str, suffix: str, is_dir: bool) -> str:
+    assert isinstance(suffix, str)  # noqa: S101 Suffix cannot be None.
+    if is_dir:
+        uid_storage = uid[:16]  # 16 chars, leave 4 chars for versioning
+    else:
+        uid_storage = uid
+    storage_key = f"{AUTO_KEY_PREFIX}{uid_storage}{suffix}"
+    return storage_key
+
+
+def check_path_is_child_of_root(path: Path | UPath, root: Path | UPath | None) -> bool:
+    # str is needed to eliminate UPath storage_options
+    # from the equality checks below
+    path = UPath(str(path))
+    root = UPath(str(root))
+    return root.resolve() in path.resolve().parents
+
+
+# returns filepath and root of the storage
+def attempt_accessing_path(
+    artifact: Artifact,
+    storage_key: str,
+    using_key: str | None = None,
+    access_token: str | None = None,
+) -> tuple[UPath, StorageSettings]:
+    # check whether the file is in the default db and whether storage
+    # matches default storage
+    if (
+        artifact._state.db in ("default", None)
+        and artifact.storage_id == settings._storage_settings.id
+    ):
+        if access_token is None:
+            storage_settings = settings._storage_settings
+        else:
+            storage_settings = StorageSettings(
+                settings.storage.root, access_token=access_token
+            )
+    else:
+        if artifact._state.db not in ("default", None) and using_key is None:
+            storage = Storage.using(artifact._state.db).get(id=artifact.storage_id)
+        else:
+            storage = Storage.objects.using(using_key).get(id=artifact.storage_id)
+        # find a better way than passing None to instance_settings in the future!
+        storage_settings = StorageSettings(storage.root, access_token=access_token)
+    path = storage_settings.key_to_filepath(storage_key)
+    return path, storage_settings
+
+
+def filepath_from_artifact(
+    artifact: Artifact, using_key: str | None = None
+) -> tuple[UPath, StorageSettings | None]:
+    if hasattr(artifact, "_local_filepath") and artifact._local_filepath is not None:
+        return artifact._local_filepath.resolve(), None
+    storage_key = auto_storage_key_from_artifact(artifact)
+    path, storage_settings = attempt_accessing_path(
+        artifact, storage_key, using_key=using_key
+    )
+    return path, storage_settings
+
+
+# virtual key is taken into consideration
+# only if the version is latest
+def _cache_key_from_artifact_storage(
+    artifact: Artifact, storage_settings: StorageSettings | None
+):
+    cache_key = None
+    if (
+        artifact._key_is_virtual
+        and artifact.key is not None
+        and storage_settings is not None
+        and artifact.is_latest
+    ):
+        cache_key = (storage_settings.root / artifact.key).path
+    return cache_key
+
+
+# return filepath and cache_key if needed
+def filepath_cache_key_from_artifact(
+    artifact: Artifact, using_key: str | None = None
+) -> tuple[UPath, str | None]:
+    filepath, storage_settings = filepath_from_artifact(artifact, using_key)
+    if isinstance(filepath, LocalPathClasses):
+        return filepath, None
+    cache_key = _cache_key_from_artifact_storage(artifact, storage_settings)
+    return filepath, cache_key
+
+
+def store_file_or_folder(
+    local_path: UPathStr, storage_path: UPath, print_progress: bool = True
+) -> None:
+    """Store file or folder (localpath) at storagepath."""
+    local_path = UPath(local_path)
+    if not isinstance(storage_path, LocalPathClasses):
+        # this uploads files and directories
+        create_folder = False if local_path.is_dir() else None
+        storage_path.upload_from(
+            local_path, create_folder=create_folder, print_progress=print_progress
+        )
+    else:  # storage path is local
+        if local_path.resolve().as_posix() == storage_path.resolve().as_posix():
+            return None
+        storage_path.parent.mkdir(parents=True, exist_ok=True)
+        if local_path.is_file():
+            shutil.copyfile(local_path, storage_path)
+        else:
+            if storage_path.exists():
+                shutil.rmtree(storage_path)
+            shutil.copytree(local_path, storage_path)
+
+
+def delete_storage_using_key(
+    artifact: Artifact, storage_key: str, using_key: str | None
+):
+    filepath, _ = attempt_accessing_path(artifact, storage_key, using_key=using_key)
+    delete_storage(filepath)
+
+
+def delete_storage(
+    storagepath: Path, raise_file_not_found_error: bool = True
+) -> None | str:
+    """Delete arbitrary artifact."""
+    if storagepath.is_file():
+        storagepath.unlink()
+    elif storagepath.is_dir():
+        if isinstance(storagepath, LocalPathClasses) or not isinstance(
+            storagepath, UPath
+        ):
+            shutil.rmtree(storagepath)
+        else:
+            storagepath.rmdir()
+    elif raise_file_not_found_error:
+        raise FileNotFoundError(f"{storagepath} is not an existing path!")
+    else:
+        logger.warning(f"{storagepath} is not an existing path!")
+    return None

lamindb/core/subsettings/__init__.py

@@ -1,12 +1,12 @@
-"""Sub settings.
-
-.. autosummary::
-   :toctree: .
-
-   TransformSettings
-   CreationSettings
-
-"""
-
-from ._creation_settings import CreationSettings
-from ._transform_settings import TransformSettings
+"""Sub settings.
+
+.. autosummary::
+   :toctree: .
+
+   TransformSettings
+   CreationSettings
+
+"""
+
+from ._creation_settings import CreationSettings
+from ._transform_settings import TransformSettings

lamindb/core/subsettings/_creation_settings.py

@@ -1,38 +1,38 @@
-from typing import Literal
-
-
-class CreationSettings:
-    artifact_if_hash_exists: Literal[
-        "warn_return_existing", "error", "warn_create_new"
-    ] = "warn_return_existing"
-    """Behavior if file hash exists (default `"warn_return_existing"`).
-
-    One of `["warn_return_existing", "error", "warn_create_new"]`.
-
-    FAQ: :doc:`/faq/idempotency`
-    """
-    artifact_skip_size_hash: bool = False
-    """To speed up registering high numbers of files (default `False`).
-
-    This bypasses queries for size and hash to AWS & GCP.
-
-    It speeds up file creation by about a factor 100.
-    """
-    search_names: bool = True
-    """To speed up creating records (default `True`).
-
-    If `True`, search for alternative names.
-
-    FAQ: :doc:`/faq/idempotency`
-    """
-    artifact_silence_missing_run_warning: bool = False
-    """Silence warning about missing run & transform during artifact creation."""
-    _artifact_use_virtual_keys: bool = True
-    """Treat `key` parameter in :class:`~lamindb.Artifact` as virtual.
-
-    If `True`, the `key` is **not** used to construct file paths, but file paths are
-    based on the `uid` of artifact.
-    """
-
-
-creation_settings = CreationSettings()
+from typing import Literal
+
+
+class CreationSettings:
+    artifact_if_hash_exists: Literal[
+        "warn_return_existing", "error", "warn_create_new"
+    ] = "warn_return_existing"
+    """Behavior if file hash exists (default `"warn_return_existing"`).
+
+    One of `["warn_return_existing", "error", "warn_create_new"]`.
+
+    FAQ: :doc:`/faq/idempotency`
+    """
+    artifact_skip_size_hash: bool = False
+    """To speed up registering high numbers of files (default `False`).
+
+    This bypasses queries for size and hash to AWS & GCP.
+
+    It speeds up file creation by about a factor 100.
+    """
+    search_names: bool = True
+    """To speed up creating records (default `True`).
+
+    If `True`, search for alternative names.
+
+    FAQ: :doc:`/faq/idempotency`
+    """
+    artifact_silence_missing_run_warning: bool = False
+    """Silence warning about missing run & transform during artifact creation."""
+    _artifact_use_virtual_keys: bool = True
+    """Treat `key` parameter in :class:`~lamindb.Artifact` as virtual.
+
+    If `True`, the `key` is **not** used to construct file paths, but file paths are
+    based on the `uid` of artifact.
+    """
+
+
+creation_settings = CreationSettings()

lamindb/core/subsettings/_transform_settings.py

@@ -1,21 +1,21 @@
-from __future__ import annotations
-
-
-class TransformSettings:
-    stem_uid: str | None = None
-    """Defines the version family of the transform.
-
-    For example, all notebooks of the same family have a uid that starts with
-    `"FPnfDtJz8qbE"`.
-
-    The full uids of the notebooks in this family are of form
-    `"{stem_uid}{suffix_uid}"` where the `suffix_uid` encodes the semantic
-    `version`.
-    """
-    version: str | None = None
-    """The version."""
-    name: str | None = None
-    """A name like a notebook or script title."""
-
-
-transform_settings = TransformSettings()
+from __future__ import annotations
+
+
+class TransformSettings:
+    stem_uid: str | None = None
+    """Defines the version family of the transform.
+
+    For example, all notebooks of the same family have a uid that starts with
+    `"FPnfDtJz8qbE"`.
+
+    The full uids of the notebooks in this family are of form
+    `"{stem_uid}{suffix_uid}"` where the `suffix_uid` encodes the semantic
+    `version`.
+    """
+    version: str | None = None
+    """The version."""
+    name: str | None = None
+    """A name like a notebook or script title."""
+
+
+transform_settings = TransformSettings()

lamindb/core/types.py

@@ -1,19 +1,19 @@
-"""Types.
-
-.. autosummary::
-   :toctree: .
-
-   UPathStr
-   StrField
-   ListLike
-   TransformType
-   ArtifactType
-"""
-
-from lamindb_setup.core.types import UPathStr
-from lnschema_core.types import (
-    ArtifactType,
-    ListLike,
-    StrField,
-    TransformType,
-)
+"""Types.
+
+.. autosummary::
+   :toctree: .
+
+   UPathStr
+   StrField
+   ListLike
+   TransformType
+   ArtifactType
+"""
+
+from lamindb_setup.core.types import UPathStr
+from lnschema_core.types import (
+    ArtifactType,
+    ListLike,
+    StrField,
+    TransformType,
+)