lamindb_setup 0.69.5__py2.py3-none-any.whl → 0.71.0__py2.py3-none-any.whl

lamindb_setup/__init__.py

@@ -1,12 +1,10 @@
 """Setup & configure LaminDB.
 
-Every function in the API has a matching command in the `lamin` CLI.
-
-Typically, you'll want to use the CLI rather than this API.
+Many functions in this "setup API" have a matching command in the :doc:`docs:cli` CLI.
 
 Guide: :doc:`docs:setup`.
 
-Setup:
+Basic operations:
 
 .. autosummary::
    :toctree:
@@ -17,7 +15,7 @@ Setup:
    close
    delete
 
-More instance operations:
+Instance operations:
 
 .. autosummary::
    :toctree:
@@ -36,22 +34,22 @@ Modules & settings:
 
 """
 
-__version__ = "0.69.5"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.71.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name
 
 from . import core
-from ._close import close  # noqa
-from ._delete import delete  # noqa
-from ._init_instance import init  # noqa
-from ._connect_instance import connect, load  # noqa
-from ._migrate import migrate
-from ._register_instance import register  # noqa
-from .core._settings import settings  # noqa
-from ._setup_user import login, logout  # noqa
-from ._django import django
 from ._check_setup import _check_instance_setup
+from ._close import close
+from ._connect_instance import connect, load
+from ._delete import delete
+from ._django import django
+from ._init_instance import init
+from ._migrate import migrate
+from ._register_instance import register
+from ._setup_user import login, logout
+from .core._settings import settings
 
 dev = core  # backward compat
 _TESTING = False  # used in lamindb tests
@@ -70,3 +68,5 @@ if _os_name == "nt":
                 _original_excepthook(args)
 
         threading.excepthook = _except_hook
+
+settings.__doc__ = """Global :class:`~lamindb.setup.core.SetupSettings`."""

lamindb_setup/_add_remote_storage.py

@@ -1,14 +1,18 @@
-from lamin_utils import logger
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 
-from lamindb_setup.core.types import UPathStr
+from lamin_utils import logger
 
-from ._init_instance import register_user_and_storage
+from ._init_instance import register_storage_in_instance
 from .core._settings import settings
-from .core._settings_instance import InstanceSettings
-from .core._settings_storage import StorageSettings
+from .core._settings_storage import init_storage
+
+if TYPE_CHECKING:
+    from lamindb_setup.core.types import UPathStr
 
 
-def switch_default_storage(root: UPathStr, **fs_kwargs):
+def add_managed_storage(root: UPathStr, **fs_kwargs):
     """Add a remote default storage location to a local instance.
 
     This can be used to selectively share data.
@@ -17,34 +21,19 @@ def switch_default_storage(root: UPathStr, **fs_kwargs):
         root: `UPathStr` - The new storage root, e.g., an S3 bucket.
         **fs_kwargs: Additional fsspec arguments for cloud root, e.g., profile.
 
-    Example:
-    >>> ln.setup.set.storage(
-    >>>    "s3://some-bucket",
-    >>>     profile="some_profile", # fsspec arg
-    >>>     cache_regions=True # fsspec arg for s3
-    >>> )
-
     """
     if settings.instance.dialect == "sqlite":
-        logger.error("can't set storage for sqlite instances.")
-        return "set-storage-failed"
-    ssettings = StorageSettings(root=root)
-    new_isettings = InstanceSettings(
-        owner=settings.instance.owner,
-        name=settings.instance.name,
-        storage=ssettings,
-        db=settings.instance.db,
-        schema=settings.instance._schema_str,
-        id=settings.instance.id,
+        raise ValueError(
+            "Can't add additional managed storage locations for sqlite instances."
+        )
+    if not settings.instance.is_on_hub:
+        raise ValueError(
+            "Can't add additional managed storage locations for instances that aren't managed through the hub."
+        )
+    ssettings = init_storage(
+        root=root, instance_id=settings.instance._id, register_hub=True
     )
-
-    new_isettings._persist()  # this also updates the settings object
-    register_user_and_storage(new_isettings, settings.user)
-    # we are not doing this for now because of difficulties to define the right RLS policy  # noqa
-    # https://laminlabs.slack.com/archives/C04FPE8V01W/p1687948324601929?thread_ts=1687531921.394119&cid=C04FPE8V01W
-    # if settings.instance.is_remote:
-    #     init_storage_hub(
-    #         root, account_handle=settings.instance.owner  # type: ignore
-    #     )
-
+    settings.instance._storage = ssettings
+    settings.instance._persist()  # this also updates the settings object
+    register_storage_in_instance(ssettings)
     settings.storage._set_fs_kwargs(**fs_kwargs)

lamindb_setup/_cache.py

@@ -1,9 +1,12 @@
+from __future__ import annotations
+
 import shutil
+
 from lamin_utils import logger
 
 
 def clear_cache_dir():
-    from lamindb_setup import settings, close
+    from lamindb_setup import close, settings
 
     if settings.instance._is_cloud_sqlite:
         logger.warning(

lamindb_setup/_check.py

@@ -1,3 +1,6 @@
+from __future__ import annotations
+
+
 def check():
     from django.core.management import call_command
 

lamindb_setup/_check_setup.py

@@ -1,12 +1,18 @@
-from lamin_utils import logger
-from typing import Optional
+from __future__ import annotations
+
 import os
+from typing import TYPE_CHECKING, Optional
+
+from lamin_utils import logger
+
 from ._silence_loggers import silence_loggers
+from .core import django
+from .core._settings import settings
 from .core._settings_store import current_instance_settings_file
 from .core.exceptions import DefaultMessageException
-from .core._settings import settings
-from .core._settings_instance import InstanceSettings
-from .core import django
+
+if TYPE_CHECKING:
+    from .core._settings_instance import InstanceSettings
 
 
 class InstanceNotSetupError(DefaultMessageException):
@@ -19,10 +25,10 @@ If you used the CLI to set up lamindb in a notebook, restart the Python session.
 """
 
 
-CURRENT_ISETTINGS: Optional[InstanceSettings] = None
+CURRENT_ISETTINGS: InstanceSettings | None = None
 
 
-def _get_current_instance_settings() -> Optional[InstanceSettings]:
+def _get_current_instance_settings() -> InstanceSettings | None:
     global CURRENT_ISETTINGS
 
     if CURRENT_ISETTINGS is not None:

lamindb_setup/_close.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from lamin_utils import logger
 
 from .core._settings import settings

lamindb_setup/_connect_instance.py

@@ -1,32 +1,49 @@
-from pathlib import Path
-from typing import Optional, Union, Dict, Tuple
-from uuid import UUID
+from __future__ import annotations
+
 import os
+from typing import TYPE_CHECKING
+from uuid import UUID
+
 from lamin_utils import logger
-from lamindb_setup.core.types import UPathStr
-from lamindb_setup.core._hub_utils import (
+
+from ._check_setup import _check_instance_setup
+from ._close import close as close_instance
+from ._init_instance import MESSAGE_NO_MULTIPLE_INSTANCE, load_from_isettings
+from ._migrate import check_whether_migrations_in_sync
+from ._silence_loggers import silence_loggers
+from .core._hub_core import connect_instance as load_instance_from_hub
+from .core._hub_utils import (
     LaminDsn,
     LaminDsnModel,
 )
-from ._close import close as close_instance
-from ._init_instance import load_from_isettings
-from .core._settings import InstanceSettings, settings
-from ._silence_loggers import silence_loggers
+from .core._settings import settings
+from .core._settings_instance import InstanceSettings
 from .core._settings_load import load_instance_settings
 from .core._settings_storage import StorageSettings
 from .core._settings_store import instance_settings_file
 from .core.cloud_sqlite_locker import unlock_cloud_sqlite_upon_exception
-from ._init_instance import MESSAGE_NO_MULTIPLE_INSTANCE
-from ._check_setup import _check_instance_setup
-from .core._hub_core import connect_instance as load_instance_from_hub
-from ._migrate import check_whether_migrations_in_sync
 
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from .core.types import UPathStr
 
 # this is for testing purposes only
 # set to True only to test failed load
 _TEST_FAILED_LOAD = False
 
 
+INSTANCE_NOT_FOUND_MESSAGE = (
+    "'{owner}/{name}' not found:"
+    " '{hub_result}'\nCheck your permissions:"
+    " https://lamin.ai/{owner}/{name}"
+)
+
+
+class InstanceNotFoundError(SystemExit):
+    pass
+
+
 def check_db_dsn_equal_up_to_credentials(db_dsn_hub, db_dsn_local):
     return (
         db_dsn_hub.scheme == db_dsn_local.scheme
@@ -37,8 +54,8 @@ def check_db_dsn_equal_up_to_credentials(db_dsn_hub, db_dsn_local):
 
 
 def update_db_using_local(
-    hub_instance_result: Dict[str, str], settings_file: Path, db: Optional[str] = None
-) -> Optional[str]:
+    hub_instance_result: dict[str, str], settings_file: Path, db: str | None = None
+) -> str | None:
     db_updated = None
     # check if postgres
     if hub_instance_result["db_scheme"] == "postgresql":
@@ -87,11 +104,11 @@ def update_db_using_local(
 def connect(
     slug: str,
     *,
-    db: Optional[str] = None,
-    storage: Optional[UPathStr] = None,
+    db: str | None = None,
+    storage: UPathStr | None = None,
     _raise_not_reachable_error: bool = True,
     _test: bool = False,
-) -> Optional[Union[str, Tuple]]:
+) -> str | tuple | None:
     """Connect to instance.
 
     Args:
@@ -121,6 +138,8 @@ def connect(
         make_hub_request = True
         if settings_file.exists():
             isettings = load_instance_settings(settings_file)
+            # mimic instance_result from hub
+            instance_result = {"id": isettings._id.hex}
             # skip hub request for a purely local instance
             make_hub_request = isettings.is_remote
 
@@ -139,7 +158,6 @@ def connect(
                     root=storage_result["root"],
                     region=storage_result["region"],
                     uid=storage_result["lnid"],
-                    is_hybrid=instance_result["storage_mode"] == "hybrid",
                 )
                 isettings = InstanceSettings(
                     id=UUID(instance_result["id"]),
@@ -149,27 +167,25 @@ def connect(
                     db=db_updated,
                     schema=instance_result["schema_str"],
                     git_repo=instance_result["git_repo"],
+                    keep_artifacts_local=bool(instance_result["keep_artifacts_local"]),
+                    is_on_hub=True,
                 )
                 check_whether_migrations_in_sync(instance_result["lamindb_version"])
             else:
-                error_message = (
-                    f"'{owner}/{name}' not loadable:"
-                    f" '{hub_result}'\nCheck your permissions:"
-                    f" https://lamin.ai/{owner}/{name}?tab=collaborators"
+                message = INSTANCE_NOT_FOUND_MESSAGE.format(
+                    owner=owner, name=name, hub_result=hub_result
                 )
                 if settings_file.exists():
                     isettings = load_instance_settings(settings_file)
                     if isettings.is_remote:
                         if _raise_not_reachable_error:
-                            raise SystemExit(error_message)
-                        return "instance-not-reachable"
+                            raise InstanceNotFoundError(message)
+                        return "instance-not-found"
 
                 else:
                     if _raise_not_reachable_error:
-                        raise SystemExit(error_message)
-                    return "instance-not-reachable"
-                # mimic instance_result from hub
-                instance_result = {"id": isettings.id.hex}
+                        raise InstanceNotFoundError(message)
+                    return "instance-not-found"
 
         if storage is not None:
             update_isettings_with_storage(isettings, storage)
@@ -194,7 +210,7 @@ def connect(
                 raise SystemExit(msg)
             else:
                 logger.warning(
-                    f"instance exists with id {isettings.id.hex}, but database is not"
+                    f"instance exists with id {isettings._id.hex}, but database is not"
                     " loadable: re-initializing"
                 )
                 return "instance-corrupted-or-deleted", instance_result
@@ -215,9 +231,9 @@ def connect(
 def load(
     slug: str,
     *,
-    db: Optional[str] = None,
-    storage: Optional[UPathStr] = None,
-) -> Optional[Union[str, Tuple]]:
+    db: str | None = None,
+    storage: UPathStr | None = None,
+) -> str | tuple | None:
     """Connect to instance and set ``auto-connect`` to true.
 
     This is exactly the same as ``ln.connect()`` except for that
@@ -255,7 +271,7 @@ def update_isettings_with_storage(
     isettings: InstanceSettings, storage: UPathStr
 ) -> None:
     ssettings = StorageSettings(storage)
-    if ssettings.is_cloud:
+    if ssettings.type_is_cloud:
         try:  # triggering ssettings.id makes a lookup in the storage table
             logger.success(f"loaded storage: {ssettings.id} / {ssettings.root_as_str}")
         except RuntimeError as e:

lamindb_setup/_delete.py

@@ -1,11 +1,24 @@
+from __future__ import annotations
+
 import shutil
-from pathlib import Path
+from typing import TYPE_CHECKING, Optional
+from uuid import UUID
+
 from lamin_utils import logger
-from typing import Optional
-from .core._settings_instance import InstanceSettings
+
+from ._connect_instance import INSTANCE_NOT_FOUND_MESSAGE
+from .core._hub_core import connect_instance as load_instance_from_hub
+from .core._hub_core import delete_instance as delete_instance_on_hub
+from .core._hub_core import get_storage_records_for_instance
 from .core._settings import settings
+from .core._settings_instance import InstanceSettings
 from .core._settings_load import load_instance_settings
+from .core._settings_storage import StorageSettings
 from .core._settings_store import instance_settings_file
+from .core.upath import HOSTED_BUCKETS, check_storage_is_empty
+
+if TYPE_CHECKING:
+    from pathlib import Path
 
 
 def delete_cache(cache_dir: Path):
@@ -13,6 +26,12 @@ def delete_cache(cache_dir: Path):
         shutil.rmtree(cache_dir)
 
 
+def delete_exclusion_dir(isettings: InstanceSettings) -> None:
+    exclusion_dir = isettings.storage.root / f".lamindb/_exclusion/{isettings._id.hex}"
+    if exclusion_dir.exists():
+        exclusion_dir.rmdir()
+
+
 def delete_by_isettings(isettings: InstanceSettings) -> None:
     settings_file = isettings._get_settings_file()
     if settings_file.exists():
@@ -22,20 +41,12 @@ def delete_by_isettings(isettings: InstanceSettings) -> None:
         try:
             if isettings._sqlite_file.exists():
                 isettings._sqlite_file.unlink()
-            exclusion_dir = (
-                isettings.storage.root / f".lamindb/_exclusion/{isettings.id.hex}"
-            )
-            if exclusion_dir.exists():
-                exclusion_dir.rmdir()
         except PermissionError:
             logger.warning(
                 "Did not have permission to delete SQLite file:"
                 f" {isettings._sqlite_file}"
             )
             pass
-    if isettings.is_remote:
-        logger.warning("manually delete your remote instance on lamin.ai")
-    logger.warning(f"manually delete your stored data: {isettings.storage.root}")
     # unset the global instance settings
     if settings._instance_exists and isettings.slug == settings.instance.slug:
         if settings._instance_settings_path.exists():
@@ -43,8 +54,16 @@ def delete_by_isettings(isettings: InstanceSettings) -> None:
         settings._instance_settings = None
 
 
-def delete(instance_name: str, force: bool = False) -> Optional[int]:
-    """Delete a LaminDB instance."""
+def delete(
+    instance_name: str, force: bool = False, require_empty: bool = True
+) -> int | None:
+    """Delete a LaminDB instance.
+
+    Args:
+        instance_name (str): The name of the instance to delete.
+        force (bool): Whether to skip the confirmation prompt.
+        require_empty (bool): Whether to check if the instance is empty before deleting.
+    """
     if "/" in instance_name:
         logger.warning(
             "Deleting the instance of another user is currently not supported with the"
@@ -53,6 +72,48 @@ def delete(instance_name: str, force: bool = False) -> Optional[int]:
         )
         raise ValueError("Invalid instance name: '/' delimiter not allowed.")
     instance_slug = f"{settings.user.handle}/{instance_name}"
+    if settings._instance_exists and settings.instance.name == instance_name:
+        isettings = settings.instance
+    else:
+        settings_file = instance_settings_file(instance_name, settings.user.handle)
+        if not settings_file.exists():
+            hub_result = load_instance_from_hub(
+                owner=settings.user.handle, name=instance_name
+            )
+            if isinstance(hub_result, str):
+                message = INSTANCE_NOT_FOUND_MESSAGE.format(
+                    owner=settings.user.handle,
+                    name=instance_name,
+                    hub_result=hub_result,
+                )
+                logger.warning(message)
+                return None
+            instance_result, storage_result = hub_result
+            ssettings = StorageSettings(
+                root=storage_result["root"],
+                region=storage_result["region"],
+                uid=storage_result["lnid"],
+            )
+            isettings = InstanceSettings(
+                id=UUID(instance_result["id"]),
+                owner=settings.user.handle,
+                name=instance_name,
+                storage=ssettings,
+                keep_artifacts_local=bool(instance_result["keep_artifacts_local"]),
+                db=instance_result["db"] if "db" in instance_result else None,
+                schema=instance_result["schema_str"],
+                git_repo=instance_result["git_repo"],
+            )
+        else:
+            isettings = load_instance_settings(settings_file)
+    if isettings.dialect != "sqlite":
+        logger.warning(
+            f"delete() does not yet affect your Postgres database at {isettings.db}"
+        )
+    if isettings.is_on_hub and settings.user.handle == "anonymous":
+        logger.warning(
+            "won't delete the hub component of this instance because you're not logged in"
+        )
     if not force:
         valid_responses = ["y", "yes"]
         user_input = (
@@ -62,17 +123,55 @@ def delete(instance_name: str, force: bool = False) -> Optional[int]:
         )
         if user_input not in valid_responses:
             return -1
-    if settings._instance_exists and settings.instance.name == instance_name:
-        isettings = settings.instance
-    else:
-        settings_file = instance_settings_file(instance_name, settings.user.handle)
-        if not settings_file.exists():
+
+    # the actual deletion process begins here
+    if isettings.dialect == "sqlite" and isettings.is_remote:
+        # delete the exlusion dir first because it's hard to count its objects
+        delete_exclusion_dir(isettings)
+    if isettings.storage.type_is_cloud and isettings.storage.root_as_str.startswith(
+        HOSTED_BUCKETS
+    ):
+        if not require_empty:
             logger.warning(
-                "could not delete as instance settings do not exist locally. did you"
-                " provide a wrong instance name? could you try loading it?"
+                "hosted storage always has to be empty, ignoring `require_empty`"
+            )
+        require_empty = True
+    # first the default storage
+    n_objects = check_storage_is_empty(
+        isettings.storage.root,
+        raise_error=require_empty,
+        account_for_sqlite_file=isettings.dialect == "sqlite",
+    )
+    if isettings.storage._mark_storage_root.exists():
+        isettings.storage._mark_storage_root.unlink(
+            missing_ok=True  # this is totally weird, but needed on Py3.11
+        )
+    # now everything that's on the hub
+    if settings.user.handle != "anonymous":
+        storage_records = get_storage_records_for_instance(isettings._id)
+        for storage_record in storage_records:
+            if storage_record["root"] == isettings.storage.root_as_str:
+                continue
+            check_storage_is_empty(
+                storage_record["root"],  # type: ignore
+                raise_error=require_empty,
             )
-            return None
-        isettings = load_instance_settings(settings_file)
+            ssettings = StorageSettings(storage_record["root"])  # type: ignore
+            if ssettings._mark_storage_root.exists():
+                ssettings._mark_storage_root.unlink(
+                    missing_ok=True  # this is totally weird, but needed on Py3.11
+                )
     logger.info(f"deleting instance {instance_slug}")
+    # below we can skip check_storage_is_empty() because we already called
+    # it above
+    if settings.user.handle != "anonymous":
+        delete_instance_on_hub(isettings._id, require_empty=False)
     delete_by_isettings(isettings)
+    # if .lndb file was delete, then we might count -1
+    if n_objects <= 0 and isettings.storage.type == "local":
+        # dir is only empty after sqlite file was delete via delete_by_isettings
+        if (isettings.storage.root / ".lamindb").exists():
+            (isettings.storage.root / ".lamindb").rmdir()
+        if isettings.storage.root.exists():
+            isettings.storage.root.rmdir()
     return None

lamindb_setup/_django.py

@@ -1,9 +1,12 @@
+from __future__ import annotations
+
 from typing import Optional
+
 from .core._settings import settings
 from .core.django import setup_django
 
 
-def django(command: str, package_name: Optional[str] = None, **kwargs):
+def django(command: str, package_name: str | None = None, **kwargs):
     r"""Manage migrations.
 
     Examples:

lamindb_setup/_exportdb.py

@@ -1,6 +1,7 @@
-from pathlib import Path
-from importlib import import_module
+from __future__ import annotations
 
+from importlib import import_module
+from pathlib import Path
 
 MODELS = {
     "core": {
@@ -46,6 +47,7 @@ def exportdb() -> None:
     directory = Path("./lamindb_export/")
     directory.mkdir(parents=True, exist_ok=True)
     import pandas as pd
+
     import lamindb_setup as ln_setup
 
     def export_registry(registry, directory):

lamindb_setup/_importdb.py

@@ -1,6 +1,9 @@
+from __future__ import annotations
+
+from importlib import import_module
 from pathlib import Path
+
 from ._exportdb import MODELS
-from importlib import import_module
 
 
 def import_registry(registry, directory, connection):
@@ -26,6 +29,7 @@ def importdb() -> None:
         if response != "y":
             return None
     from sqlalchemy import create_engine, text
+
     import lamindb_setup as ln_setup
 
     engine = create_engine(ln_setup.settings.instance.db, echo=False)