lamindb_setup 0.70.0__py2.py3-none-any.whl → 0.71.0__py2.py3-none-any.whl

lamindb_setup/__init__.py

@@ -1,12 +1,10 @@
 """Setup & configure LaminDB.
 
-Every function in the API has a matching command in the `lamin` CLI.
-
-Typically, you'll want to use the CLI rather than this API.
+Many functions in this "setup API" have a matching command in the :doc:`docs:cli` CLI.
 
 Guide: :doc:`docs:setup`.
 
-Setup:
+Basic operations:
 
 .. autosummary::
    :toctree:
@@ -17,7 +15,7 @@ Setup:
    close
    delete
 
-More instance operations:
+Instance operations:
 
 .. autosummary::
    :toctree:
@@ -36,22 +34,22 @@ Modules & settings:
 
 """
 
-__version__ = "0.70.0"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.71.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name
 
 from . import core
-from ._close import close  # noqa
-from ._delete import delete  # noqa
-from ._init_instance import init  # noqa
-from ._connect_instance import connect, load  # noqa
-from ._migrate import migrate
-from ._register_instance import register  # noqa
-from .core._settings import settings  # noqa
-from ._setup_user import login, logout  # noqa
-from ._django import django
 from ._check_setup import _check_instance_setup
+from ._close import close
+from ._connect_instance import connect, load
+from ._delete import delete
+from ._django import django
+from ._init_instance import init
+from ._migrate import migrate
+from ._register_instance import register
+from ._setup_user import login, logout
+from .core._settings import settings
 
 dev = core  # backward compat
 _TESTING = False  # used in lamindb tests
@@ -70,3 +68,5 @@ if _os_name == "nt":
                 _original_excepthook(args)
 
         threading.excepthook = _except_hook
+
+settings.__doc__ = """Global :class:`~lamindb.setup.core.SetupSettings`."""

lamindb_setup/_add_remote_storage.py

@@ -1,14 +1,18 @@
-from lamin_utils import logger
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 
-from lamindb_setup.core.types import UPathStr
+from lamin_utils import logger
 
-from ._init_instance import register_user_and_storage
+from ._init_instance import register_storage_in_instance
 from .core._settings import settings
-from .core._settings_instance import InstanceSettings
-from .core._settings_storage import StorageSettings
+from .core._settings_storage import init_storage
+
+if TYPE_CHECKING:
+    from lamindb_setup.core.types import UPathStr
 
 
-def switch_default_storage(root: UPathStr, **fs_kwargs):
+def add_managed_storage(root: UPathStr, **fs_kwargs):
     """Add a remote default storage location to a local instance.
 
     This can be used to selectively share data.
@@ -17,34 +21,19 @@ def switch_default_storage(root: UPathStr, **fs_kwargs):
         root: `UPathStr` - The new storage root, e.g., an S3 bucket.
         **fs_kwargs: Additional fsspec arguments for cloud root, e.g., profile.
 
-    Example:
-    >>> ln.setup.set.storage(
-    >>>    "s3://some-bucket",
-    >>>     profile="some_profile", # fsspec arg
-    >>>     cache_regions=True # fsspec arg for s3
-    >>> )
-
     """
     if settings.instance.dialect == "sqlite":
-        logger.error("can't set storage for sqlite instances.")
-        return "set-storage-failed"
-    ssettings = StorageSettings(root=root)
-    new_isettings = InstanceSettings(
-        owner=settings.instance.owner,
-        name=settings.instance.name,
-        storage=ssettings,
-        db=settings.instance.db,
-        schema=settings.instance._schema_str,
-        id=settings.instance.id,
+        raise ValueError(
+            "Can't add additional managed storage locations for sqlite instances."
+        )
+    if not settings.instance.is_on_hub:
+        raise ValueError(
+            "Can't add additional managed storage locations for instances that aren't managed through the hub."
+        )
+    ssettings = init_storage(
+        root=root, instance_id=settings.instance._id, register_hub=True
     )
-
-    new_isettings._persist()  # this also updates the settings object
-    register_user_and_storage(new_isettings, settings.user)
-    # we are not doing this for now because of difficulties to define the right RLS policy  # noqa
-    # https://laminlabs.slack.com/archives/C04FPE8V01W/p1687948324601929?thread_ts=1687531921.394119&cid=C04FPE8V01W
-    # if settings.instance.is_remote:
-    #     init_storage_hub(
-    #         root, account_handle=settings.instance.owner  # type: ignore
-    #     )
-
+    settings.instance._storage = ssettings
+    settings.instance._persist()  # this also updates the settings object
+    register_storage_in_instance(ssettings)
     settings.storage._set_fs_kwargs(**fs_kwargs)

lamindb_setup/_cache.py

@@ -1,9 +1,12 @@
+from __future__ import annotations
+
 import shutil
+
 from lamin_utils import logger
 
 
 def clear_cache_dir():
-    from lamindb_setup import settings, close
+    from lamindb_setup import close, settings
 
     if settings.instance._is_cloud_sqlite:
         logger.warning(

lamindb_setup/_check.py

@@ -1,3 +1,6 @@
+from __future__ import annotations
+
+
 def check():
     from django.core.management import call_command
 

lamindb_setup/_check_setup.py

@@ -1,12 +1,18 @@
-from lamin_utils import logger
-from typing import Optional
+from __future__ import annotations
+
 import os
+from typing import TYPE_CHECKING, Optional
+
+from lamin_utils import logger
+
 from ._silence_loggers import silence_loggers
+from .core import django
+from .core._settings import settings
 from .core._settings_store import current_instance_settings_file
 from .core.exceptions import DefaultMessageException
-from .core._settings import settings
-from .core._settings_instance import InstanceSettings
-from .core import django
+
+if TYPE_CHECKING:
+    from .core._settings_instance import InstanceSettings
 
 
 class InstanceNotSetupError(DefaultMessageException):
@@ -19,10 +25,10 @@ If you used the CLI to set up lamindb in a notebook, restart the Python session.
 """
 
 
-CURRENT_ISETTINGS: Optional[InstanceSettings] = None
+CURRENT_ISETTINGS: InstanceSettings | None = None
 
 
-def _get_current_instance_settings() -> Optional[InstanceSettings]:
+def _get_current_instance_settings() -> InstanceSettings | None:
     global CURRENT_ISETTINGS
 
     if CURRENT_ISETTINGS is not None:

lamindb_setup/_close.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from lamin_utils import logger
 
 from .core._settings import settings

lamindb_setup/_connect_instance.py

@@ -1,26 +1,32 @@
-from pathlib import Path
-from typing import Optional, Union, Dict, Tuple
-from uuid import UUID
+from __future__ import annotations
+
 import os
+from typing import TYPE_CHECKING
+from uuid import UUID
+
 from lamin_utils import logger
-from lamindb_setup.core.types import UPathStr
-from lamindb_setup.core._hub_utils import (
+
+from ._check_setup import _check_instance_setup
+from ._close import close as close_instance
+from ._init_instance import MESSAGE_NO_MULTIPLE_INSTANCE, load_from_isettings
+from ._migrate import check_whether_migrations_in_sync
+from ._silence_loggers import silence_loggers
+from .core._hub_core import connect_instance as load_instance_from_hub
+from .core._hub_utils import (
     LaminDsn,
     LaminDsnModel,
 )
-from ._close import close as close_instance
-from ._init_instance import load_from_isettings
-from .core._settings import InstanceSettings, settings
-from ._silence_loggers import silence_loggers
+from .core._settings import settings
+from .core._settings_instance import InstanceSettings
 from .core._settings_load import load_instance_settings
 from .core._settings_storage import StorageSettings
 from .core._settings_store import instance_settings_file
 from .core.cloud_sqlite_locker import unlock_cloud_sqlite_upon_exception
-from ._init_instance import MESSAGE_NO_MULTIPLE_INSTANCE
-from ._check_setup import _check_instance_setup
-from .core._hub_core import connect_instance as load_instance_from_hub
-from ._migrate import check_whether_migrations_in_sync
 
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from .core.types import UPathStr
 
 # this is for testing purposes only
 # set to True only to test failed load
@@ -48,8 +54,8 @@ def check_db_dsn_equal_up_to_credentials(db_dsn_hub, db_dsn_local):
 
 
 def update_db_using_local(
-    hub_instance_result: Dict[str, str], settings_file: Path, db: Optional[str] = None
-) -> Optional[str]:
+    hub_instance_result: dict[str, str], settings_file: Path, db: str | None = None
+) -> str | None:
     db_updated = None
     # check if postgres
     if hub_instance_result["db_scheme"] == "postgresql":
@@ -98,11 +104,11 @@ def update_db_using_local(
 def connect(
     slug: str,
     *,
-    db: Optional[str] = None,
-    storage: Optional[UPathStr] = None,
+    db: str | None = None,
+    storage: UPathStr | None = None,
     _raise_not_reachable_error: bool = True,
     _test: bool = False,
-) -> Optional[Union[str, Tuple]]:
+) -> str | tuple | None:
     """Connect to instance.
 
     Args:
@@ -133,7 +139,7 @@ def connect(
         if settings_file.exists():
             isettings = load_instance_settings(settings_file)
             # mimic instance_result from hub
-            instance_result = {"id": isettings.id.hex}
+            instance_result = {"id": isettings._id.hex}
             # skip hub request for a purely local instance
             make_hub_request = isettings.is_remote
 
@@ -161,7 +167,8 @@ def connect(
                     db=db_updated,
                     schema=instance_result["schema_str"],
                     git_repo=instance_result["git_repo"],
-                    local_storage=instance_result["storage_mode"] == "hybrid",
+                    keep_artifacts_local=bool(instance_result["keep_artifacts_local"]),
+                    is_on_hub=True,
                 )
                 check_whether_migrations_in_sync(instance_result["lamindb_version"])
             else:
@@ -173,12 +180,12 @@ def connect(
                     if isettings.is_remote:
                         if _raise_not_reachable_error:
                             raise InstanceNotFoundError(message)
-                        return "instance-not-reachable"
+                        return "instance-not-found"
 
                 else:
                     if _raise_not_reachable_error:
                         raise InstanceNotFoundError(message)
-                    return "instance-not-reachable"
+                    return "instance-not-found"
 
         if storage is not None:
             update_isettings_with_storage(isettings, storage)
@@ -203,7 +210,7 @@ def connect(
                 raise SystemExit(msg)
             else:
                 logger.warning(
-                    f"instance exists with id {isettings.id.hex}, but database is not"
+                    f"instance exists with id {isettings._id.hex}, but database is not"
                     " loadable: re-initializing"
                 )
                 return "instance-corrupted-or-deleted", instance_result
@@ -224,9 +231,9 @@ def connect(
 def load(
     slug: str,
     *,
-    db: Optional[str] = None,
-    storage: Optional[UPathStr] = None,
-) -> Optional[Union[str, Tuple]]:
+    db: str | None = None,
+    storage: UPathStr | None = None,
+) -> str | tuple | None:
     """Connect to instance and set ``auto-connect`` to true.
 
     This is exactly the same as ``ln.connect()`` except for that

lamindb_setup/_delete.py

@@ -1,17 +1,24 @@
+from __future__ import annotations
+
 import shutil
-from pathlib import Path
-from lamin_utils import logger
+from typing import TYPE_CHECKING, Optional
 from uuid import UUID
-from typing import Optional
-from .core._settings_instance import InstanceSettings
-from .core._settings_storage import StorageSettings
+
+from lamin_utils import logger
+
+from ._connect_instance import INSTANCE_NOT_FOUND_MESSAGE
+from .core._hub_core import connect_instance as load_instance_from_hub
+from .core._hub_core import delete_instance as delete_instance_on_hub
+from .core._hub_core import get_storage_records_for_instance
 from .core._settings import settings
+from .core._settings_instance import InstanceSettings
 from .core._settings_load import load_instance_settings
+from .core._settings_storage import StorageSettings
 from .core._settings_store import instance_settings_file
-from .core.upath import check_storage_is_empty, hosted_buckets
-from .core._hub_core import delete_instance as delete_instance_on_hub
-from .core._hub_core import connect_instance as load_instance_from_hub
-from ._connect_instance import INSTANCE_NOT_FOUND_MESSAGE
+from .core.upath import HOSTED_BUCKETS, check_storage_is_empty
+
+if TYPE_CHECKING:
+    from pathlib import Path
 
 
 def delete_cache(cache_dir: Path):
@@ -20,7 +27,7 @@ def delete_cache(cache_dir: Path):
 
 
 def delete_exclusion_dir(isettings: InstanceSettings) -> None:
-    exclusion_dir = isettings.storage.root / f".lamindb/_exclusion/{isettings.id.hex}"
+    exclusion_dir = isettings.storage.root / f".lamindb/_exclusion/{isettings._id.hex}"
     if exclusion_dir.exists():
         exclusion_dir.rmdir()
 
@@ -45,13 +52,11 @@ def delete_by_isettings(isettings: InstanceSettings) -> None:
         if settings._instance_settings_path.exists():
             settings._instance_settings_path.unlink()
         settings._instance_settings = None
-    if isettings.storage._mark_storage_root.exists():
-        isettings.storage._mark_storage_root.unlink()
 
 
 def delete(
     instance_name: str, force: bool = False, require_empty: bool = True
-) -> Optional[int]:
+) -> int | None:
     """Delete a LaminDB instance.
 
     Args:
@@ -94,7 +99,7 @@ def delete(
                 owner=settings.user.handle,
                 name=instance_name,
                 storage=ssettings,
-                local_storage=instance_result["storage_mode"] == "hybrid",
+                keep_artifacts_local=bool(instance_result["keep_artifacts_local"]),
                 db=instance_result["db"] if "db" in instance_result else None,
                 schema=instance_result["schema_str"],
                 git_repo=instance_result["git_repo"],
@@ -105,6 +110,10 @@ def delete(
         logger.warning(
             f"delete() does not yet affect your Postgres database at {isettings.db}"
         )
+    if isettings.is_on_hub and settings.user.handle == "anonymous":
+        logger.warning(
+            "won't delete the hub component of this instance because you're not logged in"
+        )
     if not force:
         valid_responses = ["y", "yes"]
         user_input = (
@@ -120,25 +129,49 @@ def delete(
         # delete the exlusion dir first because it's hard to count its objects
         delete_exclusion_dir(isettings)
     if isettings.storage.type_is_cloud and isettings.storage.root_as_str.startswith(
-        hosted_buckets
+        HOSTED_BUCKETS
     ):
         if not require_empty:
             logger.warning(
                 "hosted storage always has to be empty, ignoring `require_empty`"
             )
         require_empty = True
+    # first the default storage
     n_objects = check_storage_is_empty(
         isettings.storage.root,
         raise_error=require_empty,
         account_for_sqlite_file=isettings.dialect == "sqlite",
     )
+    if isettings.storage._mark_storage_root.exists():
+        isettings.storage._mark_storage_root.unlink(
+            missing_ok=True  # this is totally weird, but needed on Py3.11
+        )
+    # now everything that's on the hub
+    if settings.user.handle != "anonymous":
+        storage_records = get_storage_records_for_instance(isettings._id)
+        for storage_record in storage_records:
+            if storage_record["root"] == isettings.storage.root_as_str:
+                continue
+            check_storage_is_empty(
+                storage_record["root"],  # type: ignore
+                raise_error=require_empty,
+            )
+            ssettings = StorageSettings(storage_record["root"])  # type: ignore
+            if ssettings._mark_storage_root.exists():
+                ssettings._mark_storage_root.unlink(
+                    missing_ok=True  # this is totally weird, but needed on Py3.11
+                )
     logger.info(f"deleting instance {instance_slug}")
     # below we can skip check_storage_is_empty() because we already called
     # it above
-    delete_instance_on_hub(isettings.id, require_empty=False)
+    if settings.user.handle != "anonymous":
+        delete_instance_on_hub(isettings._id, require_empty=False)
     delete_by_isettings(isettings)
-    if n_objects == 0 and isettings.storage.type == "local":
+    # if .lndb file was delete, then we might count -1
+    if n_objects <= 0 and isettings.storage.type == "local":
         # dir is only empty after sqlite file was delete via delete_by_isettings
-        (isettings.storage.root / ".lamindb").rmdir()
-        isettings.storage.root.rmdir()
+        if (isettings.storage.root / ".lamindb").exists():
+            (isettings.storage.root / ".lamindb").rmdir()
+        if isettings.storage.root.exists():
+            isettings.storage.root.rmdir()
     return None

lamindb_setup/_django.py

@@ -1,9 +1,12 @@
+from __future__ import annotations
+
 from typing import Optional
+
 from .core._settings import settings
 from .core.django import setup_django
 
 
-def django(command: str, package_name: Optional[str] = None, **kwargs):
+def django(command: str, package_name: str | None = None, **kwargs):
     r"""Manage migrations.
 
     Examples:

lamindb_setup/_exportdb.py

@@ -1,6 +1,7 @@
-from pathlib import Path
-from importlib import import_module
+from __future__ import annotations
 
+from importlib import import_module
+from pathlib import Path
 
 MODELS = {
     "core": {
@@ -46,6 +47,7 @@ def exportdb() -> None:
     directory = Path("./lamindb_export/")
     directory.mkdir(parents=True, exist_ok=True)
     import pandas as pd
+
     import lamindb_setup as ln_setup
 
     def export_registry(registry, directory):

lamindb_setup/_importdb.py

@@ -1,6 +1,9 @@
+from __future__ import annotations
+
+from importlib import import_module
 from pathlib import Path
+
 from ._exportdb import MODELS
-from importlib import import_module
 
 
 def import_registry(registry, directory, connection):
@@ -26,6 +29,7 @@ def importdb() -> None:
         if response != "y":
             return None
     from sqlalchemy import create_engine, text
+
     import lamindb_setup as ln_setup
 
     engine = create_engine(ln_setup.settings.instance.db, echo=False)