lamindb_setup 1.6.1__py3-none-any.whl → 1.7.0__py3-none-any.whl

lamindb_setup/core/__init__.py

@@ -12,7 +12,7 @@ Settings:
 
 """
 
-from . import django, types, upath
+from . import django, upath
 from ._deprecated import deprecated
 from ._docs import doc_args
 from ._settings import SetupSettings

lamindb_setup/core/_aws_options.py

@@ -31,7 +31,7 @@ AWS_CREDENTIALS_EXPIRATION: int = 11 * 60 * 60  # refresh credentials after 11 h
 
 # set anon=True for these buckets if credentials fail for a public bucket
 # to be expanded
-PUBLIC_BUCKETS: tuple[str] = ("cellxgene-data-public",)
+PUBLIC_BUCKETS: tuple[str, ...] = ("cellxgene-data-public", "bionty-assets")
 
 
 # s3-comaptible endpoints managed by lamin
@@ -79,8 +79,8 @@ class AWSOptionsManager:
                 # use lamindata public bucket for this test
                 fs.call_s3("head_bucket", Bucket="lamindata")
                 self.anon_public = False
-            except Exception as e:
-                self.anon_public = isinstance(e, PermissionError)
+            except Exception:
+                self.anon_public = True
 
     def _find_root(self, path_str: str) -> str | None:
         roots = self._credentials_cache.keys()

lamindb_setup/core/_hub_client.py

@@ -5,7 +5,6 @@ import os
 from typing import Literal
 from urllib.request import urlretrieve
 
-from gotrue.errors import AuthUnknownError
 from lamin_utils import logger
 from pydantic_settings import BaseSettings
 from supabase import Client, create_client  # type: ignore
@@ -26,7 +25,7 @@ def load_fallback_connector() -> Connector:
     return connector
 
 
-PROD_URL = "https://laesaummdydllppgfchu.supabase.co"
+PROD_URL = "https://hub.lamin.ai"
 PROD_ANON_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImxhZXNhdW1tZHlkbGxwcGdmY2h1Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2NTY4NDA1NTEsImV4cCI6MTk3MjQxNjU1MX0.WUeCRiun0ExUxKIv5-CtjF6878H8u26t0JmCWx3_2-c"
 
 
@@ -173,7 +172,7 @@ def call_with_fallback(
             client = connect_hub(fallback_env=fallback_env)
             result = callable(**kwargs, client=client)
             break
-        except AuthUnknownError as e:
+        except Exception as e:
             if fallback_env:
                 raise e
         finally:

lamindb_setup/core/_hub_core.py

@@ -35,28 +35,36 @@ from ._hub_utils import (
 )
 from ._settings import settings
 from ._settings_instance import InstanceSettings
-from ._settings_storage import StorageSettings, base62
+from ._settings_storage import StorageSettings, base62, instance_uid_from_uuid
 from .hashing import hash_and_encode_as_b62
 
 if TYPE_CHECKING:
     from supabase import Client  # type: ignore
 
 
-def delete_storage_record(storage_uuid: UUID, access_token: str | None = None) -> None:
+def delete_storage_record(
+    storage_info: dict[str, str] | StorageSettings,
+    access_token: str | None = None,
+) -> None:
+    if isinstance(storage_info, StorageSettings):
+        storage_info = {"id": storage_info._uuid.hex, "root": storage_info.root_as_str}  # type: ignore
     return call_with_fallback_auth(
-        _delete_storage_record, storage_uuid=storage_uuid, access_token=access_token
+        _delete_storage_record, storage_info=storage_info, access_token=access_token
     )
 
 
-def _delete_storage_record(storage_uuid: UUID, client: Client) -> None:
+def _delete_storage_record(storage_info: dict[str, str], client: Client) -> None:
+    storage_uuid = UUID(storage_info["id"])
     if storage_uuid is None:
         return None
     response = client.table("storage").delete().eq("id", storage_uuid.hex).execute()
     if response.data:
-        logger.important(f"deleted storage record on hub {storage_uuid.hex}")
+        logger.important(
+            f"deleted storage record on hub {storage_uuid.hex} | {storage_info['root']}"
+        )
     else:
         raise PermissionError(
-            f"Deleting of storage with {storage_uuid.hex} was not successful. Probably, you"
+            f"Deleting of storage {storage_uuid.hex} ({storage_info['root']}) was not successful. Probably, you"
             " don't have sufficient permissions."
         )
 
@@ -94,21 +102,21 @@ def _select_storage(
         return False
     else:
         existing_storage = response.data[0]
-        if existing_storage["instance_id"] is not None:
-            if ssettings._instance_id is not None:
-                # consider storage settings that are meant to be managed by an instance
-                if UUID(existing_storage["instance_id"]) != ssettings._instance_id:
-                    # everything is alright if the instance_id matches
-                    # we're probably just switching storage locations
-                    # below can be turned into a warning and then delegate the error
-                    # to a unique constraint violation below
-                    raise ValueError(
-                        f"Storage root {root} is already managed by instance {existing_storage['instance_id']}."
-                    )
-            else:
-                # if the request is agnostic of the instance, that's alright,
-                # we'll update the instance_id with what's stored in the hub
-                ssettings._instance_id = UUID(existing_storage["instance_id"])
+        if existing_storage["instance_id"] is None:
+            # if there is no instance_id, the storage location should not be on the hub
+            # this can only occur if instance init fails halfway through and is not cleaned up
+            # we're patching the situation here
+            existing_storage["instance_id"] = (
+                ssettings._instance_id.hex
+                if ssettings._instance_id is not None
+                else None
+            )
+        if ssettings._instance_id is not None:
+            if UUID(existing_storage["instance_id"]) != ssettings._instance_id:
+                logger.debug(
+                    f"referencing storage location {root}, which is managed by instance {existing_storage['instance_id']}"
+                )
+        ssettings._instance_id = UUID(existing_storage["instance_id"])
         ssettings._uuid_ = UUID(existing_storage["id"])
         if update_uid:
             ssettings._uid = existing_storage["lnid"]
@@ -122,9 +130,7 @@ def _select_storage_or_parent(path: str, client: Client) -> dict | None:
     if result["root"] is None:
         return None
     result["uid"] = result.pop("lnid")
-    result["instance_uid"] = hash_and_encode_as_b62(
-        UUID(result.pop("instance_id")).hex
-    )[:12]
+    result["instance_uid"] = instance_uid_from_uuid(UUID(result.pop("instance_id")))
     return result
 
 
@@ -144,7 +150,9 @@ def init_storage_hub(
     auto_populate_instance: bool = True,
     created_by: UUID | None = None,
     access_token: str | None = None,
-) -> Literal["hub-record-retrieved", "hub-record-created"]:
+    prevent_creation: bool = False,
+) -> Literal["hub-record-retrieved", "hub-record-created", "hub-record-not-created"]:
+    """Creates or retrieves an existing storage record from the hub."""
     if settings.user.handle != "anonymous" or access_token is not None:
         return call_with_fallback_auth(
             _init_storage_hub,
@@ -152,6 +160,7 @@ def init_storage_hub(
             auto_populate_instance=auto_populate_instance,
             created_by=created_by,
             access_token=access_token,
+            prevent_creation=prevent_creation,
         )
     else:
         storage_exists = call_with_fallback(
@@ -160,7 +169,7 @@ def init_storage_hub(
         if storage_exists:
             return "hub-record-retrieved"
         else:
-            raise ValueError("Log in to create a storage location on the hub.")
+            return "hub-record-not-created"
 
 
 def _init_storage_hub(
@@ -168,7 +177,8 @@ def _init_storage_hub(
     ssettings: StorageSettings,
     auto_populate_instance: bool,
     created_by: UUID | None = None,
-) -> Literal["hub-record-retrieved", "hub-record-created"]:
+    prevent_creation: bool = False,
+) -> Literal["hub-record-retrieved", "hub-record-created", "hub-record-not-created"]:
     from lamindb_setup import settings
 
     created_by = settings.user._uuid if created_by is None else created_by
@@ -177,6 +187,8 @@ def _init_storage_hub(
     root = ssettings.root_as_str
     if _select_storage(ssettings, update_uid=True, client=client):
         return "hub-record-retrieved"
+    if prevent_creation:
+        return "hub-record-not-created"
     if ssettings.type_is_cloud:
         id = uuid.uuid5(uuid.NAMESPACE_URL, root)
     else:
@@ -272,17 +284,11 @@ def _delete_instance(
             else:
                 access_token = None
             root_path = create_path(root_string, access_token)
-            mark_storage_root(
-                root_path,
-                storage_record["lnid"],  # type: ignore
-            )  # address permission error
             check_storage_is_empty(
                 root_path, account_for_sqlite_file=account_for_sqlite_file
             )
-    # first delete the storage records because we will turn instance_id on
-    # storage into a FK soon
     for storage_record in storage_records:
-        _delete_storage_record(UUID(storage_record["id"]), client)  # type: ignore
+        _delete_storage_record(storage_record, client)  # type: ignore
     _delete_instance_record(UUID(instance_with_storage["id"]), client)
     return None
 
@@ -361,6 +367,27 @@ def _connect_instance_hub(
         "get-instance-settings-v1",
         invoke_options={"body": {"owner": owner, "name": name}},
     )
+    # check instance renames
+    if response == b"{}":
+        data = (
+            client.table("instance_previous_name")
+            .select(
+                "instance!instance_previous_name_instance_id_17ac5d61_fk_instance_id(name, account!instance_account_id_28936e8f_fk_account_id(handle))"
+            )
+            .eq("instance.account.handle", owner)
+            .eq("previous_name", name)
+            .execute()
+            .data
+        )
+        if len(data) != 0 and (instance_data := data[0]["instance"]) is not None:
+            new_name = instance_data["name"]  # the instance was renamed
+            logger.warning(
+                f"'{owner}/{name}' was renamed, please use '{owner}/{new_name}'"
+            )
+            response = client.functions.invoke(
+                "get-instance-settings-v1",
+                invoke_options={"body": {"owner": owner, "name": new_name}},
+            )
     # no instance found, check why is that
     if response == b"{}":
         # try the via single requests, will take more time
@@ -375,7 +402,7 @@ def _connect_instance_hub(
         if storage is None:
             return "default-storage-does-not-exist-on-hub"
         logger.warning(
-            "Could not find instance via API, but found directly querying hub."
+            "could not find instance via API, but found directly querying hub"
         )
     else:
         instance = json.loads(response)
@@ -467,6 +494,12 @@ def access_db(
     instance_id: UUID
     instance_slug: str
     instance_api_url: str | None
+    if (
+        "LAMIN_TEST_DB_TOKEN" in os.environ
+        and (env_db_token := os.environ["LAMIN_TEST_DB_TOKEN"]) != ""
+    ):
+        return env_db_token
+
     if isinstance(instance, InstanceSettings):
         instance_id = instance._id
         instance_slug = instance.slug

lamindb_setup/core/_hub_crud.py

@@ -68,9 +68,23 @@ def select_instance_by_name(
         .execute()
         .data
     )
-    if len(data) == 0:
-        return None
-    return data[0]
+    if len(data) != 0:
+        return data[0]
+
+    data = (
+        client.table("instance_previous_name")
+        .select(
+            "instance!instance_previous_name_instance_id_17ac5d61_fk_instance_id(*)"
+        )
+        .eq("instance.account_id", account_id)
+        .eq("previous_name", name)
+        .execute()
+        .data
+    )
+    if len(data) != 0:
+        return data[0]["instance"]
+
+    return None
 
 
 def select_instance_by_id(

lamindb_setup/core/_settings.py

@@ -4,29 +4,32 @@ import os
 import sys
 from typing import TYPE_CHECKING
 
-from appdirs import AppDirs
 from lamin_utils import logger
+from platformdirs import user_cache_dir
 
 from ._settings_load import (
+    load_cache_path_from_settings,
     load_instance_settings,
     load_or_create_user_settings,
-    load_system_storage_settings,
 )
-from ._settings_store import current_instance_settings_file, settings_dir
+from ._settings_store import (
+    current_instance_settings_file,
+    settings_dir,
+    system_settings_dir,
+)
 from .upath import LocalPathClasses, UPath
 
 if TYPE_CHECKING:
     from pathlib import Path
 
     from lamindb_setup.core import InstanceSettings, StorageSettings, UserSettings
-
-    from .types import UPathStr
+    from lamindb_setup.types import UPathStr
 
 
-DEFAULT_CACHE_DIR = UPath(AppDirs("lamindb", "laminlabs").user_cache_dir)
+DEFAULT_CACHE_DIR = UPath(user_cache_dir(appname="lamindb", appauthor="laminlabs"))
 
 
-def _process_cache_path(cache_path: UPathStr | None):
+def _process_cache_path(cache_path: UPathStr | None) -> UPath | None:
     if cache_path is None or cache_path == "null":
         return None
     cache_dir = UPath(cache_path)
@@ -34,6 +37,8 @@ def _process_cache_path(cache_path: UPathStr | None):
         raise ValueError("cache dir should be a local path.")
     if cache_dir.exists() and not cache_dir.is_dir():
         raise ValueError("cache dir should be a directory.")
+    if not cache_dir.is_absolute():
+        raise ValueError("A path to the cache dir should be absolute.")
     return cache_dir
 
 
@@ -86,6 +91,17 @@ class SetupSettings:
         else:
             self._auto_connect_path.unlink(missing_ok=True)
 
+    @property
+    def is_connected(self) -> bool:
+        """Determine whether the current instance is fully connected and ready to use.
+
+        If `True`, the current instance is connected, meaning that the db and other settings
+        are properly configured for use.
+        """
+        from .django import IS_SETUP  # always import to protect from assignment
+
+        return IS_SETUP
+
     @property
     def private_django_api(self) -> bool:
         """Turn internal Django API private to clean up the API (default `False`).
@@ -156,7 +172,7 @@ class SetupSettings:
         if "LAMIN_CACHE_DIR" in os.environ:
             cache_dir = UPath(os.environ["LAMIN_CACHE_DIR"])
         elif self._cache_dir is None:
-            cache_path = load_system_storage_settings().get("lamindb_cache_path", None)
+            cache_path = load_cache_path_from_settings()
             cache_dir = _process_cache_path(cache_path)
             if cache_dir is None:
                 cache_dir = DEFAULT_CACHE_DIR
@@ -189,10 +205,12 @@ class SetupSettings:
         # do not show current setting representation when building docs
         if "sphinx" in sys.modules:
             return object.__repr__(self)
-        repr = self.user.__repr__()
-        repr += f"\nAuto-connect in Python: {self.auto_connect}\n"
+        repr = f"Auto-connect in Python: {self.auto_connect}\n"
         repr += f"Private Django API: {self.private_django_api}\n"
         repr += f"Cache directory: {self.cache_dir.as_posix()}\n"
+        repr += f"User settings directory: {settings_dir.as_posix()}\n"
+        repr += f"System settings directory: {system_settings_dir.as_posix()}\n"
+        repr += self.user.__repr__() + "\n"
         if self._instance_exists:
             repr += self.instance.__repr__()
         else:

lamindb_setup/core/_settings_instance.py

@@ -18,6 +18,7 @@ from ._settings_storage import (
     STORAGE_UID_FILE_KEY,
     StorageSettings,
     init_storage,
+    instance_uid_from_uuid,
 )
 from ._settings_store import current_instance_settings_file, instance_settings_file
 from .cloud_sqlite_locker import (
@@ -117,6 +118,8 @@ class InstanceSettings:
                 else:
                     db_print = value
                 representation += f"\n- {attr}: {db_print}"
+            elif attr == "modules":
+                representation += f"\n- {attr}: {value if value else '{}'}"
             else:
                 representation += f"\n- {attr}: {value}"
         return representation
@@ -165,7 +168,7 @@ class InstanceSettings:
                             f"local storage location '{root_path}' is corrupted, cannot find marker file with storage uid"
                         )
                 try:
-                    uid = marker_path.read_text()
+                    uid = marker_path.read_text().splitlines()[0]
                 except PermissionError:
                     logger.warning(
                         f"ignoring the following location because no permission to read it: {marker_path}"
@@ -250,7 +253,9 @@ class InstanceSettings:
                 return None
         local_root = UPath(local_root)
         assert isinstance(local_root, LocalPathClasses)
-        self._storage_local, _ = init_storage(local_root, self._id, register_hub=True)  # type: ignore
+        self._storage_local, _ = init_storage(
+            local_root, instance_id=self._id, instance_slug=self.slug, register_hub=True
+        )  # type: ignore
         register_storage_in_instance(self._storage_local)  # type: ignore
         logger.important(f"defaulting to local storage: {self._storage_local.root}")
 
@@ -275,9 +280,7 @@ class InstanceSettings:
     @property
     def uid(self) -> str:
         """The user-facing instance id."""
-        from .hashing import hash_and_encode_as_b62
-
-        return hash_and_encode_as_b62(self._id.hex)[:12]
+        return instance_uid_from_uuid(self._id)
 
     @property
     def modules(self) -> set[str]:

lamindb_setup/core/_settings_load.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+from pathlib import Path
 from typing import TYPE_CHECKING
 from uuid import UUID, uuid4
 
@@ -8,6 +9,8 @@ from dotenv import dotenv_values
 from lamin_utils import logger
 from pydantic import ValidationError
 
+from lamindb_setup.errors import SettingsEnvFileOutdated
+
 from ._settings_instance import InstanceSettings
 from ._settings_storage import StorageSettings
 from ._settings_store import (
@@ -15,26 +18,32 @@ from ._settings_store import (
     UserSettingsStore,
     current_instance_settings_file,
     current_user_settings_file,
-    system_storage_settings_file,
+    platform_user_storage_settings_file,
+    system_settings_file,
 )
 from ._settings_user import UserSettings
 
-if TYPE_CHECKING:
-    from pathlib import Path
-
-
-class SettingsEnvFileOutdated(Exception):
-    pass
 
+def load_cache_path_from_settings(storage_settings: Path | None = None) -> Path | None:
+    if storage_settings is None:
+        paltform_user_storage_settings = platform_user_storage_settings_file()
+        if paltform_user_storage_settings.exists():
+            cache_path = dotenv_values(paltform_user_storage_settings).get(
+                "lamindb_cache_path", None
+            )
+        else:
+            cache_path = None
 
-def load_system_storage_settings(system_storage_settings: Path | None = None) -> dict:
-    if system_storage_settings is None:
-        system_storage_settings = system_storage_settings_file()
+        if cache_path in {None, "null", ""}:
+            storage_settings = system_settings_file()
+        else:
+            return Path(cache_path)
 
-    if system_storage_settings.exists():
-        return dotenv_values(system_storage_settings)
+    if storage_settings.exists():
+        cache_path = dotenv_values(storage_settings).get("lamindb_cache_path", None)
+        return Path(cache_path) if cache_path not in {None, "null", ""} else None
     else:
-        return {}
+        return None
 
 
 def load_instance_settings(instance_settings_file: Path | None = None):
@@ -113,7 +122,7 @@ def setup_instance_from_store(store: InstanceSettingsStore) -> InstanceSettings:
         git_repo=_null_to_value(store.git_repo),
         keep_artifacts_local=store.keep_artifacts_local,  # type: ignore
         api_url=_null_to_value(store.api_url),
-        schema_id=None if store.schema_id == "null" else UUID(store.schema_id),
+        schema_id=None if store.schema_id in {None, "null"} else UUID(store.schema_id),
         fine_grained_access=store.fine_grained_access,
         db_permissions=_null_to_value(store.db_permissions),
     )

lamindb_setup/core/_settings_save.py

@@ -8,14 +8,15 @@ from ._settings_store import (
     InstanceSettingsStore,
     UserSettingsStore,
     current_user_settings_file,
-    system_storage_settings_file,
+    platform_user_storage_settings_file,
     user_settings_file_email,
     user_settings_file_handle,
 )
 
 if TYPE_CHECKING:
+    from lamindb_setup.types import UPathStr
+
     from ._settings_user import UserSettings
-    from .types import UPathStr
 
 
 def save_user_settings(settings: UserSettings):
@@ -82,13 +83,13 @@ def save_instance_settings(settings: Any, settings_file: Path):
     save_settings(settings, settings_file, type_hints, prefix)
 
 
-def save_system_storage_settings(
+def save_platform_user_storage_settings(
     cache_path: UPathStr | None, settings_file: UPathStr | None = None
 ):
     cache_path = "null" if cache_path is None else cache_path
     if isinstance(cache_path, Path):  # also True for UPath
         cache_path = cache_path.as_posix()
     if settings_file is None:
-        settings_file = system_storage_settings_file()
+        settings_file = platform_user_storage_settings_file()
     with open(settings_file, "w") as f:
         f.write(f"lamindb_cache_path={cache_path}")