lamindb_setup 0.69.5__py2.py3-none-any.whl → 0.71.0__py2.py3-none-any.whl

lamindb_setup/core/_hub_core.py

@@ -1,35 +1,41 @@
+from __future__ import annotations
+
+import json
 import os
-from typing import Optional, Tuple, Union, Dict
 import uuid
-from postgrest.exceptions import APIError
-from uuid import UUID
-from lamin_utils import logger
-from supabase import Client
-import lamindb_setup
-import json
 from importlib import metadata
-from ._settings_instance import InstanceSettings
-from ._settings_storage import StorageSettings, base62
+from typing import TYPE_CHECKING
+from uuid import UUID
 
+from lamin_utils import logger
+from postgrest.exceptions import APIError
 
 from ._hub_client import (
-    connect_hub,
-    call_with_fallback_auth,
     call_with_fallback,
+    call_with_fallback_auth,
+    connect_hub,
 )
 from ._hub_crud import (
-    select_instance_by_owner_name,
-    select_instance_by_id_with_storage,
+    _delete_instance_record,
     select_account_by_handle,
     select_db_user_by_instance,
+    select_instance_by_id_with_storage,
     select_instance_by_name,
+    select_instance_by_owner_name,
     select_storage,
-    _delete_instance_record,
 )
+from ._hub_crud import update_instance as _update_instance_record
 from ._hub_utils import (
     LaminDsn,
     LaminDsnModel,
 )
+from ._settings import settings
+from ._settings_storage import StorageSettings, base62
+
+if TYPE_CHECKING:
+    from supabase import Client  # type: ignore
+
+    from ._settings_instance import InstanceSettings
 
 
 def delete_storage_record(
@@ -44,78 +50,188 @@ def delete_storage_record(
 def _delete_storage_record(storage_uuid: UUID, client: Client) -> None:
     if storage_uuid is None:
         return None
-    logger.important(f"deleting storage {storage_uuid.hex}")
-    client.table("storage").delete().eq("id", storage_uuid.hex).execute()
+    response = client.table("storage").delete().eq("id", storage_uuid.hex).execute()
+    if response.data:
+        logger.important(f"deleted storage record on hub {storage_uuid.hex}")
 
 
-def init_storage(
-    ssettings: StorageSettings,
-) -> UUID:
+def update_instance_record(instance_uuid: UUID, fields: dict) -> None:
     return call_with_fallback_auth(
-        _init_storage,
-        ssettings=ssettings,
+        _update_instance_record, instance_id=instance_uuid.hex, instance_fields=fields
+    )
+
+
+def get_storage_records_for_instance(
+    instance_id: UUID,
+) -> list[dict[str, str | int]]:
+    return call_with_fallback_auth(
+        _get_storage_records_for_instance,
+        instance_id=instance_id,
+    )
+
+
+def _get_storage_records_for_instance(
+    instance_id: UUID, client: Client
+) -> list[dict[str, str | int]]:
+    response = (
+        client.table("storage").select("*").eq("instance_id", instance_id.hex).execute()
     )
+    return response.data
+
+
+def _select_storage(
+    ssettings: StorageSettings, update_uid: bool, client: Client
+) -> bool:
+    root = ssettings.root_as_str
+    response = client.table("storage").select("*").eq("root", root).execute()
+    if not response.data:
+        return False
+    else:
+        existing_storage = response.data[0]
+        if ssettings._instance_id is not None:
+            # consider storage settings that are meant to be managed by an instance
+            if UUID(existing_storage["instance_id"]) != ssettings._instance_id:
+                # everything is alright if the instance_id matches
+                # we're probably just switching storage locations
+                # below can be turned into a warning and then delegate the error
+                # to a unique constraint violation below
+                raise ValueError(
+                    f"Storage root {root} is already managed by instance {existing_storage['instance_id']}."
+                )
+        else:
+            # if the request is agnostic of the instance, that's alright,
+            # we'll update the instance_id with what's stored in the hub
+            ssettings._instance_id = UUID(existing_storage["instance_id"])
+        ssettings._uuid_ = UUID(existing_storage["id"])
+        if update_uid:
+            ssettings._uid = existing_storage["lnid"]
+        else:
+            assert ssettings._uid == existing_storage["lnid"]
+        return True
+
+
+def init_storage(
+    ssettings: StorageSettings,
+) -> None:
+    if settings.user.handle != "anonymous":
+        return call_with_fallback_auth(
+            _init_storage,
+            ssettings=ssettings,
+        )
+    else:
+        storage_exists = call_with_fallback(
+            _select_storage, ssettings=ssettings, update_uid=True
+        )
+        if storage_exists:
+            return None
+        else:
+            raise ValueError("Log in to create a storage location on the hub.")
 
 
-def _init_storage(ssettings: StorageSettings, client: Client) -> UUID:
+def _init_storage(ssettings: StorageSettings, client: Client) -> None:
     from lamindb_setup import settings
 
     # storage roots are always stored without the trailing slash in the SQL
     # database
     root = ssettings.root_as_str
-    # in the future, we could also encode a prefix to model local storage
-    # locations
-    # f"{prefix}{root}"
-    id = uuid.uuid5(uuid.NAMESPACE_URL, root)
-    fields = dict(
-        id=id.hex,
-        lnid=ssettings.uid,
-        created_by=settings.user.uuid.hex,  # type: ignore
-        root=root,
-        region=ssettings.region,
-        type=ssettings.type,
-        aws_account_id=ssettings._aws_account_id,
-        description=ssettings._description,
-    )
+    if _select_storage(ssettings, update_uid=True, client=client):
+        return None
+    if ssettings.type_is_cloud:
+        id = uuid.uuid5(uuid.NAMESPACE_URL, root)
+    else:
+        id = uuid.uuid4()
+    if ssettings._instance_id is None:
+        logger.warning(
+            f"will manage storage location {ssettings.root_as_str} with instance {settings.instance.slug}"
+        )
+        ssettings._instance_id = settings.instance._id
+    fields = {
+        "id": id.hex,
+        "lnid": ssettings.uid,
+        "created_by": settings.user._uuid.hex,  # type: ignore
+        "root": root,
+        "region": ssettings.region,
+        "type": ssettings.type,
+        "instance_id": ssettings._instance_id.hex,
+        # the empty string is important as we want the user flow to be through LaminHub
+        # if this errors with unique constraint error, the user has to update
+        # the description in LaminHub
+        "description": "",
+    }
+    # TODO: add error message for violated unique constraint
+    # on root & description
     client.table("storage").upsert(fields).execute()
-    return id
+    ssettings._uuid_ = id
+    return None
+
+
+def delete_instance(identifier: UUID | str, require_empty: bool = True) -> str | None:
+    return call_with_fallback_auth(
+        _delete_instance, identifier=identifier, require_empty=require_empty
+    )
 
 
-def delete_instance(identifier: Union[UUID, str]):
+def _delete_instance(
+    identifier: UUID | str, require_empty: bool, client: Client
+) -> str | None:
     """Fully delete an instance in the hub.
 
     This function deletes the relevant instance and storage records in the hub,
     conditional on the emptiness of the storage location.
     """
-    from .upath import check_s3_storage_location_empty, create_path, hosted_buckets
     from ._settings_storage import mark_storage_root
+    from .upath import check_storage_is_empty, create_path
 
-    if isinstance(identifier, UUID):
-        instance_with_storage = call_with_fallback_auth(
-            select_instance_by_id_with_storage,
-            instance_id=identifier,
+    # the "/" check is for backward compatibility with the old identifier format
+    if isinstance(identifier, UUID) or "/" not in identifier:
+        if isinstance(identifier, UUID):
+            instance_id_str = identifier.hex
+        else:
+            instance_id_str = identifier
+        instance_with_storage = select_instance_by_id_with_storage(
+            instance_id=instance_id_str, client=client
         )
     else:
         owner, name = identifier.split("/")
-        instance_with_storage = call_with_fallback_auth(
-            select_instance_by_owner_name,
-            owner=owner,
-            name=name,
+        instance_with_storage = select_instance_by_owner_name(
+            owner=owner, name=name, client=client
         )
 
-    if instance_with_storage is not None:
-        root_string = instance_with_storage["storage"]["root"]
-        # gate storage and instance deletion on empty storage location for
-        # both hosted and non-hosted s3 instances
-        if root_string.startswith("s3://"):
-            root_path = create_path(root_string)
-            # only mark hosted instances
-            if root_string.startswith(hosted_buckets):
-                mark_storage_root(root_path)
-            check_s3_storage_location_empty(root_path)
+    if instance_with_storage is None:
+        logger.important("not deleting instance from hub as instance not found there")
+        return "instance-not-found"
 
-        delete_instance_record(UUID(instance_with_storage["id"]))
-        delete_storage_record(UUID(instance_with_storage["storage_id"]))
+    storage_records = _get_storage_records_for_instance(
+        UUID(instance_with_storage["id"]),
+        client,
+    )
+    if require_empty:
+        for storage_record in storage_records:
+            account_for_sqlite_file = (
+                instance_with_storage["db_scheme"] is None
+                and instance_with_storage["storage"]["root"] == storage_record["root"]
+            )
+            root_string = storage_record["root"]
+            # gate storage and instance deletion on empty storage location for
+            if client.auth.get_session() is not None:
+                access_token = client.auth.get_session().access_token
+            else:
+                access_token = None
+            root_path = create_path(root_string, access_token)
+            mark_storage_root(
+                root_path,
+                storage_record["lnid"],  # type: ignore
+            )  # address permission error
+            check_storage_is_empty(
+                root_path, account_for_sqlite_file=account_for_sqlite_file
+            )
+    _update_instance_record(instance_with_storage["id"], {"storage_id": None}, client)
+    # first delete the storage records because we will turn instance_id on
+    # storage into a FK soon
+    for storage_record in storage_records:
+        _delete_storage_record(UUID(storage_record["id"]), client)  # type: ignore
+    _delete_instance_record(UUID(instance_with_storage["id"]), client)
+    return None
 
 
 def delete_instance_record(
@@ -138,25 +254,25 @@ def _init_instance(isettings: InstanceSettings, client: Client) -> None:
         lamindb_version = metadata.version("lamindb")
     except metadata.PackageNotFoundError:
         lamindb_version = None
-    fields = dict(
-        id=isettings.id.hex,
-        account_id=settings.user.uuid.hex,  # type: ignore
-        name=isettings.name,
-        storage_id=isettings.storage.uuid.hex,  # type: ignore
-        schema_str=isettings._schema_str,
-        lamindb_version=lamindb_version,
-        public=False,
-    )
+    fields = {
+        "id": isettings._id.hex,
+        "account_id": settings.user._uuid.hex,  # type: ignore
+        "name": isettings.name,
+        "storage_id": isettings.storage._uuid.hex,  # type: ignore
+        "lnid": isettings.uid,
+        "schema_str": isettings._schema_str,
+        "lamindb_version": lamindb_version,
+        "public": False,
+    }
     if isettings.dialect != "sqlite":
         db_dsn = LaminDsnModel(db=isettings.db)
-        fields.update(
-            {
-                "db_scheme": db_dsn.db.scheme,
-                "db_host": db_dsn.db.host,
-                "db_port": db_dsn.db.port,
-                "db_database": db_dsn.db.database,
-            }
-        )
+        db_fields = {
+            "db_scheme": db_dsn.db.scheme,
+            "db_host": db_dsn.db.host,
+            "db_port": db_dsn.db.port,
+            "db_database": db_dsn.db.database,
+        }
+        fields.update(db_fields)
     # I'd like the following to be an upsert, but this seems to violate RLS
     # Similarly, if we don't specify `returning="minimal"`, we'll violate RLS
     # we could make this idempotent by catching an error, but this seems dangerous
@@ -166,6 +282,9 @@ def _init_instance(isettings: InstanceSettings, client: Client) -> None:
     except APIError as e:
         logger.warning("instance likely already exists")
         raise e
+    client.table("storage").update(
+        {"instance_id": isettings._id.hex, "is_default": True}
+    ).eq("id", isettings.storage._uuid.hex).execute()  # type: ignore
     logger.save(f"browse to: https://lamin.ai/{isettings.owner}/{isettings.name}")
 
 
@@ -173,7 +292,7 @@ def connect_instance(
     *,
     owner: str,  # account_handle
     name: str,  # instance_name
-) -> Union[Tuple[dict, dict], str]:
+) -> tuple[dict, dict] | str:
     from ._settings import settings
 
     if settings.user.handle != "anonymous":
@@ -187,7 +306,7 @@ def _connect_instance(
     owner: str,  # account_handle
     name: str,  # instance_name
     client: Client,
-) -> Union[Tuple[dict, dict], str]:
+) -> tuple[dict, dict] | str:
     instance_account_storage = select_instance_by_owner_name(owner, name, client)
     if instance_account_storage is None:
         # try the via single requests, will take more time
@@ -196,7 +315,7 @@ def _connect_instance(
             return "account-not-exists"
         instance = select_instance_by_name(account["id"], name, client)
         if instance is None:
-            return "instance-not-reachable"
+            return "instance-not-found"
         # get default storage
         storage = select_storage(instance["storage_id"], client)
         if storage is None:
@@ -227,7 +346,7 @@ def _connect_instance(
     return instance, storage
 
 
-def access_aws(storage_root: str, access_token: Optional[str] = None) -> Dict[str, str]:
+def access_aws(storage_root: str, access_token: str | None = None) -> dict[str, str]:
     from ._settings import settings
 
     if settings.user.handle != "anonymous" or access_token is not None:
@@ -239,7 +358,9 @@ def access_aws(storage_root: str, access_token: Optional[str] = None) -> Dict[st
         raise RuntimeError("Can only get access to AWS if authenticated.")
 
 
-def _access_aws(*, storage_root: str, client: Client) -> Dict[str, str]:
+def _access_aws(*, storage_root: str, client: Client) -> dict[str, str]:
+    import lamindb_setup
+
     response = client.functions.invoke(
         "access-aws",
         invoke_options={"body": {"storage_root": storage_root}},
@@ -265,7 +386,7 @@ def get_lamin_site_base_url():
     return "https://lamin.ai"
 
 
-def sign_up_local_hub(email) -> Union[str, Tuple[str, str, str]]:
+def sign_up_local_hub(email) -> str | tuple[str, str, str]:
     # raises gotrue.errors.AuthApiError: User already registered
     password = base62(40)  # generate new password
     sign_up_kwargs = {"email": email, "password": password}
@@ -279,7 +400,7 @@ def sign_up_local_hub(email) -> Union[str, Tuple[str, str, str]]:
     )
 
 
-def _sign_in_hub(email: str, password: str, handle: Optional[str], client: Client):
+def _sign_in_hub(email: str, password: str, handle: str | None, client: Client):
     auth = client.auth.sign_in_with_password(
         {
             "email": email,
@@ -309,8 +430,8 @@ def _sign_in_hub(email: str, password: str, handle: Optional[str], client: Clien
 
 
 def sign_in_hub(
-    email: str, password: str, handle: Optional[str] = None
-) -> Union[Exception, Tuple[UUID, str, str, str, str]]:
+    email: str, password: str, handle: str | None = None
+) -> Exception | tuple[UUID, str, str, str, str]:
     try:
         result = call_with_fallback(
             _sign_in_hub, email=email, password=password, handle=handle

lamindb_setup/core/_hub_crud.py

@@ -1,20 +1,23 @@
-from supabase.client import Client
-from typing import Optional, Dict
-from lamin_utils import logger
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 from uuid import UUID, uuid4
 
+from lamin_utils import logger
+from supabase.client import Client  # noqa
+
 
 def select_instance_by_owner_name(
     owner: str,
     name: str,
     client: Client,
-) -> Optional[Dict]:
+) -> dict | None:
     try:
         data = (
             client.table("instance")
             .select(
                 "*, account!inner!instance_account_id_28936e8f_fk_account_id(*),"
-                " storage(*)"
+                " storage!instance_storage_id_87963cc8_fk_storage_id(*)"
             )
             .eq("account.handle", owner)
             .eq("name", name)
@@ -82,7 +85,10 @@ def select_instance_by_id_with_storage(
     client: Client,
 ):
     response = (
-        client.table("instance").select("*, storage(*)").eq("id", instance_id).execute()
+        client.table("instance")
+        .select("*, storage!instance_storage_id_87963cc8_fk_storage_id(*)")
+        .eq("id", instance_id)
+        .execute()
     )
     if len(response.data) == 0:
         return None
@@ -178,8 +184,6 @@ def select_db_user_by_instance(instance_id: str, client: Client):
 def _delete_instance_record(instance_id: UUID, client: Client) -> None:
     if not isinstance(instance_id, UUID):
         instance_id = UUID(instance_id)
-    logger.important(f"deleting instance {instance_id.hex}")
-    client.table("instance").delete().eq("id", instance_id.hex).execute()
-
-
-sb_delete_instance = _delete_instance_record
+    response = client.table("instance").delete().eq("id", instance_id.hex).execute()
+    if response.data:
+        logger.important(f"deleted instance record on hub {instance_id.hex}")

lamindb_setup/core/_hub_utils.py

@@ -1,9 +1,12 @@
+from __future__ import annotations
+
 from typing import Optional
+
 from pydantic import BaseModel, validator
 from pydantic.networks import MultiHostDsn
 
 
-def validate_schema_arg(schema: Optional[str] = None) -> str:
+def validate_schema_arg(schema: str | None = None) -> str:
     if schema is None or schema == "":
         return ""
     # currently no actual validation, can add back if we see a need
@@ -12,7 +15,7 @@ def validate_schema_arg(schema: Optional[str] = None) -> str:
     return ",".join(to_be_validated)
 
 
-def validate_db_arg(db: Optional[str]) -> None:
+def validate_db_arg(db: str | None) -> None:
     if db is not None:
         LaminDsnModel(db=db)
 
@@ -43,13 +46,13 @@ class LaminDsn(MultiHostDsn):
         cls,
         *,
         scheme: str,
-        user: Optional[str] = None,
-        password: Optional[str] = None,
+        user: str | None = None,
+        password: str | None = None,
         host: str,
-        port: Optional[str] = None,
-        database: Optional[str] = None,
-        query: Optional[str] = None,
-        fragment: Optional[str] = None,
+        port: str | None = None,
+        database: str | None = None,
+        query: str | None = None,
+        fragment: str | None = None,
         **_kwargs: str,
     ) -> str:
         return super().build(

lamindb_setup/core/_settings.py

@@ -1,30 +1,30 @@
+from __future__ import annotations
+
 import os
-from typing import Union, Optional
-from lamindb_setup.core import InstanceSettings, StorageSettings, UserSettings
-from lamindb_setup.core._settings_load import (
+from typing import TYPE_CHECKING
+
+from ._settings_load import (
     load_instance_settings,
     load_or_create_user_settings,
 )
-from ._settings_store import current_instance_settings_file
-from pathlib import Path
-from ._settings_store import settings_dir
+from ._settings_store import current_instance_settings_file, settings_dir
 
+if TYPE_CHECKING:
+    from pathlib import Path
 
-class SetupSettings:
-    """Setup settings.
+    from lamindb_setup.core import InstanceSettings, StorageSettings, UserSettings
 
-    - :class:`~lamindb_setup.core.InstanceSettings`
-    - :class:`~lamindb_setup.core.StorageSettings`
-    - :class:`~lamindb_setup.core.UserSettings`
-    """
 
-    _using_key: Optional[str] = None  # set through lamindb.settings
+class SetupSettings:
+    """Setup settings."""
 
-    _user_settings: Union[UserSettings, None] = None
-    _instance_settings: Union[InstanceSettings, None] = None
+    _using_key: str | None = None  # set through lamindb.settings
 
-    _user_settings_env: Union[str, None] = None
-    _instance_settings_env: Union[str, None] = None
+    _user_settings: UserSettings | None = None
+    _instance_settings: InstanceSettings | None = None
+
+    _user_settings_env: str | None = None
+    _instance_settings_env: str | None = None
 
     _auto_connect_path: Path = settings_dir / "auto_connect"
 
@@ -50,11 +50,8 @@ class SetupSettings:
 
     @property
     def user(self) -> UserSettings:
-        """User."""
-        if (
-            self._user_settings is None
-            or self._user_settings_env != get_env_name()  # noqa
-        ):
+        """:class:`~lamindb.setup.core.UserSettings`."""
+        if self._user_settings is None or self._user_settings_env != get_env_name():
             self._user_settings = load_or_create_user_settings()
             self._user_settings_env = get_env_name()
             if self._user_settings and self._user_settings.uid is None:
@@ -63,10 +60,10 @@ class SetupSettings:
 
     @property
     def instance(self) -> InstanceSettings:
-        """Instance."""
+        """:class:`~lamindb.setup.core.InstanceSettings`."""
         if (
             self._instance_settings is None
-            or self._instance_settings_env != get_env_name()  # noqa
+            or self._instance_settings_env != get_env_name()
         ):
             self._instance_settings = load_instance_settings()
             self._instance_settings_env = get_env_name()
@@ -74,13 +71,13 @@ class SetupSettings:
 
     @property
     def storage(self) -> StorageSettings:
-        """Storage."""
+        """:class:`~lamindb.setup.core.StorageSettings`."""
         return self.instance.storage
 
     @property
     def _instance_exists(self):
         try:
-            self.instance
+            self.instance  # noqa
             return True
         # this is implicit logic that catches if no instance is loaded
         except SystemExit: