lamindb_setup 0.70.0__py2.py3-none-any.whl → 0.71.1__py2.py3-none-any.whl

lamindb_setup/core/_settings_instance.py

@@ -2,27 +2,25 @@ from __future__ import annotations
 
 import os
 import shutil
-from .upath import LocalPathClasses, convert_pathlike
 from pathlib import Path
-from typing import Literal, Optional, Set, Tuple
-from uuid import UUID
+from typing import TYPE_CHECKING, Literal
+
+from lamin_utils import logger
+
 from ._hub_client import call_with_fallback
 from ._hub_crud import select_account_handle_name_by_lnid
-from lamin_utils import logger
+from ._hub_utils import LaminDsn, LaminDsnModel
+from ._settings_save import save_instance_settings
+from ._settings_storage import StorageSettings, init_storage, mark_storage_root
+from ._settings_store import current_instance_settings_file, instance_settings_file
 from .cloud_sqlite_locker import (
-    InstanceLockedException,
     EXPIRATION_TIME,
+    InstanceLockedException,
 )
-from ._hub_utils import LaminDsnModel, LaminDsn
-from ._settings_save import save_instance_settings
-from ._settings_storage import StorageSettings
-from ._settings_store import current_instance_settings_file, instance_settings_file
-from .upath import UPath
+from .upath import LocalPathClasses, UPath, convert_pathlike
 
-LOCAL_STORAGE_ROOT_WARNING = (
-    "No local storage root found, set via `ln.setup.settings.instance.local_storage ="
-    " local_root`"
-)
+if TYPE_CHECKING:
+    from uuid import UUID
 
 
 def sanitize_git_repo_url(repo_url: str) -> str:
@@ -39,26 +37,28 @@ class InstanceSettings:
         owner: str,  # owner handle
         name: str,  # instance name
         storage: StorageSettings,  # storage location
-        local_storage: bool = False,  # default to local storage
-        uid: Optional[str] = None,  # instance uid/lnid
-        db: Optional[str] = None,  # DB URI
-        schema: Optional[str] = None,  # comma-separated string of schema names
-        git_repo: Optional[str] = None,  # a git repo URL
+        keep_artifacts_local: bool = False,  # default to local storage
+        uid: str | None = None,  # instance uid/lnid
+        db: str | None = None,  # DB URI
+        schema: str | None = None,  # comma-separated string of schema names
+        git_repo: str | None = None,  # a git repo URL
+        is_on_hub: bool = False,  # initialized from hub
     ):
         from ._hub_utils import validate_db_arg
 
-        self._id: UUID = id
+        self._id_: UUID = id
         self._owner: str = owner
         self._name: str = name
-        self._uid: Optional[str] = uid
+        self._uid: str | None = uid
         self._storage: StorageSettings = storage
         validate_db_arg(db)
-        self._db: Optional[str] = db
-        self._schema_str: Optional[str] = schema
+        self._db: str | None = db
+        self._schema_str: str | None = schema
         self._git_repo = None if git_repo is None else sanitize_git_repo_url(git_repo)
         # local storage
-        self._local_storage_on = local_storage
-        self._local_storage = None
+        self._keep_artifacts_local = keep_artifacts_local
+        self._storage_local: StorageSettings | None = None
+        self._is_on_hub = is_on_hub
 
     def __repr__(self):
         """Rich string representation."""
@@ -97,60 +97,111 @@ class InstanceSettings:
         """Instance name."""
         return self._name
 
-    def _search_local_root(self):
+    def _search_local_root(
+        self, local_root: str | None = None, mute_warning: bool = False
+    ) -> StorageSettings | None:
         from lnschema_core.models import Storage
 
-        records = Storage.objects.filter(type="local").all()
-        for record in records:
-            if Path(record.root).exists():
-                self._local_storage = StorageSettings(record.root)
-                logger.important(f"defaulting to local storage: {record}")
-                break
-        if self._local_storage is None:
-            logger.warning(LOCAL_STORAGE_ROOT_WARNING)
+        if local_root is not None:
+            local_records = Storage.objects.filter(root=local_root)
+        else:
+            local_records = Storage.objects.filter(type="local")
+        found = False
+        for record in local_records.all():
+            root_path = Path(record.root)
+            if root_path.exists():
+                marker_path = root_path / ".lamindb/_is_initialized"
+                if marker_path.exists():
+                    uid = marker_path.read_text()
+                    if uid == record.uid:
+                        found = True
+                        break
+                    elif uid == "":
+                        # legacy instance that was not yet marked properly
+                        mark_storage_root(record.root, record.uid)
+                    else:
+                        continue
+                else:
+                    # legacy instance that was not yet marked at all
+                    mark_storage_root(record.root, record.uid)
+                    break
+        if found:
+            return StorageSettings(record.root)
+        elif not mute_warning:
+            logger.warning(
+                f"none of the registered local storage locations were found in your environment: {local_records}"
+                "\n❗ please register a new local storage location via `ln.settings.storage_local = local_root_path` "
+                "and re-load/connect the instance"
+            )
+        return None
 
     @property
-    def local_storage(self) -> StorageSettings:
-        """Default local storage.
+    def keep_artifacts_local(self) -> bool:
+        """Default to keeping artifacts local.
 
-        Warning: Only enable if you're sure you want to use the more complicated
-        storage mode across local & cloud locations.
+        Enable this optional setting for cloud instances on lamin.ai.
 
-        As an admin, enable via: `ln.setup.settings.instance.local_storage =
-        local_root`.
+        Guide: :doc:`faq/keep-artifacts-local`
+        """
+        return self._keep_artifacts_local
 
-        If enabled, you'll save artifacts to a default local storage
-        location at :attr:`~lamindb.setup.settings.InstanceSettings.local_storage`.
+    @property
+    def storage(self) -> StorageSettings:
+        """Default storage.
 
-        Upon passing `upload=True` in `artifact.save(upload=True)`, you upload the
-        artifact to the default cloud storage location:
-        :attr:`~lamindb.setup.core.InstanceSettings.storage`.
+        For a cloud instance, this is cloud storage. For a local instance, this
+        is a local directory.
         """
-        if not self._local_storage_on:
-            raise ValueError("Local storage is not enabled for this instance.")
-        if self._local_storage is None:
-            self._search_local_root()
-        if self._local_storage is None:
-            raise ValueError(LOCAL_STORAGE_ROOT_WARNING)
-        return self._local_storage
-
-    @local_storage.setter
-    def local_storage(self, local_root: Path):
-        from ._hub_core import update_instance_record
-        from .._init_instance import register_storage
-
-        self._search_local_root()
-        if self._local_storage is not None:
-            raise ValueError(
-                "You already configured a local storage root for this instance in this"
-                f" environment: {self.local_storage.root}"
-            )
+        return self._storage
+
+    @property
+    def storage_local(self) -> StorageSettings:
+        """An additional local default storage.
+
+        Is only available if :attr:`keep_artifacts_local` is enabled.
+
+        Guide: :doc:`faq/keep-artifacts-local`
+        """
+        if not self._keep_artifacts_local:
+            raise ValueError("`keep_artifacts_local` is not enabled for this instance.")
+        if self._storage_local is None:
+            self._storage_local = self._search_local_root()
+        if self._storage_local is None:
+            # raise an error, there was a warning just before in search_local_root
+            raise ValueError()
+        return self._storage_local
+
+    @storage_local.setter
+    def storage_local(self, local_root: Path | str):
+        from lamindb_setup._init_instance import register_storage_in_instance
+
+        local_root = Path(local_root)
+        if not self._keep_artifacts_local:
+            raise ValueError("`keep_artifacts_local` is not enabled for this instance.")
+        storage_local = self._search_local_root(
+            local_root=StorageSettings(local_root).root_as_str, mute_warning=True
+        )
+        if storage_local is not None:
+            # great, we're merely switching storage location
+            self._storage_local = storage_local
+            logger.important(f"defaulting to local storage: {storage_local.root}")
+            return None
+        storage_local = self._search_local_root(mute_warning=True)
+        if storage_local is not None:
+            if os.getenv("LAMIN_TESTING") == "true":
+                response = "y"
+            else:
+                response = input(
+                    "You already configured a local storage root for this instance in this"
+                    f" environment: {self.storage_local.root}\nDo you want to register another one? (y/n)"
+                )
+            if response != "y":
+                return None
         local_root = convert_pathlike(local_root)
         assert isinstance(local_root, LocalPathClasses)
-        self._local_storage = StorageSettings(local_root)  # type: ignore
-        register_storage(self._local_storage)  # type: ignore
-        self._local_storage_on = True
-        update_instance_record(self.id, {"storage_mode": "hybrid"})
+        self._storage_local = init_storage(local_root, self._id, register_hub=True)  # type: ignore
+        register_storage_in_instance(self._storage_local)  # type: ignore
+        logger.important(f"defaulting to local storage: {self._storage_local.root}")
 
     @property
     def slug(self) -> str:
@@ -158,7 +209,7 @@ class InstanceSettings:
         return f"{self.owner}/{self.name}"
 
     @property
-    def git_repo(self) -> Optional[str]:
+    def git_repo(self) -> str | None:
         """Sync transforms with scripts in git repository.
 
         Provide the full git repo URL.
@@ -166,17 +217,19 @@ class InstanceSettings:
         return self._git_repo
 
     @property
-    def id(self) -> UUID:
+    def _id(self) -> UUID:
         """The internal instance id."""
-        return self._id
+        return self._id_
 
     @property
-    def uid(self) -> Optional[str]:
+    def uid(self) -> str:
         """The user-facing instance id."""
-        return self._uid
+        from .hashing import hash_and_encode_as_b62
+
+        return hash_and_encode_as_b62(self._id.hex)[:12]
 
     @property
-    def schema(self) -> Set[str]:
+    def schema(self) -> set[str]:
         """Schema modules in addition to core schema."""
         if self._schema_str is None:
             return {}  # type: ignore
@@ -186,7 +239,7 @@ class InstanceSettings:
     @property
     def _sqlite_file(self) -> UPath:
         """SQLite file."""
-        return self.storage.key_to_filepath(f"{self.id.hex}.lndb")
+        return self.storage.key_to_filepath(f"{self._id.hex}.lndb")
 
     @property
     def _sqlite_file_local(self) -> Path:
@@ -290,11 +343,6 @@ class InstanceSettings:
         else:
             return empty_locker
 
-    @property
-    def storage(self) -> StorageSettings:
-        """Low-level access to storage location."""
-        return self._storage
-
     @property
     def is_remote(self) -> bool:
         """Boolean indicating if an instance has no local component."""
@@ -317,6 +365,25 @@ class InstanceSettings:
         # and remote postgres
         return True
 
+    @property
+    def is_on_hub(self) -> bool:
+        """Is this instance on the hub.
+
+        Only works if user has access to the instance.
+        """
+        if self._is_on_hub is None:
+            from ._hub_client import call_with_fallback_auth
+            from ._hub_crud import select_instance_by_id
+
+            response = call_with_fallback_auth(
+                select_instance_by_id, instance_id=self._id.hex
+            )
+            if response is None:
+                self._is_on_hub = False
+            else:
+                self._is_on_hub = True
+        return self._is_on_hub
+
     def _get_settings_file(self) -> Path:
         return instance_settings_file(self.name, self.owner)
 
@@ -341,7 +408,7 @@ class InstanceSettings:
 
     def _load_db(
         self, do_not_lock_for_laminapp_admin: bool = False
-    ) -> Tuple[bool, str]:
+    ) -> tuple[bool, str]:
         # Is the database available and initialized as LaminDB?
         # returns a tuple of status code and message
         if self.dialect == "sqlite" and not self._sqlite_file.exists():
@@ -353,6 +420,7 @@ class InstanceSettings:
                 )
             return False, f"SQLite file {self._sqlite_file} does not exist"
         from lamindb_setup import settings  # to check user
+
         from .django import setup_django
 
         # lock in all cases except if do_not_lock_for_laminapp_admin is True and

lamindb_setup/core/_settings_load.py

@@ -1,6 +1,8 @@
-from pathlib import Path
-from typing import Optional
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Optional
 from uuid import UUID, uuid4
+
 from lamin_utils import logger
 from pydantic.error_wrappers import ValidationError
 
@@ -14,12 +16,15 @@ from ._settings_store import (
 )
 from ._settings_user import UserSettings
 
+if TYPE_CHECKING:
+    from pathlib import Path
+
 
 class SettingsEnvFileOutdated(Exception):
     pass
 
 
-def load_instance_settings(instance_settings_file: Optional[Path] = None):
+def load_instance_settings(instance_settings_file: Path | None = None):
     if instance_settings_file is None:
         instance_settings_file = current_instance_settings_file()
     if not instance_settings_file.exists():
@@ -33,10 +38,10 @@ def load_instance_settings(instance_settings_file: Optional[Path] = None):
             f"\n\n{error}\n\nYour instance settings file with\n\n{content}\nis invalid"
             f" (likely outdated), please delete {instance_settings_file} &"
             " re-initialize (local) or re-connect to the instance (remote)"
-        )
+        ) from error
     if settings_store.id == "null":
         raise ValueError(
-            "Your instance.id is undefined, please either load your instance from the"
+            "Your instance._id is undefined, please either load your instance from the"
             f" hub or update {instance_settings_file} with a new id: {uuid4().hex}"
         )
     isettings = setup_instance_from_store(settings_store)
@@ -60,13 +65,13 @@ def load_or_create_user_settings() -> UserSettings:
 def load_user_settings(user_settings_file: Path):
     try:
         settings_store = UserSettingsStore(_env_file=user_settings_file)
-    except (ValidationError, TypeError):
+    except (ValidationError, TypeError) as error:
         msg = (
             "Your user settings file is invalid, please delete"
             f" {user_settings_file} and log in again."
         )
         print(msg)
-        raise SettingsEnvFileOutdated(msg)
+        raise SettingsEnvFileOutdated(msg) from error
     settings = setup_user_from_store(settings_store)
     return settings
 
@@ -84,6 +89,7 @@ def setup_instance_from_store(store: InstanceSettingsStore) -> InstanceSettings:
         db=store.db if store.db != "null" else None,  # type: ignore
         schema=store.schema_str if store.schema_str != "null" else None,
         git_repo=store.git_repo if store.git_repo != "null" else None,
+        keep_artifacts_local=store.keep_artifacts_local,  # type: ignore
     )
 
 

lamindb_setup/core/_settings_save.py

@@ -1,17 +1,20 @@
-from pathlib import Path
+from __future__ import annotations
 
-from typing import Any, Dict, Union, get_type_hints, Optional
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Optional, get_type_hints
 from uuid import UUID
 
 from ._settings_store import (
-    UserSettingsStore,
     InstanceSettingsStore,
+    UserSettingsStore,
     current_user_settings_file,
     user_settings_file_email,
     user_settings_file_handle,
 )
-from ._settings_user import UserSettings
-from .upath import UPath
+
+if TYPE_CHECKING:
+    from ._settings_user import UserSettings
+    from .upath import UPath
 
 
 def save_user_settings(settings: UserSettings):
@@ -31,20 +34,22 @@ def save_user_settings(settings: UserSettings):
 def save_settings(
     settings: Any,
     settings_file: Path,
-    type_hints: Dict[str, Any],
+    type_hints: dict[str, Any],
     prefix: str,
 ):
     with open(settings_file, "w") as f:
         for store_key, type in type_hints.items():
             if type == Optional[str]:
                 type = str
+            if type == Optional[bool]:
+                type = bool
             if "__" not in store_key:
                 if store_key == "storage_root":
                     value = settings.storage.root_as_str
                 elif store_key == "storage_region":
                     value = settings.storage.region
                 else:
-                    if store_key in {"db", "schema_str", "name_", "uuid"}:
+                    if store_key in {"db", "schema_str", "name_", "uuid", "id"}:
                         settings_key = f"_{store_key.rstrip('_')}"
                     else:
                         settings_key = store_key
@@ -65,7 +70,7 @@ def save_instance_settings(settings: Any, settings_file: Path):
 
 
 def save_system_storage_settings(
-    cache_path: Union[str, Path, UPath, None], settings_file: Path
+    cache_path: str | Path | UPath | None, settings_file: Path
 ):
     cache_path = "null" if cache_path is None else cache_path
     if isinstance(cache_path, Path):  # also True for UPath