lamindb_setup 1.6.0__py3-none-any.whl → 1.7.0__py3-none-any.whl

lamindb_setup/core/_settings_storage.py

@@ -2,26 +2,27 @@ from __future__ import annotations
 
 import os
 import secrets
-import shutil
 import string
-from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal
+from uuid import UUID
 
 import fsspec
 from lamin_utils import logger
 
+from lamindb_setup.errors import StorageAlreadyManaged
+
 from ._aws_options import (
     HOSTED_REGIONS,
     LAMIN_ENDPOINTS,
     get_aws_options_manager,
 )
 from ._aws_storage import find_closest_aws_region
+from ._deprecated import deprecated
+from .hashing import hash_and_encode_as_b62
 from .upath import LocalPathClasses, UPath, _split_path_query, create_path
 
 if TYPE_CHECKING:
-    from uuid import UUID
-
-    from .types import UPathStr
+    from lamindb_setup.types import StorageType, UPathStr
 
 STORAGE_UID_FILE_KEY = ".lamindb/storage_uid.txt"
 LEGACY_STORAGE_UID_FILE_KEY = ".lamindb/_is_initialized"
@@ -38,6 +39,10 @@ def base62(n_char: int) -> str:
     return id
 
 
+def instance_uid_from_uuid(instance_id: UUID) -> str:
+    return hash_and_encode_as_b62(instance_id.hex)[:12]
+
+
 def get_storage_region(path: UPathStr) -> str | None:
     path_str = str(path)
     if path_str.startswith("s3://"):
@@ -82,7 +87,9 @@ def get_storage_region(path: UPathStr) -> str | None:
     return region
 
 
-def mark_storage_root(root: UPathStr, uid: str):
+def mark_storage_root(
+    root: UPathStr, uid: str, instance_id: UUID, instance_slug: str
+) -> Literal["__marked__"] | str:
     # we need a file in folder-like storage locations on S3 to avoid
     # permission errors from leveraging s3fs on an empty hosted storage location
     # (path.fs.find raises a PermissionError)
@@ -91,13 +98,28 @@ def mark_storage_root(root: UPathStr, uid: str):
     # path on a storage location in the registry
 
     root_upath = UPath(root)
+    existing_uid = ""
+    legacy_mark_upath = root_upath / LEGACY_STORAGE_UID_FILE_KEY
     mark_upath = root_upath / STORAGE_UID_FILE_KEY
-    mark_upath.write_text(uid)
+    if legacy_mark_upath.exists():
+        legacy_mark_upath.rename(mark_upath)
+    if mark_upath.exists():
+        existing_uid = mark_upath.read_text().splitlines()[0]
+    if existing_uid == "":
+        instance_uid = instance_uid_from_uuid(instance_id)
+        text = f"{uid}\ncreation info:\ninstance_slug={instance_slug}\ninstance_id={instance_id.hex}\ninstance_uid={instance_uid}"
+        mark_upath.write_text(text)
+    elif existing_uid != uid:
+        return uid
+    # covers the case in which existing uid is the same as uid
+    # and the case in which there was no existing uid
+    return "__is_marked__"
 
 
 def init_storage(
     root: UPathStr,
-    instance_id: UUID | None = None,
+    instance_id: UUID,
+    instance_slug: str,
     register_hub: bool | None = None,
     prevent_register_hub: bool = False,
     init_instance: bool = False,
@@ -107,6 +129,8 @@ def init_storage(
     StorageSettings,
     Literal["hub-record-not-created", "hub-record-retrieved", "hub-record-created"],
 ]:
+    from ._hub_core import delete_storage_record, init_storage_hub
+
     assert root is not None, "`root` argument can't be `None`"
 
     root_str = str(root)  # ensure we have a string
@@ -148,53 +172,61 @@ def init_storage(
         instance_id=instance_id,
         access_token=access_token,
     )
-    # this stores the result of init_storage_hub
-    hub_record_status: Literal[
-        "hub-record-not-created", "hub-record-retrieved", "hub-record-created"
-    ] = "hub-record-not-created"
-    # the below might update the uid with one that's already taken on the hub
-    if not prevent_register_hub and (ssettings.type_is_cloud or register_hub):
-        from ._hub_core import delete_storage_record, init_storage_hub
-
-        # this retrieves the storage record if it exists already in the hub
-        # and updates uid and instance_id in ssettings
-        hub_record_status = init_storage_hub(
-            ssettings,
-            auto_populate_instance=not init_instance,
-            created_by=created_by,
-            access_token=access_token,
-        )
-    # below comes last only if everything else was successful
-    # we check the write access here only if the storage record has been just created
-    # or if the storage is local
-    # also we have to check hub_record_status here because
-    # _select_storage inside init_storage_hub also populates ssettings._uuid
-    # and we don't want to delete an existing storage record here if no write access
-    # only newly created
-    # local storages not registered in the hub should be also marked
-    is_local_not_retrieved = not (
-        ssettings.type_is_cloud or hub_record_status == "hub-record-retrieved"
+    # this retrieves the storage record if it exists already in the hub
+    # and updates uid and instance_id in ssettings
+    register_hub = (
+        register_hub or ssettings.type_is_cloud
+    )  # default to registering cloud storage
+    hub_record_status = init_storage_hub(
+        ssettings,
+        auto_populate_instance=not init_instance,
+        created_by=created_by,
+        access_token=access_token,
+        prevent_creation=prevent_register_hub or not register_hub,
     )
-    if hub_record_status == "hub-record-created" or is_local_not_retrieved:
+    # we check the write access here if the storage record has not been retrieved from the hub
+    if hub_record_status != "hub-record-retrieved":
         try:
             # (federated) credentials for AWS access are provisioned under-the-hood
             # discussion: https://laminlabs.slack.com/archives/C04FPE8V01W/p1719260587167489
             # if access_token was passed in ssettings, it is used here
-            mark_storage_root(ssettings.root, ssettings.uid)  # type: ignore
-        except Exception:
-            logger.important(
-                f"due to lack of write access, LaminDB won't manage storage location: {ssettings.root_as_str}"
+            marking_result = mark_storage_root(
+                root=ssettings.root,
+                uid=ssettings.uid,
+                instance_id=instance_id,
+                instance_slug=instance_slug,
             )
+        except Exception:
+            marking_result = "no-write-access"
+        if marking_result != "__is_marked__":
+            if marking_result == "no-write-access":
+                logger.important(
+                    f"due to lack of write access, LaminDB won't manage this storage location: {ssettings.root_as_str}"
+                )
+                ssettings._instance_id = None  # indicate that this storage location is not managed by the instance
+            else:
+                s = "S" if init_instance else "s"  # upper case for error message
+                message = (
+                    f"{s}torage location {ssettings.root_as_str} is already marked with uid {marking_result}, meaning that it is managed by another LaminDB instance -- "
+                    "if you manage your instance with LaminHub you get an overview of all your storage locations"
+                )
+                if init_instance:
+                    raise StorageAlreadyManaged(message)
+                logger.warning(message)
+                ssettings._instance_id = UUID(
+                    "00000000000000000000000000000000"
+                )  # indicate not known
+                ssettings._uid = marking_result
+            # this condition means that the hub record was created
             if ssettings._uuid is not None:
-                delete_storage_record(ssettings._uuid, access_token=access_token)  # type: ignore
+                delete_storage_record(ssettings, access_token=access_token)  # type: ignore
                 ssettings._uuid_ = None
                 hub_record_status = "hub-record-not-created"
-            ssettings._instance_id = None
     return ssettings, hub_record_status
 
 
 class StorageSettings:
-    """Settings for a given storage location (local or cloud)."""
+    """Settings for a storage location (local or cloud)."""
 
     def __init__(
         self,
@@ -208,13 +240,15 @@ class StorageSettings:
     ):
         self._uid = uid
         self._uuid_ = uuid
-        self._root_init = UPath(root)
+        self._root_init = UPath(root).expanduser()
         if isinstance(self._root_init, LocalPathClasses):  # local paths
             try:
                 (self._root_init / ".lamindb").mkdir(parents=True, exist_ok=True)
                 self._root_init = self._root_init.resolve()
             except Exception:
-                logger.warning(f"unable to create .lamindb folder in {self._root_init}")
+                logger.warning(
+                    f"unable to create .lamindb/ folder in {self._root_init}"
+                )
         self._root = None
         self._instance_id = instance_id
         # we don't yet infer region here to make init fast
@@ -229,8 +263,16 @@ class StorageSettings:
         self._local = None
 
     @property
+    @deprecated("_id")
     def id(self) -> int:
-        """Storage id in current instance."""
+        return self._id
+
+    @property
+    def _id(self) -> int:
+        """Storage id.
+
+        This id is only valid in the current instance and not globally unique. Only for internal use.
+        """
         return self.record.id
 
     @property
@@ -239,12 +281,27 @@ class StorageSettings:
         return self._uuid_
 
     @property
-    def uid(self) -> str | None:
-        """Storage id."""
+    def uid(self) -> str:
+        """Storage uid."""
         if self._uid is None:
             self._uid = self.record.uid
         return self._uid
 
+    @property
+    def instance_uid(self) -> str | None:
+        """The `uid` of the managing LaminDB instance.
+
+        If `None`, the storage location is not managed by any LaminDB instance.
+        """
+        if self._instance_id is not None:
+            if self._instance_id.hex == "00000000000000000000000000000000":
+                instance_uid = "__unknown__"
+            else:
+                instance_uid = instance_uid_from_uuid(self._instance_id)
+        else:
+            instance_uid = None
+        return instance_uid
+
     @property
     def _mark_storage_root(self) -> UPath:
         marker_path = self.root / STORAGE_UID_FILE_KEY
@@ -336,10 +393,10 @@ class StorageSettings:
         return self._region
 
     @property
-    def type(self) -> Literal["local", "s3", "gs", "hf", "http", "https"]:
+    def type(self) -> StorageType:
         """AWS S3 vs. Google Cloud vs. local.
 
-        Returns the protocol as a string: "local", "s3", "gs", "http", "https".
+        Returns the protocol as a stringe, e.g., "local", "s3", "gs", "http", "https".
         """
         import fsspec
 

lamindb_setup/core/_settings_store.py

@@ -3,13 +3,14 @@ from pathlib import Path
 from typing import Optional
 
 from lamin_utils import logger
+from platformdirs import site_config_dir
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 if "LAMIN_SETTINGS_DIR" in os.environ:
     # Needed when running with AWS Lambda, as only tmp/ directory has a write access
     settings_dir = Path(f"{os.environ['LAMIN_SETTINGS_DIR']}/.lamin")
 else:
-    # user_config_dir in appdirs is weird on MacOS!
+    # user_config_dir is weird on MacOS!
     # hence, let's take home/.lamin
     settings_dir = Path.home() / ".lamin"
 
@@ -20,6 +21,9 @@ except Exception as e:
     logger.warning(f"Failed to create lamin settings directory at {settings_dir}: {e}")
 
 
+system_settings_dir = Path(site_config_dir(appname="lamindb", appauthor="laminlabs"))
+
+
 def get_settings_file_name_prefix():
     if "LAMIN_ENV" in os.environ:
         if os.environ["LAMIN_ENV"] != "prod":
@@ -49,10 +53,15 @@ def user_settings_file_handle(handle: str):
     return settings_dir / f"{get_settings_file_name_prefix()}user--{handle}.env"
 
 
-def system_storage_settings_file():
+# here user means the user directory on os, not a lamindb user
+def platform_user_storage_settings_file():
     return settings_dir / "storage.env"
 
 
+def system_settings_file():
+    return system_settings_dir / "system.env"
+
+
 class InstanceSettingsStore(BaseSettings):
     api_url: Optional[str] = None
     owner: str

lamindb_setup/core/cloud_sqlite_locker.py

@@ -6,26 +6,27 @@ from typing import TYPE_CHECKING
 
 from lamin_utils import logger
 
+from lamindb_setup.errors import InstanceLockedException
+
 from .upath import UPath, create_mapper, infer_filesystem
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
     from pathlib import Path
+    from typing import ParamSpec, TypeVar
     from uuid import UUID
 
     from ._settings_instance import InstanceSettings
     from ._settings_user import UserSettings
 
+    P = ParamSpec("P")
+    R = TypeVar("R")
+
 EXPIRATION_TIME = 24 * 60 * 60 * 7  # 7 days
 
 MAX_MSG_COUNTER = 100  # print the msg after this number of iterations
 
 
-# raise if an instance is already locked
-# ignored by unlock_cloud_sqlite_upon_exception
-class InstanceLockedException(Exception):
-    pass
-
-
 class empty_locker:
     has_lock = True
 
@@ -207,7 +208,9 @@ def clear_locker():
 
 
 # decorator
-def unlock_cloud_sqlite_upon_exception(ignore_prev_locker: bool = False):
+def unlock_cloud_sqlite_upon_exception(
+    ignore_prev_locker: bool = False,
+) -> Callable[[Callable[P, R]], Callable[P, R]]:
     """Decorator to unlock a cloud sqlite instance upon an exception.
 
     Ignores `InstanceLockedException`.

lamindb_setup/core/django.py

@@ -11,6 +11,8 @@ from pathlib import Path
 import time
 from ._settings_instance import InstanceSettings
 
+from lamin_utils import logger
+
 
 IS_RUN_FROM_IPYTHON = getattr(builtins, "__IPYTHON__", False)
 IS_SETUP = False
@@ -114,10 +116,9 @@ class DBTokenManager:
                     connection.connection.cursor().execute(token.token_query)
 
         Atomic.__enter__ = __enter__
+        logger.debug("django.db.transaction.Atomic.__enter__ has been patched")
 
     def reset(self, connection_name: str = "default"):
-        from django.db.transaction import Atomic
-
         connection = self.get_connection(connection_name)
 
         connection.execute_wrappers = [
@@ -150,6 +151,7 @@ def setup_django(
 ):
     if IS_RUN_FROM_IPYTHON:
         os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
+        logger.debug("DJANGO_ALLOW_ASYNC_UNSAFE env variable has been set to 'true'")
 
     import dj_database_url
     import django
@@ -216,6 +218,9 @@ def setup_django(
         from django.db.backends.base.base import BaseDatabaseWrapper
 
         BaseDatabaseWrapper.close_if_health_check_failed = close_if_health_check_failed
+        logger.debug(
+            "django.db.backends.base.base.BaseDatabaseWrapper.close_if_health_check_failed has been patched"
+        )
 
         if isettings._fine_grained_access and isettings._db_permissions == "jwt":
             db_token = DBToken(isettings)

lamindb_setup/core/exceptions.py

@@ -1,10 +1 @@
-from __future__ import annotations
-
-
-class DefaultMessageException(Exception):
-    default_message: str | None = None
-
-    def __init__(self, message: str | None = None):
-        if message is None:
-            message = self.default_message
-        super().__init__(message)
+from lamindb_setup.errors import DefaultMessageException  # backwards compatibility

lamindb_setup/core/hashing.py

@@ -23,7 +23,7 @@ HASH_LENGTH = 22
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
-    from .types import Path, UPathStr
+    from lamindb_setup.types import Path, UPathStr
 
 
 def hash_and_encode_as_b62(s: str) -> str:

lamindb_setup/core/types.py

@@ -1,17 +1 @@
-from __future__ import annotations
-
-"""Types.
-
-.. autosummary::
-   :toctree: .
-
-   UPathStr
-"""
-# we need Union here because __future__ annotations doesn't work with TypeAlias
-from pathlib import Path
-from typing import Union
-
-# UPath is subclass of Path, hence, it's not necessary to list UPath
-# we keep it in the name of the TypeAlias to make it clear to users that
-# cloud paths are allowed / PathStr is often associated with local paths
-UPathStr = Union[str, Path]  # typing.TypeAlias, >3.10 on but already deprecated
+from lamindb_setup.types import UPathStr  # backward compatibility

lamindb_setup/core/upath.py

@@ -13,7 +13,6 @@ from pathlib import Path, PosixPath, PurePosixPath, WindowsPath
 from typing import TYPE_CHECKING, Any, Literal
 from urllib.parse import parse_qs, urlsplit
 
-import click
 import fsspec
 from lamin_utils import logger
 from upath import UPath
@@ -21,11 +20,13 @@ from upath.implementations.cloud import CloudPath, S3Path  # keep CloudPath!
 from upath.implementations.local import LocalPath
 from upath.registry import register_implementation
 
+from lamindb_setup.errors import StorageNotEmpty
+
 from ._aws_options import HOSTED_BUCKETS, get_aws_options_manager
 from .hashing import HASH_LENGTH, b16_to_b64, hash_from_hashes_list, hash_string
 
 if TYPE_CHECKING:
-    from .types import UPathStr
+    from lamindb_setup.types import UPathStr
 
 LocalPathClasses = (PosixPath, WindowsPath, LocalPath)
 
@@ -778,6 +779,8 @@ Args:
     pathlike: A string or Path to a local/cloud file/directory/folder.
 """
 
+logger.debug("upath.UPath has been patched")
+
 # suppress the warning from upath about hf (huggingface) filesystem
 # not being explicitly implemented in upath
 warnings.filterwarnings(
@@ -821,7 +824,7 @@ register_implementation("s3", S3QueryPath, clobber=True)
 
 
 def create_path(path: UPathStr, access_token: str | None = None) -> UPath:
-    upath = UPath(path)
+    upath = UPath(path).expanduser()
 
     if upath.protocol == "s3":
         # add managed credentials and other options for AWS s3 paths
@@ -913,18 +916,21 @@ def get_stat_dir_cloud(path: UPath) -> tuple[int, str | None, str | None, int]:
     return size, hash, hash_type, n_files
 
 
-class InstanceNotEmpty(click.ClickException):
-    def show(self, file=None):
-        pass
-
-
 # is as fast as boto3: https://lamin.ai/laminlabs/lamin-site-assets/transform/krGp3hT1f78N5zKv
 def check_storage_is_empty(
     root: UPathStr, *, raise_error: bool = True, account_for_sqlite_file: bool = False
 ) -> int:
+    from ._settings_storage import STORAGE_UID_FILE_KEY
+
     root_upath = UPath(root)
     root_string = root_upath.as_posix()  # type: ignore
     n_offset_objects = 1  # because of storage_uid.txt file, see mark_storage_root()
+    # if the storage_uid.txt was somehow deleted, we restore a dummy version of it
+    # because we need it to count files in an empty directory on S3 (otherwise permission error)
+    if not (root_upath / STORAGE_UID_FILE_KEY).exists():
+        (root_upath / STORAGE_UID_FILE_KEY).write_text(
+            "was deleted, restored during delete"
+        )
     if account_for_sqlite_file:
         n_offset_objects += 1  # the SQLite file is in the ".lamindb" directory
     if root_string.startswith(HOSTED_BUCKETS):
@@ -939,17 +945,17 @@ def check_storage_is_empty(
     n_files = len(objects)
     n_diff = n_files - n_offset_objects
     ask_for_deletion = (
-        "delete them prior to deleting the instance"
+        "delete them prior to deleting the storage location"
         if raise_error
         else "consider deleting them"
     )
     message = (
-        f"Storage '{directory_string}' contains {n_files - n_offset_objects} objects"
+        f"'{directory_string}' contains {n_files - n_offset_objects} objects"
         f" - {ask_for_deletion}"
     )
     if n_diff > 0:
         if raise_error:
-            raise InstanceNotEmpty(message) from None
+            raise StorageNotEmpty(message) from None
         else:
             logger.warning(message)
     return n_diff

lamindb_setup/errors.py

@@ -0,0 +1,70 @@
+"""Errors.
+
+.. autosummary::
+   :toctree: .
+
+   InstanceNotSetupError
+   ModuleWasntConfigured
+   StorageAlreadyManaged
+   StorageNotEmpty
+   InstanceLockedException
+   SettingsEnvFileOutdated
+   CannotSwitchDefaultInstance
+
+"""
+
+from __future__ import annotations
+
+import click
+
+
+class DefaultMessageException(Exception):
+    default_message: str | None = None
+
+    def __init__(self, message: str | None = None):
+        if message is None:
+            message = self.default_message
+        super().__init__(message)
+
+
+class InstanceNotSetupError(DefaultMessageException):
+    default_message = """\
+To use lamindb, you need to connect to an instance.
+
+Connect to an instance: `ln.connect()`. Init an instance: `ln.setup.init()`.
+
+If you used the CLI to set up lamindb in a notebook, restart the Python session.
+"""
+
+
+MODULE_WASNT_CONFIGURED_MESSAGE_TEMPLATE = (
+    "'{}' wasn't configured for this instance -- "
+    "if you want it, go to your instance settings page and add it under 'schema modules' (or ask an admin to do so)"
+)
+
+
+class ModuleWasntConfigured(Exception):
+    pass
+
+
+class StorageAlreadyManaged(Exception):
+    pass
+
+
+class StorageNotEmpty(click.ClickException):
+    def show(self, file=None):
+        pass
+
+
+# raise if a cloud SQLite instance is already locked
+# ignored by unlock_cloud_sqlite_upon_exception
+class InstanceLockedException(Exception):
+    pass
+
+
+class SettingsEnvFileOutdated(Exception):
+    pass
+
+
+class CannotSwitchDefaultInstance(Exception):
+    pass

lamindb_setup/types.py

@@ -0,0 +1,20 @@
+"""Types.
+
+.. autosummary::
+   :toctree: .
+
+   UPathStr
+   StorageType
+"""
+
+from __future__ import annotations
+
+# we need Union here because __future__ annotations doesn't work with TypeAlias
+from pathlib import Path
+from typing import Literal, Union
+
+# UPath is subclass of Path, hence, it's not necessary to list UPath
+# we keep it in the name of the TypeAlias to make it clear to users that
+# cloud paths are allowed / PathStr is often associated with local paths
+UPathStr = Union[str, Path]  # typing.TypeAlias, >3.10 on but already deprecated
+StorageType = Literal["local", "s3", "gs", "hf", "http", "https"]

{lamindb_setup-1.6.0.dist-info → lamindb_setup-1.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.6.0
+Version: 1.7.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10
@@ -9,7 +9,7 @@ Requires-Dist: lamin_utils>=0.3.3
 Requires-Dist: django>=5.1,<5.2
 Requires-Dist: dj_database_url>=1.3.0,<3.0.0
 Requires-Dist: pydantic-settings
-Requires-Dist: appdirs<2.0.0
+Requires-Dist: platformdirs<5.0.0
 Requires-Dist: requests
 Requires-Dist: universal_pathlib==0.2.6
 Requires-Dist: botocore<2.0.0