lamindb_setup 0.69.5__py2.py3-none-any.whl → 0.71.0__py2.py3-none-any.whl

lamindb_setup/core/_settings_storage.py

@@ -1,19 +1,30 @@
+from __future__ import annotations
+
 import os
+import secrets
 import shutil
-from lamin_utils import logger
+import string
 from pathlib import Path
-from typing import Any, Optional, Union, Literal
-from ._aws_storage import find_closest_aws_region
+from typing import TYPE_CHECKING, Any, Literal, Optional, Union
+
 from appdirs import AppDirs
+from lamin_utils import logger
+
+from ._aws_storage import find_closest_aws_region
 from ._settings_save import save_system_storage_settings
 from ._settings_store import system_storage_settings_file
-from .upath import LocalPathClasses, UPath, create_path, convert_pathlike
-from uuid import UUID
-import string
-import secrets
-from .types import UPathStr
-from .upath import hosted_regions
+from .upath import (
+    HOSTED_REGIONS,
+    LocalPathClasses,
+    UPath,
+    convert_pathlike,
+    create_path,
+)
 
+if TYPE_CHECKING:
+    from uuid import UUID
+
+    from .types import UPathStr
 
 DIRS = AppDirs("lamindb", "laminlabs")
 IS_INITIALIZED_KEY = ".lamindb/_is_initialized"
@@ -26,7 +37,7 @@ def base62(n_char: int) -> str:
     return id
 
 
-def get_storage_region(storage_root: UPathStr) -> Optional[str]:
+def get_storage_region(storage_root: UPathStr) -> str | None:
     storage_root_str = str(storage_root)
     if storage_root_str.startswith("s3://"):
         import botocore.session as session
@@ -55,19 +66,22 @@ def get_storage_region(storage_root: UPathStr) -> Optional[str]:
     return storage_region
 
 
-def mark_storage_root(root: UPathStr):
-    # we need to touch a 0-byte object in the storage location to avoid
+def mark_storage_root(root: UPathStr, uid: str):
+    # we need to touch a 0-byte object in folder-like storage location on S3 to avoid
     # permission errors from leveraging s3fs on an empty hosted storage location
+    # for consistency, we write this file everywhere
     root_upath = convert_pathlike(root)
     mark_upath = root_upath / IS_INITIALIZED_KEY
-    mark_upath.touch()
+    mark_upath.write_text(uid)
 
 
-def init_storage(root: UPathStr) -> "StorageSettings":
+def init_storage(
+    root: UPathStr, instance_id: UUID | None = None, register_hub: bool | None = None
+) -> StorageSettings:
     if root is None:
         raise ValueError("`storage` argument can't be `None`")
     root_str = str(root)  # ensure we have a string
-    uid = base62(8)
+    uid = base62(12)
     region = None
     lamin_env = os.getenv("LAMIN_ENV")
     if root_str.startswith("create-s3"):
@@ -77,8 +91,8 @@ def init_storage(root: UPathStr) -> "StorageSettings":
         if region is None:
             region = find_closest_aws_region()
         else:
-            if region not in hosted_regions:
-                raise ValueError(f"region has to be one of {hosted_regions}")
+            if region not in HOSTED_REGIONS:
+                raise ValueError(f"region has to be one of {HOSTED_REGIONS}")
         if lamin_env is None or lamin_env == "prod":
             root_str = f"s3://lamin-{region}/{uid}"
         else:
@@ -91,19 +105,23 @@ def init_storage(root: UPathStr) -> "StorageSettings":
         except Exception as e:
             logger.error("`storage` is not a valid local, GCP storage or AWS S3 path")
             raise e
-    ssettings = StorageSettings(uid=uid, root=root_str, region=region)
-    if ssettings.is_cloud:
+    ssettings = StorageSettings(
+        uid=uid,
+        root=root_str,
+        region=region,
+        instance_id=instance_id,
+    )
+    # the below might update the uid with one that's already taken on the hub
+    if ssettings.type_is_cloud or register_hub:
         from ._hub_core import init_storage as init_storage_hub
 
-        ssettings._description = f"Created as default storage for instance {uid}"
-        ssettings._uuid = init_storage_hub(ssettings)
-        logger.important(f"registered storage: {ssettings.root_as_str}")
-    if ssettings.is_cloud and root_str.startswith("create-s3"):
-        mark_storage_root(ssettings.root)
+        init_storage_hub(ssettings)
+    # below comes last only if everything else was successful
+    mark_storage_root(ssettings.root, ssettings.uid)  # type: ignore
     return ssettings
 
 
-def _process_cache_path(cache_path: Union[str, Path, UPath, None]):
+def _process_cache_path(cache_path: str | Path | UPath | None):
     if cache_path is None or cache_path == "null":
         return None
     cache_dir = UPath(cache_path)
@@ -115,32 +133,33 @@ def _process_cache_path(cache_path: Union[str, Path, UPath, None]):
 
 
 class StorageSettings:
-    """Manage cloud or local storage settings."""
+    """Settings for a given storage location (local or cloud)."""
 
     def __init__(
         self,
         root: UPathStr,
-        region: Optional[str] = None,
-        is_hybrid: bool = False,  # refers to storage mode
-        uid: Optional[str] = None,
-        uuid: Optional[UUID] = None,
-        access_token: Optional[str] = None,
+        region: str | None = None,
+        uid: str | None = None,
+        uuid: UUID | None = None,
+        instance_id: UUID | None = None,
+        access_token: str | None = None,
     ):
         self._uid = uid
-        self._uuid = uuid
-        self._is_hybrid = is_hybrid
+        self._uuid_ = uuid
         self._root_init = convert_pathlike(root)
         if isinstance(self._root_init, LocalPathClasses):  # local paths
-            self._root_init.mkdir(parents=True, exist_ok=True)
-            self._root_init = self._root_init.resolve()
+            try:
+                (self._root_init / ".lamindb").mkdir(parents=True, exist_ok=True)
+                self._root_init = self._root_init.resolve()
+            except Exception:
+                logger.warning("unable to create .lamindb folder")
+                pass
         self._root = None
-        self._remote_root = None
-        self._aws_account_id: Optional[int] = None
-        self._description: Optional[str] = None
+        self._instance_id = instance_id
         # we don't yet infer region here to make init fast
         self._region = region
         # would prefer to type below as Registry, but need to think through import order
-        self._record: Optional[Any] = None
+        self._record: Any | None = None
         # cache settings
         self._storage_settings_file = system_storage_settings_file()
         if self._storage_settings_file.exists():
@@ -155,88 +174,61 @@ class StorageSettings:
         # save access_token here for use in self.root
         self.access_token = access_token
 
+        # local storage
+        self._has_local = False
+        self._local = None
+        self._is_on_hub: bool | None = None
+
     @property
     def id(self) -> int:
-        """Storage id."""
+        """Storage id in current instance."""
         return self.record.id
 
     @property
-    def uuid(self) -> Optional[UUID]:
-        """Storage uuid."""
-        return self._uuid
+    def _uuid(self) -> UUID | None:
+        """Lamin's internal storage uuid."""
+        return self._uuid_
 
     @property
-    def uid(self) -> Optional[str]:
+    def uid(self) -> str | None:
         """Storage id."""
         if self._uid is None:
             self._uid = self.record.uid
         return self._uid
 
+    @property
+    def _mark_storage_root(self) -> UPath:
+        return self.root / IS_INITIALIZED_KEY
+
     @property
     def record(self) -> Any:
-        """Storage record."""
+        """Storage record in current instance."""
         if self._record is None:
             # dynamic import because of import order
             from lnschema_core.models import Storage
+
             from ._settings import settings
 
-            if not self.is_hybrid:
-                self._record = Storage.objects.using(settings._using_key).get(
-                    root=self.root_as_str
-                )
-            else:
-                # this has to be redone
-                records = Storage.objects.filter(type="local").all()
-                for record in records:
-                    if Path(record.root).exists():
-                        self._record = record
-                        logger.warning("found local storage location")
-                        break
+            self._record = Storage.objects.using(settings._using_key).get(
+                root=self.root_as_str
+            )
         return self._record
 
     def __repr__(self):
         """String rep."""
         s = f"root='{self.root_as_str}', uid='{self.uid}'"
-        if self.uuid is not None:
-            s += f", uuid='{self.uuid.hex}'"
+        if self._uuid is not None:
+            s += f", uuid='{self._uuid.hex}'"
         return f"StorageSettings({s})"
 
-    @property
-    def is_hybrid(self) -> bool:
-        """Qualifies storage mode.
-
-        A storage location can be local, in the cloud, or hybrid. See
-        :attr:`~lamindb.setup.core.StorageSettings.type`.
-
-        Hybrid means that a default local storage location is backed by an
-        optional cloud storage location.
-        """
-        return self._is_hybrid
-
     @property
     def root(self) -> UPath:
         """Root storage location."""
         if self._root is None:
-            if not self.is_hybrid:
-                # below makes network requests to get credentials
-                root_path = create_path(self._root_init, access_token=self.access_token)
-            else:
-                # this is a local path
-                root_path = create_path(self.record.root)
-            self._root = root_path
+            # below makes network requests to get credentials
+            self._root = create_path(self._root_init, access_token=self.access_token)
         return self._root
 
-    @property
-    def remote_root(self) -> UPath:
-        """Remote storage location. Only needed for hybrid storage."""
-        if not self.is_hybrid:
-            raise ValueError("remote_root is only defined for hybrid storage")
-        if self._remote_root is None:
-            self._remote_root = create_path(
-                self._root_init, access_token=self.access_token
-            )
-        return self._remote_root
-
     def _set_fs_kwargs(self, **kwargs):
         """Set additional fsspec arguments for cloud root.
 
@@ -299,12 +291,12 @@ class StorageSettings:
             raise e
 
     @property
-    def is_cloud(self) -> bool:
+    def type_is_cloud(self) -> bool:
         """`True` if `storage_root` is in cloud, `False` otherwise."""
         return self.type != "local"
 
     @property
-    def region(self) -> Optional[str]:
+    def region(self) -> str | None:
         """Storage region."""
         if self._region is None:
             self._region = get_storage_region(self.root_as_str)
@@ -322,11 +314,28 @@ class StorageSettings:
         protocol = fsspec.utils.get_protocol(self.root_as_str)
         return convert.get(protocol, protocol)  # type: ignore
 
-    def key_to_filepath(self, filekey: Union[Path, UPath, str]) -> UPath:
+    @property
+    def is_on_hub(self) -> bool:
+        """Is this instance on the hub.
+
+        Only works if user has access to the instance.
+        """
+        if self._is_on_hub is None:
+            from ._hub_client import call_with_fallback_auth
+            from ._hub_crud import select_storage
+
+            response = call_with_fallback_auth(select_storage, id=self._uuid.hex)  # type: ignore
+            if response is None:
+                self._is_on_hub = False
+            else:
+                self._is_on_hub = True
+        return self._is_on_hub
+
+    def key_to_filepath(self, filekey: Path | UPath | str) -> UPath:
         """Cloud or local filepath from filekey."""
         return self.root / filekey
 
-    def cloud_to_local(self, filepath: Union[Path, UPath], **kwargs) -> UPath:
+    def cloud_to_local(self, filepath: Path | UPath, **kwargs) -> UPath:
         """Local (cache) filepath from filepath."""
         local_filepath = self.cloud_to_local_no_update(filepath)  # type: ignore
         if isinstance(filepath, UPath) and not isinstance(filepath, LocalPathClasses):
@@ -341,9 +350,9 @@ class StorageSettings:
     # using the `.parts` attribute in the following line
     def cloud_to_local_no_update(self, filepath: UPath) -> UPath:
         if isinstance(filepath, UPath) and not isinstance(filepath, LocalPathClasses):
-            return self.cache_dir.joinpath(filepath._url.netloc, *filepath.parts[1:])  # type: ignore # noqa
+            return self.cache_dir.joinpath(filepath._url.netloc, *filepath.parts[1:])  # type: ignore
         return filepath
 
-    def local_filepath(self, filekey: Union[Path, UPath, str]) -> UPath:
+    def local_filepath(self, filekey: Path | UPath | str) -> UPath:
         """Local (cache) filepath from filekey: `local(filepath(...))`."""
         return self.cloud_to_local(self.key_to_filepath(filekey))

lamindb_setup/core/_settings_store.py

@@ -1,6 +1,7 @@
 import os
 from pathlib import Path
 from typing import Optional
+
 from pydantic import BaseSettings
 
 if "LAMIN_SETTINGS_DIR" in os.environ:
@@ -51,8 +52,8 @@ class InstanceSettingsStore(BaseSettings):
     owner: str
     name: str
     storage_root: str
-    storage_region: Optional[str]
-    db: Optional[str]
+    storage_region: Optional[str]  # take old type annotations here because pydantic
+    db: Optional[str]  # doesn't like new types on 3.9 even with future annotations
     schema_str: Optional[str]
     id: str
     git_repo: Optional[str]

lamindb_setup/core/_settings_user.py

@@ -1,6 +1,10 @@
+from __future__ import annotations
+
 from dataclasses import dataclass
-from typing import Optional
-from uuid import UUID
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from uuid import UUID
 
 
 class user_description:
@@ -19,15 +23,15 @@ class UserSettings:
     """Unique handle."""
     email: str = None  # type: ignore
     """User email."""
-    password: Optional[str] = None
+    password: str | None = None
     """API key or legacy password."""
-    access_token: Optional[str] = None
+    access_token: str | None = None
     """User access token."""
     uid: str = "null"
     """Universal user ID."""
-    uuid: Optional[UUID] = None
+    _uuid: UUID | None = None
     """Lamin's internal user ID."""
-    name: Optional[str] = None
+    name: str | None = None
     """Full name."""
 
     def __repr__(self) -> str:

lamindb_setup/core/_setup_bionty_sources.py

@@ -1,14 +1,21 @@
-from ._settings_instance import InstanceSettings
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 from django.db.utils import OperationalError, ProgrammingError
 
+if TYPE_CHECKING:
+    from ._settings_instance import InstanceSettings
+
 
 def write_bionty_sources(isettings: InstanceSettings) -> None:
     """Write bionty sources to PublicSource table."""
     if "bionty" not in isettings.schema:
         return None
     import shutil
-    from bionty_base.dev._handle_sources import parse_sources_yaml
+
     import bionty_base
+    from bionty_base.dev._handle_sources import parse_sources_yaml
     from lnschema_bionty.models import PublicSource
 
     shutil.copy(

lamindb_setup/core/cloud_sqlite_locker.py

@@ -1,11 +1,16 @@
+from __future__ import annotations
+
 from datetime import datetime, timezone
-from pathlib import Path
-from typing import Optional, Union
 from functools import wraps
-from uuid import UUID
+from typing import TYPE_CHECKING, Optional, Union
+
 from lamin_utils import logger
 
-from .upath import UPath, infer_filesystem, create_mapper
+from .upath import UPath, create_mapper, infer_filesystem
+
+if TYPE_CHECKING:
+    from pathlib import Path
+    from uuid import UUID
 
 EXPIRATION_TIME = 24 * 60 * 60 * 7  # 7 days
 
@@ -31,9 +36,7 @@ class empty_locker:
 
 
 class Locker:
-    def __init__(
-        self, user_uid: str, storage_root: Union[UPath, Path], instance_id: UUID
-    ):
+    def __init__(self, user_uid: str, storage_root: UPath | Path, instance_id: UUID):
         logger.debug(
             f"init cloud sqlite locker: {user_uid}, {storage_root}, {instance_id}."
         )
@@ -169,7 +172,7 @@ class Locker:
         return self._has_lock
 
 
-_locker: Optional[Locker] = None
+_locker: Locker | None = None
 
 
 def get_locker(isettings) -> Locker:
@@ -184,9 +187,9 @@ def get_locker(isettings) -> Locker:
         _locker is None
         or _locker.user != user_uid
         or _locker.root is not storage_root
-        or _locker.instance_id != isettings.id
+        or _locker.instance_id != isettings._id
     ):
-        _locker = Locker(user_uid, storage_root, isettings.id)
+        _locker = Locker(user_uid, storage_root, isettings._id)
 
     return _locker
 

lamindb_setup/core/django.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 # flake8: noqa
 import builtins
 import os
@@ -103,3 +105,6 @@ def setup_django(
 
     global IS_SETUP
     IS_SETUP = True
+
+    if isettings.keep_artifacts_local:
+        isettings._search_local_root()

lamindb_setup/core/exceptions.py

@@ -1,10 +1,12 @@
+from __future__ import annotations
+
 from typing import Optional
 
 
 class DefaultMessageException(Exception):
-    default_message: Optional[str] = None
+    default_message: str | None = None
 
-    def __init__(self, message: Optional[str] = None):
+    def __init__(self, message: str | None = None):
         if message is None:
             message = self.default_message
         super().__init__(message)

lamindb_setup/core/hashing.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 """Hashing.
 
 .. autosummary::
@@ -10,8 +12,16 @@
 
 import base64
 import hashlib
-from typing import List, Set, Tuple
-from .types import Path, UPathStr
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .types import Path, UPathStr
+
+
+def hash_and_encode_as_b62(s: str) -> str:
+    from lamin_utils._base62 import encodebytes
+
+    return encodebytes(hashlib.md5(s.encode()).digest())
 
 
 def to_b64_str(bstr: bytes):
@@ -24,13 +34,13 @@ def b16_to_b64(s: str):
 
 
 # a lot to read about this: lamin-notes/2022/hashing
-def hash_set(s: Set[str]) -> str:
+def hash_set(s: set[str]) -> str:
     bstr = ":".join(sorted(s)).encode("utf-8")
     # as we're truncating at 20 b64, we choose md5 over sha512
     return to_b64_str(hashlib.md5(bstr).digest())[:20]
 
 
-def hash_md5s_from_dir(etags: List[str]) -> Tuple[str, str]:
+def hash_md5s_from_dir(etags: list[str]) -> tuple[str, str]:
     # need to sort below because we don't want the order of parsing the dir to
     # affect the hash
     digests = b"".join(
@@ -49,7 +59,7 @@ def hash_code(file_path: UPathStr):
     return hashlib.sha1(blob)
 
 
-def hash_file(file_path: Path, chunk_size=50 * 1024 * 1024) -> Tuple[str, str]:
+def hash_file(file_path: Path, chunk_size=50 * 1024 * 1024) -> tuple[str, str]:
     chunks = []
     with open(file_path, "rb") as fp:
         # read first chunk

lamindb_setup/core/types.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 """Types.
 
 .. autosummary::
@@ -5,10 +7,11 @@
 
    UPathStr
 """
+# we need Union here because __future__ annotations doesn't work with TypeAlias
+from pathlib import Path
 from typing import (
     Union,
-)  # we need Union here because __future__ annotations doesn't work with TypeAlias
-from pathlib import Path
+)
 
 # UPath is subclass of Path, hence, it's not necessary to list UPath
 # we keep it in the name of the TypeAlias to make it clear to users that