lamindb_setup 0.71.3__tar.gz → 0.72.0__tar.gz

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.71.3
+Version: 0.72.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <laminlabs@gmail.com>
 Description-Content-Type: text/markdown

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/docs/changelog.md

@@ -3,7 +3,14 @@
 <!-- prettier-ignore -->
 Name | PR | Developer | Date | Version
 --- | --- | --- | --- | ---
-♻️ Better treatment of current_user_id | [754](https://github.com/laminlabs/lamindb-setup/pull/754) | [falexwolf](https://github.com/falexwolf) | 2024-05-06 | 0.72.2
+🐛 Check empty after storage record root init in delete | [763](https://github.com/laminlabs/lamindb-setup/pull/763) | [Koncopd](https://github.com/Koncopd) | 2024-05-18 |
+✨ Call `access_aws` for all paths and cache | [762](https://github.com/laminlabs/lamindb-setup/pull/762) | [Koncopd](https://github.com/Koncopd) | 2024-05-18 |
+⚡️ Speed-up file hash | [761](https://github.com/laminlabs/lamindb-setup/pull/761) | [Koncopd](https://github.com/Koncopd) | 2024-05-11 | 0.71.4
+♻️ Account for public storage locations | [758](https://github.com/laminlabs/lamindb-setup/pull/758) | [falexwolf](https://github.com/falexwolf) | 2024-05-10 |
+♻️ Make init_instance_hub idempotent | [757](https://github.com/laminlabs/lamindb-setup/pull/757) | [falexwolf](https://github.com/falexwolf) | 2024-05-09 |
+♻️ Refactor delete & connect | [756](https://github.com/laminlabs/lamindb-setup/pull/756) | [falexwolf](https://github.com/falexwolf) | 2024-05-09 |
+🔥 Remove laminapp-admin logic | [755](https://github.com/laminlabs/lamindb-setup/pull/755) | [falexwolf](https://github.com/falexwolf) | 2024-05-08 |
+♻️ Better treatment of current_user_id | [754](https://github.com/laminlabs/lamindb-setup/pull/754) | [falexwolf](https://github.com/falexwolf) | 2024-05-06 | 0.71.2
 💚 Fix tests | [753](https://github.com/laminlabs/lamindb-setup/pull/753) | [fredericenard](https://github.com/fredericenard) | 2024-05-06 |
 ♻️ Actually use local db | [751](https://github.com/laminlabs/lamindb-setup/pull/751) | [falexwolf](https://github.com/falexwolf) | 2024-05-06 | 0.71.2
 ✏️ Restore view_tree py3.9 compatibility | [750](https://github.com/laminlabs/lamindb-setup/pull/750) | [sunnyosun](https://github.com/sunnyosun) | 2024-05-06 |

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/docs/hub-cloud/03-add-managed-storage.ipynb

@@ -255,7 +255,8 @@
     "# root.fs contains the underlying fsspec filesystem\n",
     "assert (\n",
     "    ln_setup.settings.storage.root.fs.cache_regions  # set by lamindb to True for s3 by default\n",
-    ")"
+    ")\n",
+    "assert ln_setup.settings.storage._instance_id is not None"
    ]
   },
   {
@@ -287,6 +288,26 @@
     "assert not ln_setup.settings.storage.root.fs.cache_regions"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "015b8f36",
+   "metadata": {},
+   "source": [
+    "Cloud storage with mere read access:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4ef50cab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with pytest.raises(ValueError) as error:\n",
+    "    set_managed_storage(\"gs://rxrx1-europe-west4/images/test/HEPG2-08\")\n",
+    "assert error.exconly().startswith(\"ValueError: Cannot manage storage without write access\")"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "83a2ee6b",

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/docs/hub-cloud/05-init-hosted-instance.ipynb

@@ -81,7 +81,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from lamindb_setup.core.upath import HOSTED_BUCKETS"
+    "from lamindb_setup.core._aws_credentials import HOSTED_BUCKETS"
    ]
   },
   {

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.71.3"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.72.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/lamindb_setup/_connect_instance.py

@@ -55,7 +55,10 @@ def check_db_dsn_equal_up_to_credentials(db_dsn_hub, db_dsn_local):
 
 
 def update_db_using_local(
-    hub_instance_result: dict[str, str], settings_file: Path, db: str | None = None
+    hub_instance_result: dict[str, str],
+    settings_file: Path,
+    db: str | None = None,
+    raise_permission_error=True,
 ) -> str | None:
     db_updated = None
     # check if postgres
@@ -77,7 +80,11 @@ def update_db_using_local(
                 db_dsn_local = LaminDsnModel(db=isettings.db)
             else:
                 # just take the default hub result and ensure there is actually a user
-                if db_dsn_hub.db.user == "none" and db_dsn_hub.db.password == "none":
+                if (
+                    db_dsn_hub.db.user == "none"
+                    and db_dsn_hub.db.password == "none"
+                    and raise_permission_error
+                ):
                     raise PermissionError(
                         "No database access, please ask your admin to provide you with"
                         " a DB URL and pass it via --db <db_url>"
@@ -101,6 +108,65 @@ def update_db_using_local(
     return db_updated
 
 
+def _connect_instance(
+    owner: str,
+    name: str,
+    *,
+    db: str | None = None,
+    raise_permission_error: bool = True,
+) -> InstanceSettings:
+    settings_file = instance_settings_file(name, owner)
+    make_hub_request = True
+    if settings_file.exists():
+        isettings = load_instance_settings(settings_file)
+        # skip hub request for a purely local instance
+        make_hub_request = isettings.is_remote
+    if make_hub_request:
+        # the following will return a string if the instance does not exist
+        # on the hub
+        hub_result = load_instance_from_hub(owner=owner, name=name)
+        # if hub_result is not a string, it means it made a request
+        # that successfully returned metadata
+        if not isinstance(hub_result, str):
+            instance_result, storage_result = hub_result
+            db_updated = update_db_using_local(
+                instance_result,
+                settings_file,
+                db=db,
+                raise_permission_error=raise_permission_error,
+            )
+            ssettings = StorageSettings(
+                root=storage_result["root"],
+                region=storage_result["region"],
+                uid=storage_result["lnid"],
+                uuid=UUID(storage_result["id"]),
+                instance_id=UUID(instance_result["id"]),
+            )
+            isettings = InstanceSettings(
+                id=UUID(instance_result["id"]),
+                owner=owner,
+                name=name,
+                storage=ssettings,
+                db=db_updated,
+                schema=instance_result["schema_str"],
+                git_repo=instance_result["git_repo"],
+                keep_artifacts_local=bool(instance_result["keep_artifacts_local"]),
+                is_on_hub=True,
+            )
+            check_whether_migrations_in_sync(instance_result["lamindb_version"])
+        else:
+            message = INSTANCE_NOT_FOUND_MESSAGE.format(
+                owner=owner, name=name, hub_result=hub_result
+            )
+            if settings_file.exists():
+                isettings = load_instance_settings(settings_file)
+                if isettings.is_remote:
+                    raise InstanceNotFoundError(message)
+            else:
+                raise InstanceNotFoundError(message)
+    return isettings
+
+
 @unlock_cloud_sqlite_upon_exception(ignore_prev_locker=True)
 def connect(
     slug: str,
@@ -134,71 +200,22 @@ def connect(
         elif settings._instance_exists and f"{owner}/{name}" != settings.instance.slug:
             close_instance(mute=True)
 
-        settings_file = instance_settings_file(name, owner)
-
-        make_hub_request = True
-        if settings_file.exists():
-            isettings = load_instance_settings(settings_file)
-            # mimic instance_result from hub
-            instance_result = {"id": isettings._id.hex}
-            # skip hub request for a purely local instance
-            make_hub_request = isettings.is_remote
-
-        if make_hub_request:
-            # the following will return a string if the instance does not exist
-            # on the hub
-            hub_result = load_instance_from_hub(owner=owner, name=name)
-            # if hub_result is not a string, it means it made a request
-            # that successfully returned metadata
-            if not isinstance(hub_result, str):
-                instance_result, storage_result = hub_result
-                db_updated = update_db_using_local(
-                    instance_result, settings_file, db=db
-                )
-                ssettings = StorageSettings(
-                    root=storage_result["root"],
-                    region=storage_result["region"],
-                    uid=storage_result["lnid"],
-                    uuid=UUID(storage_result["id"]),
-                    instance_id=UUID(instance_result["id"]),
-                )
-                isettings = InstanceSettings(
-                    id=UUID(instance_result["id"]),
-                    owner=owner,
-                    name=name,
-                    storage=ssettings,
-                    db=db_updated,
-                    schema=instance_result["schema_str"],
-                    git_repo=instance_result["git_repo"],
-                    keep_artifacts_local=bool(instance_result["keep_artifacts_local"]),
-                    is_on_hub=True,
-                )
-                check_whether_migrations_in_sync(instance_result["lamindb_version"])
+        try:
+            isettings = _connect_instance(owner, name, db=db)
+        except InstanceNotFoundError as e:
+            if _raise_not_found_error:
+                raise e
             else:
-                message = INSTANCE_NOT_FOUND_MESSAGE.format(
-                    owner=owner, name=name, hub_result=hub_result
-                )
-                if settings_file.exists():
-                    isettings = load_instance_settings(settings_file)
-                    if isettings.is_remote:
-                        if _raise_not_found_error:
-                            raise InstanceNotFoundError(message)
-                        return "instance-not-found"
-
-                else:
-                    if _raise_not_found_error:
-                        raise InstanceNotFoundError(message)
-                    return "instance-not-found"
-
+                return "instance-not-found"
+        if isinstance(isettings, str):
+            return isettings
         if storage is not None:
             update_isettings_with_storage(isettings, storage)
         isettings._persist()
         if _test:
             return None
         silence_loggers()
-        check, msg = isettings._load_db(
-            do_not_lock_for_laminapp_admin=True
-        )  # this also updates local SQLite
+        check, msg = isettings._load_db()
         if not check:
             local_db = (
                 isettings._is_cloud_sqlite and isettings._sqlite_file_local.exists()
@@ -216,7 +233,7 @@ def connect(
                     f"instance exists with id {isettings._id.hex}, but database is not"
                     " loadable: re-initializing"
                 )
-                return "instance-corrupted-or-deleted", instance_result
+                return "instance-corrupted-or-deleted"
         # this is for testing purposes only
         if _TEST_FAILED_LOAD:
             raise RuntimeError("Technical testing error.")

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/lamindb_setup/_delete.py

@@ -1,29 +1,24 @@
 from __future__ import annotations
 
 import shutil
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 from uuid import UUID
 
 from lamin_utils import logger
 
-from ._connect_instance import (
-    INSTANCE_NOT_FOUND_MESSAGE,
-    InstanceNotFoundError,
-    get_owner_name_from_identifier,
-)
-from .core._hub_core import connect_instance as load_instance_from_hub
+from ._connect_instance import _connect_instance, get_owner_name_from_identifier
+from .core._aws_credentials import HOSTED_BUCKETS
 from .core._hub_core import delete_instance as delete_instance_on_hub
 from .core._hub_core import get_storage_records_for_instance
 from .core._settings import settings
-from .core._settings_instance import InstanceSettings
-from .core._settings_load import load_instance_settings
 from .core._settings_storage import StorageSettings
-from .core._settings_store import instance_settings_file
-from .core.upath import HOSTED_BUCKETS, check_storage_is_empty
+from .core.upath import check_storage_is_empty
 
 if TYPE_CHECKING:
     from pathlib import Path
 
+    from .core._settings_instance import InstanceSettings
+
 
 def delete_cache(cache_dir: Path):
     if cache_dir is not None and cache_dir.exists():
@@ -67,42 +62,8 @@ def delete(slug: str, force: bool = False, require_empty: bool = True) -> int |
         force: Whether to skip the confirmation prompt.
         require_empty: Whether to check if the instance is empty before deleting.
     """
-    instance_owner, instance_name = get_owner_name_from_identifier(slug)
-    if settings._instance_exists and settings.instance.name == instance_name:
-        isettings = settings.instance
-    else:
-        settings_file = instance_settings_file(instance_name, instance_owner)
-        if not settings_file.exists():
-            hub_result = load_instance_from_hub(
-                owner=instance_owner, name=instance_name
-            )
-            if isinstance(hub_result, str):
-                message = INSTANCE_NOT_FOUND_MESSAGE.format(
-                    owner=instance_owner,
-                    name=instance_name,
-                    hub_result=hub_result,
-                )
-                raise InstanceNotFoundError(message)
-            instance_result, storage_result = hub_result
-            ssettings = StorageSettings(
-                root=storage_result["root"],
-                region=storage_result["region"],
-                uid=storage_result["lnid"],
-                uuid=UUID(storage_result["id"]),
-            )
-            isettings = InstanceSettings(
-                id=UUID(instance_result["id"]),
-                owner=instance_owner,
-                name=instance_name,
-                storage=ssettings,
-                keep_artifacts_local=bool(instance_result["keep_artifacts_local"]),
-                db=instance_result["db"] if "db" in instance_result else None,
-                schema=instance_result["schema_str"],
-                git_repo=instance_result["git_repo"],
-                is_on_hub=True,
-            )
-        else:
-            isettings = load_instance_settings(settings_file)
+    owner, name = get_owner_name_from_identifier(slug)
+    isettings = _connect_instance(owner, name, raise_permission_error=False)
     if isettings.dialect != "sqlite":
         logger.warning(
             f"delete() does not yet affect your Postgres database at {isettings.db}"
@@ -149,11 +110,11 @@ def delete(slug: str, force: bool = False, require_empty: bool = True) -> int |
         for storage_record in storage_records:
             if storage_record["root"] == isettings.storage.root_as_str:
                 continue
+            ssettings = StorageSettings(storage_record["root"])  # type: ignore
             check_storage_is_empty(
-                storage_record["root"],  # type: ignore
+                ssettings.root,  # type: ignore
                 raise_error=require_empty,
             )
-            ssettings = StorageSettings(storage_record["root"])  # type: ignore
             if ssettings._mark_storage_root.exists():
                 ssettings._mark_storage_root.unlink(
                     missing_ok=True  # this is totally weird, but needed on Py3.11

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/lamindb_setup/_init_instance.py

@@ -44,15 +44,17 @@ def register_storage_in_instance(ssettings: StorageSettings):
 
     from .core.hashing import hash_and_encode_as_b62
 
-    assert ssettings._instance_id is not None
-
+    if ssettings._instance_id is not None:
+        instance_uid = hash_and_encode_as_b62(ssettings._instance_id.hex)[:12]
+    else:
+        instance_uid = None
     # how do we ensure that this function is only called passing
     # the managing instance?
     defaults = {
         "root": ssettings.root_as_str,
         "type": ssettings.type,
         "region": ssettings.region,
-        "instance_uid": hash_and_encode_as_b62(ssettings._instance_id.hex)[:12],
+        "instance_uid": instance_uid,
         "created_by_id": current_user_id(),
     }
     if ssettings._uid is not None:
@@ -67,20 +69,19 @@ def register_storage_in_instance(ssettings: StorageSettings):
 def register_user(usettings):
     from lnschema_core.models import User
 
-    if usettings.handle != "laminapp-admin":
-        try:
-            # need to have try except because of integer primary key migration
-            user, created = User.objects.update_or_create(
-                uid=usettings.uid,
-                defaults={
-                    "handle": usettings.handle,
-                    "name": usettings.name,
-                },
-            )
-        # for users with only read access, except via ProgrammingError
-        # ProgrammingError: permission denied for table lnschema_core_user
-        except (OperationalError, FieldError, ProgrammingError):
-            pass
+    try:
+        # need to have try except because of integer primary key migration
+        user, created = User.objects.update_or_create(
+            uid=usettings.uid,
+            defaults={
+                "handle": usettings.handle,
+                "name": usettings.name,
+            },
+        )
+    # for users with only read access, except via ProgrammingError
+    # ProgrammingError: permission denied for table lnschema_core_user
+    except (OperationalError, FieldError, ProgrammingError):
+        pass
 
 
 def register_user_and_storage_in_instance(isettings: InstanceSettings, usettings):

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/lamindb_setup/_set_managed_storage.py

@@ -31,6 +31,10 @@ def set_managed_storage(root: UPathStr, **fs_kwargs):
     ssettings = init_storage(
         root=root, instance_id=settings.instance._id, register_hub=True
     )
+    if ssettings._instance_id is None:
+        raise ValueError(
+            f"Cannot manage storage without write access: {ssettings.root}"
+        )
     settings.instance._storage = ssettings
     settings.instance._persist()  # this also updates the settings object
     register_storage_in_instance(ssettings)

lamindb_setup-0.72.0/lamindb_setup/core/_aws_credentials.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import os
+import time
+
+from upath.implementations.cloud import S3Path
+
+HOSTED_REGIONS = [
+    "eu-central-1",
+    "eu-west-2",
+    "us-east-1",
+    "us-east-2",
+    "us-west-1",
+    "us-west-2",
+]
+lamin_env = os.getenv("LAMIN_ENV")
+if lamin_env is None or lamin_env == "prod":
+    hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
+    hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
+    HOSTED_BUCKETS = tuple(hosted_buckets_list)
+else:
+    HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
+
+
+AWS_CREDENTIALS_EXPIRATION = 11 * 60 * 60  # refresh credentials after 11 hours
+
+
+class AWSCredentialsManager:
+    def __init__(self):
+        self._credentials_cache = {}
+
+        from s3fs import S3FileSystem
+
+        # this is cached so will be resued with the connection initialized
+        fs = S3FileSystem(cache_regions=True)
+        fs.connect()
+        self.anon = fs.session._credentials is None
+
+    def _find_root(self, path_str: str) -> str | None:
+        roots = self._credentials_cache.keys()
+        if path_str in roots:
+            return path_str
+        roots = sorted(roots, key=len, reverse=True)
+        for root in roots:
+            if path_str.startswith(root):
+                return root
+        return None
+
+    def _is_active(self, root: str) -> bool:
+        return (
+            time.time() - self._credentials_cache[root]["time"]
+        ) < AWS_CREDENTIALS_EXPIRATION
+
+    def _set_cached_credentials(self, root: str, credentials: dict):
+        if root not in self._credentials_cache:
+            self._credentials_cache[root] = {}
+        self._credentials_cache[root]["credentials"] = credentials
+        self._credentials_cache[root]["time"] = time.time()
+
+    def _get_cached_credentials(self, root: str) -> dict:
+        return self._credentials_cache[root]["credentials"]
+
+    def _path_inject_options(self, path: S3Path, credentials: dict) -> S3Path:
+        if credentials == {}:
+            # credentials were specified manually for the path
+            if "anon" in path._kwargs:
+                anon = path._kwargs["anon"]
+            elif path.fs.key is not None and path.fs.secret is not None:
+                anon = False
+            else:
+                anon = self.anon
+            connection_options = {"anon": anon}
+        else:
+            connection_options = credentials
+
+        if "cache_regions" in path._kwargs:
+            cache_regions = path._kwargs["cache_regions"]
+        else:
+            cache_regions = True
+
+        return S3Path(path, cache_regions=cache_regions, **connection_options)
+
+    def enrich_path(self, path: S3Path, access_token: str | None = None) -> S3Path:
+        path_str = path.as_posix().rstrip("/")
+        root = self._find_root(path_str)
+
+        if root is not None:
+            set_cache = False
+            credentials = self._get_cached_credentials(root)
+
+            if access_token is not None:
+                set_cache = True
+            elif credentials != {}:
+                # update credentials
+                if not self._is_active(root):
+                    set_cache = True
+        else:
+            set_cache = True
+
+        if set_cache:
+            from ._hub_core import access_aws
+
+            storage_root_info = access_aws(path_str, access_token=access_token)
+            accessibility = storage_root_info["accessibility"]
+
+            is_managed = accessibility.get("is_managed", False)
+            if is_managed:
+                credentials = storage_root_info["credentials"]
+            else:
+                credentials = {}
+
+            if access_token is None:
+                if "storage_root" in accessibility:
+                    root = accessibility["storage_root"]
+                # just to be safe
+                root = None if root == "" else root
+                if root is None:
+                    # heuristic
+                    # do not write the first level for the known hosted buckets
+                    if path_str.startswith(HOSTED_BUCKETS):
+                        root = "/".join(path.path.rstrip("/").split("/")[:2])
+                    else:
+                        # write the bucket for everything else
+                        root = path._url.netloc
+                    root = "s3://" + root
+                self._set_cached_credentials(root, credentials)
+
+        return self._path_inject_options(path, credentials)
+
+
+_aws_credentials_manager: AWSCredentialsManager | None = None
+
+
+def get_aws_credentials_manager() -> AWSCredentialsManager:
+    global _aws_credentials_manager
+
+    if _aws_credentials_manager is None:
+        _aws_credentials_manager = AWSCredentialsManager()
+
+    return _aws_credentials_manager

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/lamindb_setup/core/_hub_core.py

@@ -284,9 +284,11 @@ def _init_instance(isettings: InstanceSettings, client: Client) -> None:
     # as then init_instance is no longer idempotent
     try:
         client.table("instance").insert(fields, returning="minimal").execute()
-    except APIError as e:
-        logger.warning("instance likely already exists")
-        raise e
+    except APIError:
+        logger.warning(
+            f"instance already existed at: https://lamin.ai/{isettings.owner}/{isettings.name}"
+        )
+        return None
     client.table("storage").update(
         {"instance_id": isettings._id.hex, "is_default": True}
     ).eq("id", isettings.storage._uuid.hex).execute()  # type: ignore
@@ -351,35 +353,43 @@ def _connect_instance(
     return instance, storage
 
 
-def access_aws(storage_root: str, access_token: str | None = None) -> dict[str, str]:
+def access_aws(storage_root: str, access_token: str | None = None) -> dict[str, dict]:
     from ._settings import settings
 
     if settings.user.handle != "anonymous" or access_token is not None:
-        credentials = call_with_fallback_auth(
+        storage_root_info = call_with_fallback_auth(
             _access_aws, storage_root=storage_root, access_token=access_token
         )
-        return credentials
+        return storage_root_info
     else:
         raise RuntimeError("Can only get access to AWS if authenticated.")
 
 
-def _access_aws(*, storage_root: str, client: Client) -> dict[str, str]:
+def _access_aws(*, storage_root: str, client: Client) -> dict[str, dict]:
     import lamindb_setup
 
+    storage_root_info: dict[str, dict] = {"credentials": {}, "accessibility": {}}
     response = client.functions.invoke(
         "access-aws",
         invoke_options={"body": {"storage_root": storage_root}},
     )
     if response is not None and response != b"{}":
-        loaded_credentials = json.loads(response)["Credentials"]
-        credentials = {}
+        data = json.loads(response)
+
+        loaded_credentials = data["Credentials"]
+        loaded_accessibility = data["StorageAccessibility"]
+
+        credentials = storage_root_info["credentials"]
         credentials["key"] = loaded_credentials["AccessKeyId"]
         credentials["secret"] = loaded_credentials["SecretAccessKey"]
         credentials["token"] = loaded_credentials["SessionToken"]
-        return credentials
+
+        accessibility = storage_root_info["accessibility"]
+        accessibility["storage_root"] = loaded_accessibility["storageRoot"]
+        accessibility["is_managed"] = loaded_accessibility["isManaged"]
     elif lamindb_setup._TESTING:
         raise RuntimeError(f"access-aws errored: {response}")
-    return {}
+    return storage_root_info
 
 
 def get_lamin_site_base_url():

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/lamindb_setup/core/_settings_instance.py

@@ -42,7 +42,7 @@ class InstanceSettings:
         db: str | None = None,  # DB URI
         schema: str | None = None,  # comma-separated string of schema names
         git_repo: str | None = None,  # a git repo URL
-        is_on_hub: bool = False,  # initialized from hub
+        is_on_hub: bool | None = None,  # initialized from hub
     ):
         from ._hub_utils import validate_db_arg
 
@@ -367,17 +367,25 @@ class InstanceSettings:
 
     @property
     def is_on_hub(self) -> bool:
-        """Is this instance on the hub.
+        """Is this instance on the hub?
 
-        Only works if user has access to the instance.
+        Can only reliably establish if user has access to the instance. Will
+        return `False` in case the instance isn't found.
         """
         if self._is_on_hub is None:
             from ._hub_client import call_with_fallback_auth
             from ._hub_crud import select_instance_by_id
+            from ._settings import settings
 
-            response = call_with_fallback_auth(
-                select_instance_by_id, instance_id=self._id.hex
-            )
+            if settings.user.handle != "anonymous":
+                response = call_with_fallback_auth(
+                    select_instance_by_id, instance_id=self._id.hex
+                )
+            else:
+                response = call_with_fallback(
+                    select_instance_by_id, instance_id=self._id.hex
+                )
+                logger.warning("calling anonymously, will miss private instances")
             if response is None:
                 self._is_on_hub = False
             else:
@@ -406,9 +414,7 @@ class InstanceSettings:
 
         setup_django(self, init=True)
 
-    def _load_db(
-        self, do_not_lock_for_laminapp_admin: bool = False
-    ) -> tuple[bool, str]:
+    def _load_db(self) -> tuple[bool, str]:
         # Is the database available and initialized as LaminDB?
         # returns a tuple of status code and message
         if self.dialect == "sqlite" and not self._sqlite_file.exists():
@@ -423,15 +429,8 @@ class InstanceSettings:
 
         from .django import setup_django
 
-        # lock in all cases except if do_not_lock_for_laminapp_admin is True and
-        # user is `laminapp-admin`
-        # value doesn't matter if not a cloud sqlite instance
-        lock_cloud_sqlite = self._is_cloud_sqlite and (
-            not do_not_lock_for_laminapp_admin
-            or settings.user.handle != "laminapp-admin"
-        )
         # we need the local sqlite to setup django
-        self._update_local_sqlite_file(lock_cloud_sqlite=lock_cloud_sqlite)
+        self._update_local_sqlite_file(lock_cloud_sqlite=self._is_cloud_sqlite)
         # setting up django also performs a check for migrations & prints them
         # as warnings
         # this should fail, e.g., if the db is not reachable

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/lamindb_setup/core/_settings_storage.py

@@ -10,11 +10,11 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, Union
 from appdirs import AppDirs
 from lamin_utils import logger
 
+from ._aws_credentials import HOSTED_REGIONS, get_aws_credentials_manager
 from ._aws_storage import find_closest_aws_region
 from ._settings_save import save_system_storage_settings
 from ._settings_store import system_storage_settings_file
 from .upath import (
-    HOSTED_REGIONS,
     LocalPathClasses,
     UPath,
     convert_pathlike,
@@ -113,11 +113,20 @@ def init_storage(
     )
     # the below might update the uid with one that's already taken on the hub
     if ssettings.type_is_cloud or register_hub:
+        from ._hub_core import delete_storage_record
         from ._hub_core import init_storage as init_storage_hub
 
         init_storage_hub(ssettings)
     # below comes last only if everything else was successful
-    mark_storage_root(ssettings.root, ssettings.uid)  # type: ignore
+    try:
+        mark_storage_root(ssettings.root, ssettings.uid)  # type: ignore
+    except Exception:
+        logger.important(
+            f"due to lack of write access, LaminDB won't manage storage location: {ssettings.root}"
+        )
+        if ssettings._uuid is not None:
+            delete_storage_record(ssettings._uuid)  # type: ignore
+        ssettings._instance_id = None
     return ssettings
 
 
@@ -142,6 +151,7 @@ class StorageSettings:
         uid: str | None = None,
         uuid: UUID | None = None,
         instance_id: UUID | None = None,
+        # note that passing access_token prevents credentials caching
         access_token: str | None = None,
     ):
         self._uid = uid
@@ -226,6 +236,12 @@ class StorageSettings:
         if self._root is None:
             # below makes network requests to get credentials
             self._root = create_path(self._root_init, access_token=self.access_token)
+        elif getattr(self._root, "protocol", "") == "s3":
+            # this is needed to be sure that the root always has nonexpired credentials
+            # this just checks for time of the cached credentials in most cases
+            return get_aws_credentials_manager().enrich_path(
+                self._root, access_token=self.access_token
+            )
         return self._root
 
     def _set_fs_kwargs(self, **kwargs):

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/lamindb_setup/core/hashing.py

@@ -40,11 +40,11 @@ def hash_set(s: set[str]) -> str:
     return to_b64_str(hashlib.md5(bstr).digest())[:20]
 
 
-def hash_md5s_from_dir(etags: list[str]) -> tuple[str, str]:
+def hash_md5s_from_dir(hashes: list[str]) -> tuple[str, str]:
     # need to sort below because we don't want the order of parsing the dir to
     # affect the hash
     digests = b"".join(
-        hashlib.md5(etag.encode("utf-8")).digest() for etag in sorted(etags)
+        hashlib.md5(hash.encode("utf-8")).digest() for hash in sorted(hashes)
     )
     digest = hashlib.md5(digests).digest()
     return to_b64_str(digest)[:22], "md5-d"
@@ -59,24 +59,27 @@ def hash_code(file_path: UPathStr):
     return hashlib.sha1(blob)
 
 
-def hash_file(file_path: Path, chunk_size=50 * 1024 * 1024) -> tuple[str, str]:
-    chunks = []
+def hash_file(
+    file_path: Path,
+    file_size: int | None = None,
+    chunk_size: int | None = 50 * 1024 * 1024,
+) -> tuple[str, str]:
     with open(file_path, "rb") as fp:
-        # read first chunk
-        chunks = [fp.read(chunk_size)]
-        # try reading the 2nd chunk
-        data = fp.read(chunk_size)
-        if data:
-            # go to end of file
+        if file_size is None:
+            fp.seek(0, 2)
+            file_size = fp.tell()
+            fp.seek(0, 0)
+        if chunk_size is None:
+            chunk_size = file_size
+        first_chunk = fp.read(chunk_size)
+        if file_size <= chunk_size:
+            digest = hashlib.md5(first_chunk).digest()
+            hash_type = "md5"
+        else:
             fp.seek(-chunk_size, 2)
-            # read last chunk
-            data = fp.read(chunk_size)
-            chunks.append(data)
-    if len(chunks) == 1:
-        digest = hashlib.md5(chunks[0]).digest()
-        hash_type = "md5"
-    else:
-        digests = b"".join(hashlib.sha1(chunk).digest() for chunk in chunks)
-        digest = hashlib.sha1(digests).digest()
-        hash_type = "sha1-fl"  # sha1 first last chunk
+            last_chunk = fp.read(chunk_size)
+            digest = hashlib.sha1(
+                hashlib.sha1(first_chunk).digest() + hashlib.sha1(last_chunk).digest()
+            ).digest()
+            hash_type = "sha1-fl"
     return to_b64_str(digest)[:22], hash_type

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/lamindb_setup/core/upath.py

@@ -11,13 +11,13 @@ from itertools import islice
 from pathlib import Path, PurePosixPath
 from typing import TYPE_CHECKING, Any, Literal
 
-import botocore.session
 import fsspec
 from lamin_utils import logger
 from upath import UPath
 from upath.implementations.cloud import CloudPath, S3Path  # keep CloudPath!
 from upath.implementations.local import LocalPath, PosixUPath, WindowsUPath
 
+from ._aws_credentials import HOSTED_BUCKETS, get_aws_credentials_manager
 from .hashing import b16_to_b64, hash_md5s_from_dir
 
 if TYPE_CHECKING:
@@ -158,7 +158,8 @@ def print_hook(size: int, value: int, objectname: str, action: str):
     progress_in_percent = (value / size) * 100
     out = f"... {action} {objectname}:" f" {min(progress_in_percent, 100):4.1f}%"
     if "NBPRJ_TEST_NBPATH" not in os.environ:
-        print(out, end="\r")
+        end = "\n" if progress_in_percent >= 100 else "\r"
+        print(out, end=end)
 
 
 class ProgressCallback(fsspec.callbacks.Callback):
@@ -670,77 +671,12 @@ def convert_pathlike(pathlike: UPathStr) -> UPath:
     return path
 
 
-HOSTED_REGIONS = [
-    "eu-central-1",
-    "eu-west-2",
-    "us-east-1",
-    "us-east-2",
-    "us-west-1",
-    "us-west-2",
-]
-lamin_env = os.getenv("LAMIN_ENV")
-if lamin_env is None or lamin_env == "prod":
-    hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
-    hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
-    HOSTED_BUCKETS = tuple(hosted_buckets_list)
-else:
-    HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
-credentials_cache: dict[str, dict[str, str]] = {}
-AWS_CREDENTIALS_PRESENT = None
-
-
 def create_path(path: UPath, access_token: str | None = None) -> UPath:
     path = convert_pathlike(path)
     # test whether we have an AWS S3 path
     if not isinstance(path, S3Path):
         return path
-    # test whether AWS credentials are present
-    global AWS_CREDENTIALS_PRESENT
-
-    if AWS_CREDENTIALS_PRESENT is not None:
-        anon = AWS_CREDENTIALS_PRESENT
-    else:
-        if path.fs.key is not None and path.fs.secret is not None:
-            anon = False
-        else:
-            # we could do path.fs.connect()
-            # and check path.fs.session._credentials, but it'd be slower
-            session = botocore.session.get_session()
-            credentials = session.get_credentials()
-            if credentials is None or credentials.access_key is None:
-                anon = True
-            else:
-                anon = False
-            # cache credentials and reuse further
-            AWS_CREDENTIALS_PRESENT = anon
-
-    # test whether we are on hosted storage or not
-    path_str = path.as_posix()
-    is_hosted_storage = path_str.startswith(HOSTED_BUCKETS)
-
-    if not is_hosted_storage:
-        # make anon request if no credentials present
-        return UPath(path, cache_regions=True, anon=anon)
-
-    root_folder = "/".join(path_str.replace("s3://", "").split("/")[:2])
-
-    if access_token is None and root_folder in credentials_cache:
-        credentials = credentials_cache[root_folder]
-    else:
-        from ._hub_core import access_aws
-
-        credentials = access_aws(
-            storage_root=f"s3://{root_folder}", access_token=access_token
-        )
-        if access_token is None:
-            credentials_cache[root_folder] = credentials
-
-    return UPath(
-        path,
-        key=credentials["key"],
-        secret=credentials["secret"],
-        token=credentials["token"],
-    )
+    return get_aws_credentials_manager().enrich_path(path, access_token)
 
 
 def get_stat_file_cloud(stat: dict) -> tuple[int, str, str]:

{lamindb_setup-0.71.3 → lamindb_setup-0.72.0}/tests/hub-prod/conftest.py

@@ -1,5 +1,4 @@
 import os
-from typing import TYPE_CHECKING
 from uuid import UUID
 
 import lamindb_setup
@@ -10,6 +9,7 @@ from lamin_utils import logger
 def pytest_sessionstart(session: pytest.Session):
     lamindb_instance_id = UUID("e1a2d3ab762e4592af5a1e53f288284e")
     os.environ["LAMINDB_INSTANCE_ID_INIT"] = lamindb_instance_id.hex
+    assert lamindb_setup.settings.user.handle == "testuser2"
     lamindb_setup.init(
         storage="./default_storage",
         schema="bionty",
@@ -21,4 +21,4 @@ def pytest_sessionstart(session: pytest.Session):
 
 def pytest_sessionfinish(session: pytest.Session):
     logger.set_verbosity(1)
-    lamindb_setup.delete("lamindb-setup-unit-tests", force=True)
+    lamindb_setup.delete("testuser2/lamindb-setup-unit-tests", force=True)