lamindb_setup 0.71.3__py2.py3-none-any.whl → 0.72.0__py2.py3-none-any.whl

lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.71.3"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.72.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

lamindb_setup/_connect_instance.py

@@ -55,7 +55,10 @@ def check_db_dsn_equal_up_to_credentials(db_dsn_hub, db_dsn_local):
 
 
 def update_db_using_local(
-    hub_instance_result: dict[str, str], settings_file: Path, db: str | None = None
+    hub_instance_result: dict[str, str],
+    settings_file: Path,
+    db: str | None = None,
+    raise_permission_error=True,
 ) -> str | None:
     db_updated = None
     # check if postgres
@@ -77,7 +80,11 @@ def update_db_using_local(
                 db_dsn_local = LaminDsnModel(db=isettings.db)
             else:
                 # just take the default hub result and ensure there is actually a user
-                if db_dsn_hub.db.user == "none" and db_dsn_hub.db.password == "none":
+                if (
+                    db_dsn_hub.db.user == "none"
+                    and db_dsn_hub.db.password == "none"
+                    and raise_permission_error
+                ):
                     raise PermissionError(
                         "No database access, please ask your admin to provide you with"
                         " a DB URL and pass it via --db <db_url>"
@@ -101,6 +108,65 @@ def update_db_using_local(
     return db_updated
 
 
+def _connect_instance(
+    owner: str,
+    name: str,
+    *,
+    db: str | None = None,
+    raise_permission_error: bool = True,
+) -> InstanceSettings:
+    settings_file = instance_settings_file(name, owner)
+    make_hub_request = True
+    if settings_file.exists():
+        isettings = load_instance_settings(settings_file)
+        # skip hub request for a purely local instance
+        make_hub_request = isettings.is_remote
+    if make_hub_request:
+        # the following will return a string if the instance does not exist
+        # on the hub
+        hub_result = load_instance_from_hub(owner=owner, name=name)
+        # if hub_result is not a string, it means it made a request
+        # that successfully returned metadata
+        if not isinstance(hub_result, str):
+            instance_result, storage_result = hub_result
+            db_updated = update_db_using_local(
+                instance_result,
+                settings_file,
+                db=db,
+                raise_permission_error=raise_permission_error,
+            )
+            ssettings = StorageSettings(
+                root=storage_result["root"],
+                region=storage_result["region"],
+                uid=storage_result["lnid"],
+                uuid=UUID(storage_result["id"]),
+                instance_id=UUID(instance_result["id"]),
+            )
+            isettings = InstanceSettings(
+                id=UUID(instance_result["id"]),
+                owner=owner,
+                name=name,
+                storage=ssettings,
+                db=db_updated,
+                schema=instance_result["schema_str"],
+                git_repo=instance_result["git_repo"],
+                keep_artifacts_local=bool(instance_result["keep_artifacts_local"]),
+                is_on_hub=True,
+            )
+            check_whether_migrations_in_sync(instance_result["lamindb_version"])
+        else:
+            message = INSTANCE_NOT_FOUND_MESSAGE.format(
+                owner=owner, name=name, hub_result=hub_result
+            )
+            if settings_file.exists():
+                isettings = load_instance_settings(settings_file)
+                if isettings.is_remote:
+                    raise InstanceNotFoundError(message)
+            else:
+                raise InstanceNotFoundError(message)
+    return isettings
+
+
 @unlock_cloud_sqlite_upon_exception(ignore_prev_locker=True)
 def connect(
     slug: str,
@@ -134,71 +200,22 @@ def connect(
         elif settings._instance_exists and f"{owner}/{name}" != settings.instance.slug:
             close_instance(mute=True)
 
-        settings_file = instance_settings_file(name, owner)
-
-        make_hub_request = True
-        if settings_file.exists():
-            isettings = load_instance_settings(settings_file)
-            # mimic instance_result from hub
-            instance_result = {"id": isettings._id.hex}
-            # skip hub request for a purely local instance
-            make_hub_request = isettings.is_remote
-
-        if make_hub_request:
-            # the following will return a string if the instance does not exist
-            # on the hub
-            hub_result = load_instance_from_hub(owner=owner, name=name)
-            # if hub_result is not a string, it means it made a request
-            # that successfully returned metadata
-            if not isinstance(hub_result, str):
-                instance_result, storage_result = hub_result
-                db_updated = update_db_using_local(
-                    instance_result, settings_file, db=db
-                )
-                ssettings = StorageSettings(
-                    root=storage_result["root"],
-                    region=storage_result["region"],
-                    uid=storage_result["lnid"],
-                    uuid=UUID(storage_result["id"]),
-                    instance_id=UUID(instance_result["id"]),
-                )
-                isettings = InstanceSettings(
-                    id=UUID(instance_result["id"]),
-                    owner=owner,
-                    name=name,
-                    storage=ssettings,
-                    db=db_updated,
-                    schema=instance_result["schema_str"],
-                    git_repo=instance_result["git_repo"],
-                    keep_artifacts_local=bool(instance_result["keep_artifacts_local"]),
-                    is_on_hub=True,
-                )
-                check_whether_migrations_in_sync(instance_result["lamindb_version"])
+        try:
+            isettings = _connect_instance(owner, name, db=db)
+        except InstanceNotFoundError as e:
+            if _raise_not_found_error:
+                raise e
             else:
-                message = INSTANCE_NOT_FOUND_MESSAGE.format(
-                    owner=owner, name=name, hub_result=hub_result
-                )
-                if settings_file.exists():
-                    isettings = load_instance_settings(settings_file)
-                    if isettings.is_remote:
-                        if _raise_not_found_error:
-                            raise InstanceNotFoundError(message)
-                        return "instance-not-found"
-
-                else:
-                    if _raise_not_found_error:
-                        raise InstanceNotFoundError(message)
-                    return "instance-not-found"
-
+                return "instance-not-found"
+        if isinstance(isettings, str):
+            return isettings
         if storage is not None:
             update_isettings_with_storage(isettings, storage)
         isettings._persist()
         if _test:
             return None
         silence_loggers()
-        check, msg = isettings._load_db(
-            do_not_lock_for_laminapp_admin=True
-        )  # this also updates local SQLite
+        check, msg = isettings._load_db()
         if not check:
             local_db = (
                 isettings._is_cloud_sqlite and isettings._sqlite_file_local.exists()
@@ -216,7 +233,7 @@ def connect(
                     f"instance exists with id {isettings._id.hex}, but database is not"
                     " loadable: re-initializing"
                 )
-                return "instance-corrupted-or-deleted", instance_result
+                return "instance-corrupted-or-deleted"
         # this is for testing purposes only
         if _TEST_FAILED_LOAD:
             raise RuntimeError("Technical testing error.")

lamindb_setup/_delete.py

@@ -1,29 +1,24 @@
 from __future__ import annotations
 
 import shutil
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 from uuid import UUID
 
 from lamin_utils import logger
 
-from ._connect_instance import (
-    INSTANCE_NOT_FOUND_MESSAGE,
-    InstanceNotFoundError,
-    get_owner_name_from_identifier,
-)
-from .core._hub_core import connect_instance as load_instance_from_hub
+from ._connect_instance import _connect_instance, get_owner_name_from_identifier
+from .core._aws_credentials import HOSTED_BUCKETS
 from .core._hub_core import delete_instance as delete_instance_on_hub
 from .core._hub_core import get_storage_records_for_instance
 from .core._settings import settings
-from .core._settings_instance import InstanceSettings
-from .core._settings_load import load_instance_settings
 from .core._settings_storage import StorageSettings
-from .core._settings_store import instance_settings_file
-from .core.upath import HOSTED_BUCKETS, check_storage_is_empty
+from .core.upath import check_storage_is_empty
 
 if TYPE_CHECKING:
     from pathlib import Path
 
+    from .core._settings_instance import InstanceSettings
+
 
 def delete_cache(cache_dir: Path):
     if cache_dir is not None and cache_dir.exists():
@@ -67,42 +62,8 @@ def delete(slug: str, force: bool = False, require_empty: bool = True) -> int |
         force: Whether to skip the confirmation prompt.
         require_empty: Whether to check if the instance is empty before deleting.
     """
-    instance_owner, instance_name = get_owner_name_from_identifier(slug)
-    if settings._instance_exists and settings.instance.name == instance_name:
-        isettings = settings.instance
-    else:
-        settings_file = instance_settings_file(instance_name, instance_owner)
-        if not settings_file.exists():
-            hub_result = load_instance_from_hub(
-                owner=instance_owner, name=instance_name
-            )
-            if isinstance(hub_result, str):
-                message = INSTANCE_NOT_FOUND_MESSAGE.format(
-                    owner=instance_owner,
-                    name=instance_name,
-                    hub_result=hub_result,
-                )
-                raise InstanceNotFoundError(message)
-            instance_result, storage_result = hub_result
-            ssettings = StorageSettings(
-                root=storage_result["root"],
-                region=storage_result["region"],
-                uid=storage_result["lnid"],
-                uuid=UUID(storage_result["id"]),
-            )
-            isettings = InstanceSettings(
-                id=UUID(instance_result["id"]),
-                owner=instance_owner,
-                name=instance_name,
-                storage=ssettings,
-                keep_artifacts_local=bool(instance_result["keep_artifacts_local"]),
-                db=instance_result["db"] if "db" in instance_result else None,
-                schema=instance_result["schema_str"],
-                git_repo=instance_result["git_repo"],
-                is_on_hub=True,
-            )
-        else:
-            isettings = load_instance_settings(settings_file)
+    owner, name = get_owner_name_from_identifier(slug)
+    isettings = _connect_instance(owner, name, raise_permission_error=False)
     if isettings.dialect != "sqlite":
         logger.warning(
             f"delete() does not yet affect your Postgres database at {isettings.db}"
@@ -149,11 +110,11 @@ def delete(slug: str, force: bool = False, require_empty: bool = True) -> int |
         for storage_record in storage_records:
             if storage_record["root"] == isettings.storage.root_as_str:
                 continue
+            ssettings = StorageSettings(storage_record["root"])  # type: ignore
             check_storage_is_empty(
-                storage_record["root"],  # type: ignore
+                ssettings.root,  # type: ignore
                 raise_error=require_empty,
             )
-            ssettings = StorageSettings(storage_record["root"])  # type: ignore
             if ssettings._mark_storage_root.exists():
                 ssettings._mark_storage_root.unlink(
                     missing_ok=True  # this is totally weird, but needed on Py3.11

lamindb_setup/_init_instance.py

@@ -44,15 +44,17 @@ def register_storage_in_instance(ssettings: StorageSettings):
 
     from .core.hashing import hash_and_encode_as_b62
 
-    assert ssettings._instance_id is not None
-
+    if ssettings._instance_id is not None:
+        instance_uid = hash_and_encode_as_b62(ssettings._instance_id.hex)[:12]
+    else:
+        instance_uid = None
     # how do we ensure that this function is only called passing
     # the managing instance?
     defaults = {
         "root": ssettings.root_as_str,
         "type": ssettings.type,
         "region": ssettings.region,
-        "instance_uid": hash_and_encode_as_b62(ssettings._instance_id.hex)[:12],
+        "instance_uid": instance_uid,
         "created_by_id": current_user_id(),
     }
     if ssettings._uid is not None:
@@ -67,20 +69,19 @@ def register_storage_in_instance(ssettings: StorageSettings):
 def register_user(usettings):
     from lnschema_core.models import User
 
-    if usettings.handle != "laminapp-admin":
-        try:
-            # need to have try except because of integer primary key migration
-            user, created = User.objects.update_or_create(
-                uid=usettings.uid,
-                defaults={
-                    "handle": usettings.handle,
-                    "name": usettings.name,
-                },
-            )
-        # for users with only read access, except via ProgrammingError
-        # ProgrammingError: permission denied for table lnschema_core_user
-        except (OperationalError, FieldError, ProgrammingError):
-            pass
+    try:
+        # need to have try except because of integer primary key migration
+        user, created = User.objects.update_or_create(
+            uid=usettings.uid,
+            defaults={
+                "handle": usettings.handle,
+                "name": usettings.name,
+            },
+        )
+    # for users with only read access, except via ProgrammingError
+    # ProgrammingError: permission denied for table lnschema_core_user
+    except (OperationalError, FieldError, ProgrammingError):
+        pass
 
 
 def register_user_and_storage_in_instance(isettings: InstanceSettings, usettings):

lamindb_setup/_set_managed_storage.py

@@ -31,6 +31,10 @@ def set_managed_storage(root: UPathStr, **fs_kwargs):
     ssettings = init_storage(
         root=root, instance_id=settings.instance._id, register_hub=True
     )
+    if ssettings._instance_id is None:
+        raise ValueError(
+            f"Cannot manage storage without write access: {ssettings.root}"
+        )
     settings.instance._storage = ssettings
     settings.instance._persist()  # this also updates the settings object
     register_storage_in_instance(ssettings)

lamindb_setup/core/_aws_credentials.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import os
+import time
+
+from upath.implementations.cloud import S3Path
+
+HOSTED_REGIONS = [
+    "eu-central-1",
+    "eu-west-2",
+    "us-east-1",
+    "us-east-2",
+    "us-west-1",
+    "us-west-2",
+]
+lamin_env = os.getenv("LAMIN_ENV")
+if lamin_env is None or lamin_env == "prod":
+    hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
+    hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
+    HOSTED_BUCKETS = tuple(hosted_buckets_list)
+else:
+    HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
+
+
+AWS_CREDENTIALS_EXPIRATION = 11 * 60 * 60  # refresh credentials after 11 hours
+
+
+class AWSCredentialsManager:
+    def __init__(self):
+        self._credentials_cache = {}
+
+        from s3fs import S3FileSystem
+
+        # this is cached so will be resued with the connection initialized
+        fs = S3FileSystem(cache_regions=True)
+        fs.connect()
+        self.anon = fs.session._credentials is None
+
+    def _find_root(self, path_str: str) -> str | None:
+        roots = self._credentials_cache.keys()
+        if path_str in roots:
+            return path_str
+        roots = sorted(roots, key=len, reverse=True)
+        for root in roots:
+            if path_str.startswith(root):
+                return root
+        return None
+
+    def _is_active(self, root: str) -> bool:
+        return (
+            time.time() - self._credentials_cache[root]["time"]
+        ) < AWS_CREDENTIALS_EXPIRATION
+
+    def _set_cached_credentials(self, root: str, credentials: dict):
+        if root not in self._credentials_cache:
+            self._credentials_cache[root] = {}
+        self._credentials_cache[root]["credentials"] = credentials
+        self._credentials_cache[root]["time"] = time.time()
+
+    def _get_cached_credentials(self, root: str) -> dict:
+        return self._credentials_cache[root]["credentials"]
+
+    def _path_inject_options(self, path: S3Path, credentials: dict) -> S3Path:
+        if credentials == {}:
+            # credentials were specified manually for the path
+            if "anon" in path._kwargs:
+                anon = path._kwargs["anon"]
+            elif path.fs.key is not None and path.fs.secret is not None:
+                anon = False
+            else:
+                anon = self.anon
+            connection_options = {"anon": anon}
+        else:
+            connection_options = credentials
+
+        if "cache_regions" in path._kwargs:
+            cache_regions = path._kwargs["cache_regions"]
+        else:
+            cache_regions = True
+
+        return S3Path(path, cache_regions=cache_regions, **connection_options)
+
+    def enrich_path(self, path: S3Path, access_token: str | None = None) -> S3Path:
+        path_str = path.as_posix().rstrip("/")
+        root = self._find_root(path_str)
+
+        if root is not None:
+            set_cache = False
+            credentials = self._get_cached_credentials(root)
+
+            if access_token is not None:
+                set_cache = True
+            elif credentials != {}:
+                # update credentials
+                if not self._is_active(root):
+                    set_cache = True
+        else:
+            set_cache = True
+
+        if set_cache:
+            from ._hub_core import access_aws
+
+            storage_root_info = access_aws(path_str, access_token=access_token)
+            accessibility = storage_root_info["accessibility"]
+
+            is_managed = accessibility.get("is_managed", False)
+            if is_managed:
+                credentials = storage_root_info["credentials"]
+            else:
+                credentials = {}
+
+            if access_token is None:
+                if "storage_root" in accessibility:
+                    root = accessibility["storage_root"]
+                # just to be safe
+                root = None if root == "" else root
+                if root is None:
+                    # heuristic
+                    # do not write the first level for the known hosted buckets
+                    if path_str.startswith(HOSTED_BUCKETS):
+                        root = "/".join(path.path.rstrip("/").split("/")[:2])
+                    else:
+                        # write the bucket for everything else
+                        root = path._url.netloc
+                    root = "s3://" + root
+                self._set_cached_credentials(root, credentials)
+
+        return self._path_inject_options(path, credentials)
+
+
+_aws_credentials_manager: AWSCredentialsManager | None = None
+
+
+def get_aws_credentials_manager() -> AWSCredentialsManager:
+    global _aws_credentials_manager
+
+    if _aws_credentials_manager is None:
+        _aws_credentials_manager = AWSCredentialsManager()
+
+    return _aws_credentials_manager

lamindb_setup/core/_hub_core.py

@@ -284,9 +284,11 @@ def _init_instance(isettings: InstanceSettings, client: Client) -> None:
     # as then init_instance is no longer idempotent
     try:
         client.table("instance").insert(fields, returning="minimal").execute()
-    except APIError as e:
-        logger.warning("instance likely already exists")
-        raise e
+    except APIError:
+        logger.warning(
+            f"instance already existed at: https://lamin.ai/{isettings.owner}/{isettings.name}"
+        )
+        return None
     client.table("storage").update(
         {"instance_id": isettings._id.hex, "is_default": True}
     ).eq("id", isettings.storage._uuid.hex).execute()  # type: ignore
@@ -351,35 +353,43 @@ def _connect_instance(
     return instance, storage
 
 
-def access_aws(storage_root: str, access_token: str | None = None) -> dict[str, str]:
+def access_aws(storage_root: str, access_token: str | None = None) -> dict[str, dict]:
     from ._settings import settings
 
     if settings.user.handle != "anonymous" or access_token is not None:
-        credentials = call_with_fallback_auth(
+        storage_root_info = call_with_fallback_auth(
             _access_aws, storage_root=storage_root, access_token=access_token
         )
-        return credentials
+        return storage_root_info
     else:
         raise RuntimeError("Can only get access to AWS if authenticated.")
 
 
-def _access_aws(*, storage_root: str, client: Client) -> dict[str, str]:
+def _access_aws(*, storage_root: str, client: Client) -> dict[str, dict]:
     import lamindb_setup
 
+    storage_root_info: dict[str, dict] = {"credentials": {}, "accessibility": {}}
     response = client.functions.invoke(
         "access-aws",
         invoke_options={"body": {"storage_root": storage_root}},
     )
     if response is not None and response != b"{}":
-        loaded_credentials = json.loads(response)["Credentials"]
-        credentials = {}
+        data = json.loads(response)
+
+        loaded_credentials = data["Credentials"]
+        loaded_accessibility = data["StorageAccessibility"]
+
+        credentials = storage_root_info["credentials"]
         credentials["key"] = loaded_credentials["AccessKeyId"]
         credentials["secret"] = loaded_credentials["SecretAccessKey"]
         credentials["token"] = loaded_credentials["SessionToken"]
-        return credentials
+
+        accessibility = storage_root_info["accessibility"]
+        accessibility["storage_root"] = loaded_accessibility["storageRoot"]
+        accessibility["is_managed"] = loaded_accessibility["isManaged"]
     elif lamindb_setup._TESTING:
         raise RuntimeError(f"access-aws errored: {response}")
-    return {}
+    return storage_root_info
 
 
 def get_lamin_site_base_url():

lamindb_setup/core/_settings_instance.py

@@ -42,7 +42,7 @@ class InstanceSettings:
         db: str | None = None,  # DB URI
         schema: str | None = None,  # comma-separated string of schema names
         git_repo: str | None = None,  # a git repo URL
-        is_on_hub: bool = False,  # initialized from hub
+        is_on_hub: bool | None = None,  # initialized from hub
     ):
         from ._hub_utils import validate_db_arg
 
@@ -367,17 +367,25 @@ class InstanceSettings:
 
     @property
     def is_on_hub(self) -> bool:
-        """Is this instance on the hub.
+        """Is this instance on the hub?
 
-        Only works if user has access to the instance.
+        Can only reliably establish if user has access to the instance. Will
+        return `False` in case the instance isn't found.
         """
         if self._is_on_hub is None:
             from ._hub_client import call_with_fallback_auth
             from ._hub_crud import select_instance_by_id
+            from ._settings import settings
 
-            response = call_with_fallback_auth(
-                select_instance_by_id, instance_id=self._id.hex
-            )
+            if settings.user.handle != "anonymous":
+                response = call_with_fallback_auth(
+                    select_instance_by_id, instance_id=self._id.hex
+                )
+            else:
+                response = call_with_fallback(
+                    select_instance_by_id, instance_id=self._id.hex
+                )
+                logger.warning("calling anonymously, will miss private instances")
             if response is None:
                 self._is_on_hub = False
             else:
@@ -406,9 +414,7 @@ class InstanceSettings:
 
         setup_django(self, init=True)
 
-    def _load_db(
-        self, do_not_lock_for_laminapp_admin: bool = False
-    ) -> tuple[bool, str]:
+    def _load_db(self) -> tuple[bool, str]:
         # Is the database available and initialized as LaminDB?
         # returns a tuple of status code and message
         if self.dialect == "sqlite" and not self._sqlite_file.exists():
@@ -423,15 +429,8 @@ class InstanceSettings:
 
         from .django import setup_django
 
-        # lock in all cases except if do_not_lock_for_laminapp_admin is True and
-        # user is `laminapp-admin`
-        # value doesn't matter if not a cloud sqlite instance
-        lock_cloud_sqlite = self._is_cloud_sqlite and (
-            not do_not_lock_for_laminapp_admin
-            or settings.user.handle != "laminapp-admin"
-        )
         # we need the local sqlite to setup django
-        self._update_local_sqlite_file(lock_cloud_sqlite=lock_cloud_sqlite)
+        self._update_local_sqlite_file(lock_cloud_sqlite=self._is_cloud_sqlite)
         # setting up django also performs a check for migrations & prints them
         # as warnings
         # this should fail, e.g., if the db is not reachable

lamindb_setup/core/_settings_storage.py

@@ -10,11 +10,11 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, Union
 from appdirs import AppDirs
 from lamin_utils import logger
 
+from ._aws_credentials import HOSTED_REGIONS, get_aws_credentials_manager
 from ._aws_storage import find_closest_aws_region
 from ._settings_save import save_system_storage_settings
 from ._settings_store import system_storage_settings_file
 from .upath import (
-    HOSTED_REGIONS,
     LocalPathClasses,
     UPath,
     convert_pathlike,
@@ -113,11 +113,20 @@ def init_storage(
     )
     # the below might update the uid with one that's already taken on the hub
     if ssettings.type_is_cloud or register_hub:
+        from ._hub_core import delete_storage_record
         from ._hub_core import init_storage as init_storage_hub
 
         init_storage_hub(ssettings)
     # below comes last only if everything else was successful
-    mark_storage_root(ssettings.root, ssettings.uid)  # type: ignore
+    try:
+        mark_storage_root(ssettings.root, ssettings.uid)  # type: ignore
+    except Exception:
+        logger.important(
+            f"due to lack of write access, LaminDB won't manage storage location: {ssettings.root}"
+        )
+        if ssettings._uuid is not None:
+            delete_storage_record(ssettings._uuid)  # type: ignore
+        ssettings._instance_id = None
     return ssettings
 
 
@@ -142,6 +151,7 @@ class StorageSettings:
         uid: str | None = None,
         uuid: UUID | None = None,
         instance_id: UUID | None = None,
+        # note that passing access_token prevents credentials caching
         access_token: str | None = None,
     ):
         self._uid = uid
@@ -226,6 +236,12 @@ class StorageSettings:
         if self._root is None:
             # below makes network requests to get credentials
             self._root = create_path(self._root_init, access_token=self.access_token)
+        elif getattr(self._root, "protocol", "") == "s3":
+            # this is needed to be sure that the root always has nonexpired credentials
+            # this just checks for time of the cached credentials in most cases
+            return get_aws_credentials_manager().enrich_path(
+                self._root, access_token=self.access_token
+            )
         return self._root
 
     def _set_fs_kwargs(self, **kwargs):

lamindb_setup/core/hashing.py

@@ -40,11 +40,11 @@ def hash_set(s: set[str]) -> str:
     return to_b64_str(hashlib.md5(bstr).digest())[:20]
 
 
-def hash_md5s_from_dir(etags: list[str]) -> tuple[str, str]:
+def hash_md5s_from_dir(hashes: list[str]) -> tuple[str, str]:
     # need to sort below because we don't want the order of parsing the dir to
     # affect the hash
     digests = b"".join(
-        hashlib.md5(etag.encode("utf-8")).digest() for etag in sorted(etags)
+        hashlib.md5(hash.encode("utf-8")).digest() for hash in sorted(hashes)
     )
     digest = hashlib.md5(digests).digest()
     return to_b64_str(digest)[:22], "md5-d"
@@ -59,24 +59,27 @@ def hash_code(file_path: UPathStr):
     return hashlib.sha1(blob)
 
 
-def hash_file(file_path: Path, chunk_size=50 * 1024 * 1024) -> tuple[str, str]:
-    chunks = []
+def hash_file(
+    file_path: Path,
+    file_size: int | None = None,
+    chunk_size: int | None = 50 * 1024 * 1024,
+) -> tuple[str, str]:
     with open(file_path, "rb") as fp:
-        # read first chunk
-        chunks = [fp.read(chunk_size)]
-        # try reading the 2nd chunk
-        data = fp.read(chunk_size)
-        if data:
-            # go to end of file
+        if file_size is None:
+            fp.seek(0, 2)
+            file_size = fp.tell()
+            fp.seek(0, 0)
+        if chunk_size is None:
+            chunk_size = file_size
+        first_chunk = fp.read(chunk_size)
+        if file_size <= chunk_size:
+            digest = hashlib.md5(first_chunk).digest()
+            hash_type = "md5"
+        else:
             fp.seek(-chunk_size, 2)
-            # read last chunk
-            data = fp.read(chunk_size)
-            chunks.append(data)
-    if len(chunks) == 1:
-        digest = hashlib.md5(chunks[0]).digest()
-        hash_type = "md5"
-    else:
-        digests = b"".join(hashlib.sha1(chunk).digest() for chunk in chunks)
-        digest = hashlib.sha1(digests).digest()
-        hash_type = "sha1-fl"  # sha1 first last chunk
+            last_chunk = fp.read(chunk_size)
+            digest = hashlib.sha1(
+                hashlib.sha1(first_chunk).digest() + hashlib.sha1(last_chunk).digest()
+            ).digest()
+            hash_type = "sha1-fl"
     return to_b64_str(digest)[:22], hash_type

lamindb_setup/core/upath.py

@@ -11,13 +11,13 @@ from itertools import islice
 from pathlib import Path, PurePosixPath
 from typing import TYPE_CHECKING, Any, Literal
 
-import botocore.session
 import fsspec
 from lamin_utils import logger
 from upath import UPath
 from upath.implementations.cloud import CloudPath, S3Path  # keep CloudPath!
 from upath.implementations.local import LocalPath, PosixUPath, WindowsUPath
 
+from ._aws_credentials import HOSTED_BUCKETS, get_aws_credentials_manager
 from .hashing import b16_to_b64, hash_md5s_from_dir
 
 if TYPE_CHECKING:
@@ -158,7 +158,8 @@ def print_hook(size: int, value: int, objectname: str, action: str):
     progress_in_percent = (value / size) * 100
     out = f"... {action} {objectname}:" f" {min(progress_in_percent, 100):4.1f}%"
     if "NBPRJ_TEST_NBPATH" not in os.environ:
-        print(out, end="\r")
+        end = "\n" if progress_in_percent >= 100 else "\r"
+        print(out, end=end)
 
 
 class ProgressCallback(fsspec.callbacks.Callback):
@@ -670,77 +671,12 @@ def convert_pathlike(pathlike: UPathStr) -> UPath:
     return path
 
 
-HOSTED_REGIONS = [
-    "eu-central-1",
-    "eu-west-2",
-    "us-east-1",
-    "us-east-2",
-    "us-west-1",
-    "us-west-2",
-]
-lamin_env = os.getenv("LAMIN_ENV")
-if lamin_env is None or lamin_env == "prod":
-    hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
-    hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
-    HOSTED_BUCKETS = tuple(hosted_buckets_list)
-else:
-    HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
-credentials_cache: dict[str, dict[str, str]] = {}
-AWS_CREDENTIALS_PRESENT = None
-
-
 def create_path(path: UPath, access_token: str | None = None) -> UPath:
     path = convert_pathlike(path)
     # test whether we have an AWS S3 path
     if not isinstance(path, S3Path):
         return path
-    # test whether AWS credentials are present
-    global AWS_CREDENTIALS_PRESENT
-
-    if AWS_CREDENTIALS_PRESENT is not None:
-        anon = AWS_CREDENTIALS_PRESENT
-    else:
-        if path.fs.key is not None and path.fs.secret is not None:
-            anon = False
-        else:
-            # we could do path.fs.connect()
-            # and check path.fs.session._credentials, but it'd be slower
-            session = botocore.session.get_session()
-            credentials = session.get_credentials()
-            if credentials is None or credentials.access_key is None:
-                anon = True
-            else:
-                anon = False
-            # cache credentials and reuse further
-            AWS_CREDENTIALS_PRESENT = anon
-
-    # test whether we are on hosted storage or not
-    path_str = path.as_posix()
-    is_hosted_storage = path_str.startswith(HOSTED_BUCKETS)
-
-    if not is_hosted_storage:
-        # make anon request if no credentials present
-        return UPath(path, cache_regions=True, anon=anon)
-
-    root_folder = "/".join(path_str.replace("s3://", "").split("/")[:2])
-
-    if access_token is None and root_folder in credentials_cache:
-        credentials = credentials_cache[root_folder]
-    else:
-        from ._hub_core import access_aws
-
-        credentials = access_aws(
-            storage_root=f"s3://{root_folder}", access_token=access_token
-        )
-        if access_token is None:
-            credentials_cache[root_folder] = credentials
-
-    return UPath(
-        path,
-        key=credentials["key"],
-        secret=credentials["secret"],
-        token=credentials["token"],
-    )
+    return get_aws_credentials_manager().enrich_path(path, access_token)
 
 
 def get_stat_file_cloud(stat: dict) -> tuple[int, str, str]:

{lamindb_setup-0.71.3.dist-info → lamindb_setup-0.72.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.71.3
+Version: 0.72.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <laminlabs@gmail.com>
 Description-Content-Type: text/markdown

{lamindb_setup-0.71.3.dist-info → lamindb_setup-0.72.0.dist-info}/RECORD

@@ -1,43 +1,44 @@
-lamindb_setup/__init__.py,sha256=4OkxL3f7ZblC9iBXMfusjWga8XsOaieSAMix4ZDG8LE,1542
+lamindb_setup/__init__.py,sha256=4RLRB4zcYMk9ztnkPkCMOiGJ1hmqyhLXUsSRwtN932I,1542
 lamindb_setup/_cache.py,sha256=wA7mbysANwe8hPNbjDo9bOmXJ0xIyaS5iyxIpxSWji4,846
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=cNEL9Q4yPpmEkGKHH8JgullWl1VUZwALJ4RHn9wZypY,2613
 lamindb_setup/_close.py,sha256=1QS9p2SCacgovYn6xqWU4zFvwHN1RgIccvzwJgFvKgU,1186
-lamindb_setup/_connect_instance.py,sha256=6DiKFpw0k_gcWlZxI8ZJoqHungW-el3JbaWcmV9wGIg,12600
-lamindb_setup/_delete.py,sha256=hf8zfVJfW74QR7eK4xJNQ6HbkkZBsl5eTqj-Ni5jPo0,7232
+lamindb_setup/_connect_instance.py,sha256=C4WURprypLpvS1oJmfyXHvXCMPc7yOOvKmOabyFr7dE,12728
+lamindb_setup/_delete.py,sha256=Y8KSFYgY0UHAvjd7cCL6hZ_XiLeJwx50BguVATcj_Xo,5524
 lamindb_setup/_django.py,sha256=EoyWvFzH0i9wxjy4JZhcoXCTckztP_Mrl6FbYQnMmLE,1534
 lamindb_setup/_exportdb.py,sha256=uTIZjKKTB7arzEr1j0O6lONiT2pRBKeOFdLvOV8ZwzE,2120
 lamindb_setup/_importdb.py,sha256=yYYShzUajTsR-cTW4CZ-UNDWZY2uE5PAgNbp-wn8Ogc,1874
-lamindb_setup/_init_instance.py,sha256=2NT4ERk0RFrcgpnH5O-sAwkd7E1QLIOaFavyFzdAZbc,11924
+lamindb_setup/_init_instance.py,sha256=lR-6txbf3Z2O7ki-DMZWFg36QNiUZ_B5lc6JXjScdus,11897
 lamindb_setup/_migrate.py,sha256=4nBTFg5-BK4A2gH-D3_tcFf8EtvMnIo5Mq0e_C6_9-U,8815
 lamindb_setup/_register_instance.py,sha256=Jeu0wyvJVSVQ_n-A_7yn7xOZIP0ncJD92DRABqzPIjA,940
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
-lamindb_setup/_set_managed_storage.py,sha256=BUUJzKNWNEA5KnKnFZsas0ANU6w-LBZL-CKRu-sNLPE,1268
+lamindb_setup/_set_managed_storage.py,sha256=mNZrANn-9rwZ0oGWxxg0wS0T0VOQCWyo4nSSyNAE15Q,1419
 lamindb_setup/_setup_user.py,sha256=6Oc7Rke-yRQSZbuntdUAz8QbJ6UuPzYHI9FnYlf_q-A,3670
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
 lamindb_setup/core/__init__.py,sha256=dV9S-rQpNK9JcBn4hiEmiLnmNqfpPFJD9pqagMCaIew,416
+lamindb_setup/core/_aws_credentials.py,sha256=0vDs7pWFguBE5UHXf4ysvsjieWygye-Akig6E2Ewl-Q,4749
 lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
 lamindb_setup/core/_deprecated.py,sha256=3qxUI1dnDlSeR0BYrv7ucjqRBEojbqotPgpShXs4KF8,2520
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
 lamindb_setup/core/_hub_client.py,sha256=V0qKDsCdRn-tQy2YIk4VgXcpJFmuum6N3gcavAC7gBQ,5504
-lamindb_setup/core/_hub_core.py,sha256=Cp8w0QY3DNOGWOhzzWmElj-pUm6tAkrQCLikbq_ksq8,15971
+lamindb_setup/core/_hub_core.py,sha256=GSPPwBQlqg1l3TfzoaDulcG8u_HAjeqqBgGUJQVY4HQ,16447
 lamindb_setup/core/_hub_crud.py,sha256=b1XF7AJpM9Q-ttm9nPG-r3OTRWHQaGzAGIyvmb83NTo,4859
 lamindb_setup/core/_hub_utils.py,sha256=b_M1LkdCjiMWm1EOlSb9GuPdLijwVgQDtATTpeZuXI0,1875
 lamindb_setup/core/_settings.py,sha256=jjZ_AxRXB3Y3UP6m04BAw_dhFbJbdg2-nZWmEv2LNZ8,3141
-lamindb_setup/core/_settings_instance.py,sha256=RFUcnBBUp303dbVEHcAaIm_q7lzlWg56OrKLwdam8Pg,16588
+lamindb_setup/core/_settings_instance.py,sha256=w5SBrp6nIJMegzNJSnfQl8HvqEtwgmR7OrayffVedLc,16612
 lamindb_setup/core/_settings_load.py,sha256=NGgCDpN85j1EqoKlrYFIlZBMlBJm33gx2-wc96CP_ZQ,3922
 lamindb_setup/core/_settings_save.py,sha256=d1A-Ex-7H08mb8l7I0Oe0j0GilrfaDuprh_NMxhQAsQ,2704
-lamindb_setup/core/_settings_storage.py,sha256=VgsqdIImQRfOZ6FGNY6DLVohaSxerj_F-sWtjD9hzcs,12382
+lamindb_setup/core/_settings_storage.py,sha256=7f0jt1zcSltpOYDPQ5CVvbBon_d7aneKTte935-2REY,13236
 lamindb_setup/core/_settings_store.py,sha256=dagS5c7wAMRnuZTRfCU4sKaIOyF_HwAP5Fnnn8vphno,2084
 lamindb_setup/core/_settings_user.py,sha256=P2lC4WDRAFfT-Xq3MlXJ-wMKIHCoGNhMTQfRGIAyUNQ,1344
 lamindb_setup/core/_setup_bionty_sources.py,sha256=OgPpZxN2_Wffy-ogEBz_97c_k8d2bD-DDVt89-u9GLY,3002
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=NIBNAGq7TTRrip9OzMdiQKj8QOuwhL9esyM0aehUqBA,6893
 lamindb_setup/core/django.py,sha256=QUQm3zt5QIiD8uv6o9vbSm_bshqiSWzKSkgD3z2eJCg,3542
 lamindb_setup/core/exceptions.py,sha256=eoI7AXgATgDVzgArtN7CUvpaMUC067vsBg5LHCsWzDM,305
-lamindb_setup/core/hashing.py,sha256=mv9UCvAsSrdHYQAv3Kz7UOvjd5tIjvDTIYv_ettBuVY,2218
+lamindb_setup/core/hashing.py,sha256=7r96h5JBzuwfOR_gNNqTyWNPKMuiOUfBYwn6sCbZkf8,2269
 lamindb_setup/core/types.py,sha256=bcYnZ0uM_2NXKJCl94Mmc-uYrQlRUUVKG3sK2N-F-N4,532
-lamindb_setup/core/upath.py,sha256=mou3NdF-tRTLW3E4eNDRFdCZrFZUyt-NPOVyC-DLe24,28342
-lamindb_setup-0.71.3.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-0.71.3.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
-lamindb_setup-0.71.3.dist-info/METADATA,sha256=fAgts1w5BrUmsVUYnGoAmYdT7oljBvhdLmg_1fMicoo,1620
-lamindb_setup-0.71.3.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=QnAiaOZgT1TLUaX0PEs9dSJ0E4ZDD431hCfKrJIbmqQ,26339
+lamindb_setup-0.72.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-0.72.0.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
+lamindb_setup-0.72.0.dist-info/METADATA,sha256=8Bp06EbsyZPsRjXGPGb0Le95RfKbQXlMsf-4sFUgESM,1620
+lamindb_setup-0.72.0.dist-info/RECORD,,