lamindb_setup 0.71.4__py2.py3-none-any.whl → 0.72.0__py2.py3-none-any.whl

lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.71.4"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.72.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

lamindb_setup/_delete.py

@@ -7,11 +7,12 @@ from uuid import UUID
 from lamin_utils import logger
 
 from ._connect_instance import _connect_instance, get_owner_name_from_identifier
+from .core._aws_credentials import HOSTED_BUCKETS
 from .core._hub_core import delete_instance as delete_instance_on_hub
 from .core._hub_core import get_storage_records_for_instance
 from .core._settings import settings
 from .core._settings_storage import StorageSettings
-from .core.upath import HOSTED_BUCKETS, check_storage_is_empty
+from .core.upath import check_storage_is_empty
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -109,11 +110,11 @@ def delete(slug: str, force: bool = False, require_empty: bool = True) -> int |
         for storage_record in storage_records:
             if storage_record["root"] == isettings.storage.root_as_str:
                 continue
+            ssettings = StorageSettings(storage_record["root"])  # type: ignore
             check_storage_is_empty(
-                storage_record["root"],  # type: ignore
+                ssettings.root,  # type: ignore
                 raise_error=require_empty,
             )
-            ssettings = StorageSettings(storage_record["root"])  # type: ignore
             if ssettings._mark_storage_root.exists():
                 ssettings._mark_storage_root.unlink(
                     missing_ok=True  # this is totally weird, but needed on Py3.11

lamindb_setup/core/_aws_credentials.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import os
+import time
+
+from upath.implementations.cloud import S3Path
+
+HOSTED_REGIONS = [
+    "eu-central-1",
+    "eu-west-2",
+    "us-east-1",
+    "us-east-2",
+    "us-west-1",
+    "us-west-2",
+]
+lamin_env = os.getenv("LAMIN_ENV")
+if lamin_env is None or lamin_env == "prod":
+    hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
+    hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
+    HOSTED_BUCKETS = tuple(hosted_buckets_list)
+else:
+    HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
+
+
+AWS_CREDENTIALS_EXPIRATION = 11 * 60 * 60  # refresh credentials after 11 hours
+
+
+class AWSCredentialsManager:
+    def __init__(self):
+        self._credentials_cache = {}
+
+        from s3fs import S3FileSystem
+
+        # this is cached so will be resued with the connection initialized
+        fs = S3FileSystem(cache_regions=True)
+        fs.connect()
+        self.anon = fs.session._credentials is None
+
+    def _find_root(self, path_str: str) -> str | None:
+        roots = self._credentials_cache.keys()
+        if path_str in roots:
+            return path_str
+        roots = sorted(roots, key=len, reverse=True)
+        for root in roots:
+            if path_str.startswith(root):
+                return root
+        return None
+
+    def _is_active(self, root: str) -> bool:
+        return (
+            time.time() - self._credentials_cache[root]["time"]
+        ) < AWS_CREDENTIALS_EXPIRATION
+
+    def _set_cached_credentials(self, root: str, credentials: dict):
+        if root not in self._credentials_cache:
+            self._credentials_cache[root] = {}
+        self._credentials_cache[root]["credentials"] = credentials
+        self._credentials_cache[root]["time"] = time.time()
+
+    def _get_cached_credentials(self, root: str) -> dict:
+        return self._credentials_cache[root]["credentials"]
+
+    def _path_inject_options(self, path: S3Path, credentials: dict) -> S3Path:
+        if credentials == {}:
+            # credentials were specified manually for the path
+            if "anon" in path._kwargs:
+                anon = path._kwargs["anon"]
+            elif path.fs.key is not None and path.fs.secret is not None:
+                anon = False
+            else:
+                anon = self.anon
+            connection_options = {"anon": anon}
+        else:
+            connection_options = credentials
+
+        if "cache_regions" in path._kwargs:
+            cache_regions = path._kwargs["cache_regions"]
+        else:
+            cache_regions = True
+
+        return S3Path(path, cache_regions=cache_regions, **connection_options)
+
+    def enrich_path(self, path: S3Path, access_token: str | None = None) -> S3Path:
+        path_str = path.as_posix().rstrip("/")
+        root = self._find_root(path_str)
+
+        if root is not None:
+            set_cache = False
+            credentials = self._get_cached_credentials(root)
+
+            if access_token is not None:
+                set_cache = True
+            elif credentials != {}:
+                # update credentials
+                if not self._is_active(root):
+                    set_cache = True
+        else:
+            set_cache = True
+
+        if set_cache:
+            from ._hub_core import access_aws
+
+            storage_root_info = access_aws(path_str, access_token=access_token)
+            accessibility = storage_root_info["accessibility"]
+
+            is_managed = accessibility.get("is_managed", False)
+            if is_managed:
+                credentials = storage_root_info["credentials"]
+            else:
+                credentials = {}
+
+            if access_token is None:
+                if "storage_root" in accessibility:
+                    root = accessibility["storage_root"]
+                # just to be safe
+                root = None if root == "" else root
+                if root is None:
+                    # heuristic
+                    # do not write the first level for the known hosted buckets
+                    if path_str.startswith(HOSTED_BUCKETS):
+                        root = "/".join(path.path.rstrip("/").split("/")[:2])
+                    else:
+                        # write the bucket for everything else
+                        root = path._url.netloc
+                    root = "s3://" + root
+                self._set_cached_credentials(root, credentials)
+
+        return self._path_inject_options(path, credentials)
+
+
+_aws_credentials_manager: AWSCredentialsManager | None = None
+
+
+def get_aws_credentials_manager() -> AWSCredentialsManager:
+    global _aws_credentials_manager
+
+    if _aws_credentials_manager is None:
+        _aws_credentials_manager = AWSCredentialsManager()
+
+    return _aws_credentials_manager

lamindb_setup/core/_hub_core.py

@@ -353,35 +353,43 @@ def _connect_instance(
     return instance, storage
 
 
-def access_aws(storage_root: str, access_token: str | None = None) -> dict[str, str]:
+def access_aws(storage_root: str, access_token: str | None = None) -> dict[str, dict]:
     from ._settings import settings
 
     if settings.user.handle != "anonymous" or access_token is not None:
-        credentials = call_with_fallback_auth(
+        storage_root_info = call_with_fallback_auth(
             _access_aws, storage_root=storage_root, access_token=access_token
         )
-        return credentials
+        return storage_root_info
     else:
         raise RuntimeError("Can only get access to AWS if authenticated.")
 
 
-def _access_aws(*, storage_root: str, client: Client) -> dict[str, str]:
+def _access_aws(*, storage_root: str, client: Client) -> dict[str, dict]:
     import lamindb_setup
 
+    storage_root_info: dict[str, dict] = {"credentials": {}, "accessibility": {}}
     response = client.functions.invoke(
         "access-aws",
         invoke_options={"body": {"storage_root": storage_root}},
     )
     if response is not None and response != b"{}":
-        loaded_credentials = json.loads(response)["Credentials"]
-        credentials = {}
+        data = json.loads(response)
+
+        loaded_credentials = data["Credentials"]
+        loaded_accessibility = data["StorageAccessibility"]
+
+        credentials = storage_root_info["credentials"]
         credentials["key"] = loaded_credentials["AccessKeyId"]
         credentials["secret"] = loaded_credentials["SecretAccessKey"]
         credentials["token"] = loaded_credentials["SessionToken"]
-        return credentials
+
+        accessibility = storage_root_info["accessibility"]
+        accessibility["storage_root"] = loaded_accessibility["storageRoot"]
+        accessibility["is_managed"] = loaded_accessibility["isManaged"]
     elif lamindb_setup._TESTING:
         raise RuntimeError(f"access-aws errored: {response}")
-    return {}
+    return storage_root_info
 
 
 def get_lamin_site_base_url():

lamindb_setup/core/_settings_instance.py

@@ -367,17 +367,25 @@ class InstanceSettings:
 
     @property
     def is_on_hub(self) -> bool:
-        """Is this instance on the hub.
+        """Is this instance on the hub?
 
-        Only works if user has access to the instance.
+        Can only reliably establish if user has access to the instance. Will
+        return `False` in case the instance isn't found.
         """
         if self._is_on_hub is None:
             from ._hub_client import call_with_fallback_auth
             from ._hub_crud import select_instance_by_id
+            from ._settings import settings
 
-            response = call_with_fallback_auth(
-                select_instance_by_id, instance_id=self._id.hex
-            )
+            if settings.user.handle != "anonymous":
+                response = call_with_fallback_auth(
+                    select_instance_by_id, instance_id=self._id.hex
+                )
+            else:
+                response = call_with_fallback(
+                    select_instance_by_id, instance_id=self._id.hex
+                )
+                logger.warning("calling anonymously, will miss private instances")
             if response is None:
                 self._is_on_hub = False
             else:

lamindb_setup/core/_settings_storage.py

@@ -10,11 +10,11 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, Union
 from appdirs import AppDirs
 from lamin_utils import logger
 
+from ._aws_credentials import HOSTED_REGIONS, get_aws_credentials_manager
 from ._aws_storage import find_closest_aws_region
 from ._settings_save import save_system_storage_settings
 from ._settings_store import system_storage_settings_file
 from .upath import (
-    HOSTED_REGIONS,
     LocalPathClasses,
     UPath,
     convert_pathlike,
@@ -151,6 +151,7 @@ class StorageSettings:
         uid: str | None = None,
         uuid: UUID | None = None,
         instance_id: UUID | None = None,
+        # note that passing access_token prevents credentials caching
         access_token: str | None = None,
     ):
         self._uid = uid
@@ -235,6 +236,12 @@ class StorageSettings:
         if self._root is None:
             # below makes network requests to get credentials
             self._root = create_path(self._root_init, access_token=self.access_token)
+        elif getattr(self._root, "protocol", "") == "s3":
+            # this is needed to be sure that the root always has nonexpired credentials
+            # this just checks for time of the cached credentials in most cases
+            return get_aws_credentials_manager().enrich_path(
+                self._root, access_token=self.access_token
+            )
         return self._root
 
     def _set_fs_kwargs(self, **kwargs):

lamindb_setup/core/upath.py

@@ -11,13 +11,13 @@ from itertools import islice
 from pathlib import Path, PurePosixPath
 from typing import TYPE_CHECKING, Any, Literal
 
-import botocore.session
 import fsspec
 from lamin_utils import logger
 from upath import UPath
 from upath.implementations.cloud import CloudPath, S3Path  # keep CloudPath!
 from upath.implementations.local import LocalPath, PosixUPath, WindowsUPath
 
+from ._aws_credentials import HOSTED_BUCKETS, get_aws_credentials_manager
 from .hashing import b16_to_b64, hash_md5s_from_dir
 
 if TYPE_CHECKING:
@@ -158,7 +158,8 @@ def print_hook(size: int, value: int, objectname: str, action: str):
     progress_in_percent = (value / size) * 100
     out = f"... {action} {objectname}:" f" {min(progress_in_percent, 100):4.1f}%"
     if "NBPRJ_TEST_NBPATH" not in os.environ:
-        print(out, end="\r")
+        end = "\n" if progress_in_percent >= 100 else "\r"
+        print(out, end=end)
 
 
 class ProgressCallback(fsspec.callbacks.Callback):
@@ -670,77 +671,12 @@ def convert_pathlike(pathlike: UPathStr) -> UPath:
     return path
 
 
-HOSTED_REGIONS = [
-    "eu-central-1",
-    "eu-west-2",
-    "us-east-1",
-    "us-east-2",
-    "us-west-1",
-    "us-west-2",
-]
-lamin_env = os.getenv("LAMIN_ENV")
-if lamin_env is None or lamin_env == "prod":
-    hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
-    hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
-    HOSTED_BUCKETS = tuple(hosted_buckets_list)
-else:
-    HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
-credentials_cache: dict[str, dict[str, str]] = {}
-AWS_CREDENTIALS_PRESENT = None
-
-
 def create_path(path: UPath, access_token: str | None = None) -> UPath:
     path = convert_pathlike(path)
     # test whether we have an AWS S3 path
     if not isinstance(path, S3Path):
         return path
-    # test whether AWS credentials are present
-    global AWS_CREDENTIALS_PRESENT
-
-    if AWS_CREDENTIALS_PRESENT is not None:
-        anon = AWS_CREDENTIALS_PRESENT
-    else:
-        if path.fs.key is not None and path.fs.secret is not None:
-            anon = False
-        else:
-            # we could do path.fs.connect()
-            # and check path.fs.session._credentials, but it'd be slower
-            session = botocore.session.get_session()
-            credentials = session.get_credentials()
-            if credentials is None or credentials.access_key is None:
-                anon = True
-            else:
-                anon = False
-            # cache credentials and reuse further
-            AWS_CREDENTIALS_PRESENT = anon
-
-    # test whether we are on hosted storage or not
-    path_str = path.as_posix()
-    is_hosted_storage = path_str.startswith(HOSTED_BUCKETS)
-
-    if not is_hosted_storage:
-        # make anon request if no credentials present
-        return UPath(path, cache_regions=True, anon=anon)
-
-    root_folder = "/".join(path_str.replace("s3://", "").split("/")[:2])
-
-    if access_token is None and root_folder in credentials_cache:
-        credentials = credentials_cache[root_folder]
-    else:
-        from ._hub_core import access_aws
-
-        credentials = access_aws(
-            storage_root=f"s3://{root_folder}", access_token=access_token
-        )
-        if access_token is None:
-            credentials_cache[root_folder] = credentials
-
-    return UPath(
-        path,
-        key=credentials["key"],
-        secret=credentials["secret"],
-        token=credentials["token"],
-    )
+    return get_aws_credentials_manager().enrich_path(path, access_token)
 
 
 def get_stat_file_cloud(stat: dict) -> tuple[int, str, str]:

{lamindb_setup-0.71.4.dist-info → lamindb_setup-0.72.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.71.4
+Version: 0.72.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <laminlabs@gmail.com>
 Description-Content-Type: text/markdown

{lamindb_setup-0.71.4.dist-info → lamindb_setup-0.72.0.dist-info}/RECORD

@@ -1,10 +1,10 @@
-lamindb_setup/__init__.py,sha256=w7Zj50IVdvhD0ctVPpTi0Uh4X2FLlm-YplosftlWzEs,1542
+lamindb_setup/__init__.py,sha256=4RLRB4zcYMk9ztnkPkCMOiGJ1hmqyhLXUsSRwtN932I,1542
 lamindb_setup/_cache.py,sha256=wA7mbysANwe8hPNbjDo9bOmXJ0xIyaS5iyxIpxSWji4,846
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=cNEL9Q4yPpmEkGKHH8JgullWl1VUZwALJ4RHn9wZypY,2613
 lamindb_setup/_close.py,sha256=1QS9p2SCacgovYn6xqWU4zFvwHN1RgIccvzwJgFvKgU,1186
 lamindb_setup/_connect_instance.py,sha256=C4WURprypLpvS1oJmfyXHvXCMPc7yOOvKmOabyFr7dE,12728
-lamindb_setup/_delete.py,sha256=gRqtlGWUaYDYL97kI7EyLvgI23DIaAoVyOx6hF6LPNw,5498
+lamindb_setup/_delete.py,sha256=Y8KSFYgY0UHAvjd7cCL6hZ_XiLeJwx50BguVATcj_Xo,5524
 lamindb_setup/_django.py,sha256=EoyWvFzH0i9wxjy4JZhcoXCTckztP_Mrl6FbYQnMmLE,1534
 lamindb_setup/_exportdb.py,sha256=uTIZjKKTB7arzEr1j0O6lONiT2pRBKeOFdLvOV8ZwzE,2120
 lamindb_setup/_importdb.py,sha256=yYYShzUajTsR-cTW4CZ-UNDWZY2uE5PAgNbp-wn8Ogc,1874
@@ -16,18 +16,19 @@ lamindb_setup/_set_managed_storage.py,sha256=mNZrANn-9rwZ0oGWxxg0wS0T0VOQCWyo4nS
 lamindb_setup/_setup_user.py,sha256=6Oc7Rke-yRQSZbuntdUAz8QbJ6UuPzYHI9FnYlf_q-A,3670
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
 lamindb_setup/core/__init__.py,sha256=dV9S-rQpNK9JcBn4hiEmiLnmNqfpPFJD9pqagMCaIew,416
+lamindb_setup/core/_aws_credentials.py,sha256=0vDs7pWFguBE5UHXf4ysvsjieWygye-Akig6E2Ewl-Q,4749
 lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
 lamindb_setup/core/_deprecated.py,sha256=3qxUI1dnDlSeR0BYrv7ucjqRBEojbqotPgpShXs4KF8,2520
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
 lamindb_setup/core/_hub_client.py,sha256=V0qKDsCdRn-tQy2YIk4VgXcpJFmuum6N3gcavAC7gBQ,5504
-lamindb_setup/core/_hub_core.py,sha256=Jxjf6qyae25NlnrHLqArNaCq1ZjWYuE3pEStbx7jCI0,16043
+lamindb_setup/core/_hub_core.py,sha256=GSPPwBQlqg1l3TfzoaDulcG8u_HAjeqqBgGUJQVY4HQ,16447
 lamindb_setup/core/_hub_crud.py,sha256=b1XF7AJpM9Q-ttm9nPG-r3OTRWHQaGzAGIyvmb83NTo,4859
 lamindb_setup/core/_hub_utils.py,sha256=b_M1LkdCjiMWm1EOlSb9GuPdLijwVgQDtATTpeZuXI0,1875
 lamindb_setup/core/_settings.py,sha256=jjZ_AxRXB3Y3UP6m04BAw_dhFbJbdg2-nZWmEv2LNZ8,3141
-lamindb_setup/core/_settings_instance.py,sha256=PtMg-0ADnvH5bsTUPt7mD87qbOqj9pZE4RujWTR3nw8,16191
+lamindb_setup/core/_settings_instance.py,sha256=w5SBrp6nIJMegzNJSnfQl8HvqEtwgmR7OrayffVedLc,16612
 lamindb_setup/core/_settings_load.py,sha256=NGgCDpN85j1EqoKlrYFIlZBMlBJm33gx2-wc96CP_ZQ,3922
 lamindb_setup/core/_settings_save.py,sha256=d1A-Ex-7H08mb8l7I0Oe0j0GilrfaDuprh_NMxhQAsQ,2704
-lamindb_setup/core/_settings_storage.py,sha256=KnSvMABdGNQYzx6TPVTAprahY23LY1pAfxzYulqtWaw,12751
+lamindb_setup/core/_settings_storage.py,sha256=7f0jt1zcSltpOYDPQ5CVvbBon_d7aneKTte935-2REY,13236
 lamindb_setup/core/_settings_store.py,sha256=dagS5c7wAMRnuZTRfCU4sKaIOyF_HwAP5Fnnn8vphno,2084
 lamindb_setup/core/_settings_user.py,sha256=P2lC4WDRAFfT-Xq3MlXJ-wMKIHCoGNhMTQfRGIAyUNQ,1344
 lamindb_setup/core/_setup_bionty_sources.py,sha256=OgPpZxN2_Wffy-ogEBz_97c_k8d2bD-DDVt89-u9GLY,3002
@@ -36,8 +37,8 @@ lamindb_setup/core/django.py,sha256=QUQm3zt5QIiD8uv6o9vbSm_bshqiSWzKSkgD3z2eJCg,
 lamindb_setup/core/exceptions.py,sha256=eoI7AXgATgDVzgArtN7CUvpaMUC067vsBg5LHCsWzDM,305
 lamindb_setup/core/hashing.py,sha256=7r96h5JBzuwfOR_gNNqTyWNPKMuiOUfBYwn6sCbZkf8,2269
 lamindb_setup/core/types.py,sha256=bcYnZ0uM_2NXKJCl94Mmc-uYrQlRUUVKG3sK2N-F-N4,532
-lamindb_setup/core/upath.py,sha256=mou3NdF-tRTLW3E4eNDRFdCZrFZUyt-NPOVyC-DLe24,28342
-lamindb_setup-0.71.4.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-0.71.4.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
-lamindb_setup-0.71.4.dist-info/METADATA,sha256=ge5_4_y1g-3nbwgsX_3xP11x27UrZGdr9O0wghjrD3s,1620
-lamindb_setup-0.71.4.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=QnAiaOZgT1TLUaX0PEs9dSJ0E4ZDD431hCfKrJIbmqQ,26339
+lamindb_setup-0.72.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-0.72.0.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
+lamindb_setup-0.72.0.dist-info/METADATA,sha256=8Bp06EbsyZPsRjXGPGb0Le95RfKbQXlMsf-4sFUgESM,1620
+lamindb_setup-0.72.0.dist-info/RECORD,,