lamindb_setup 0.79.0__py3-none-any.whl → 0.80.1__py3-none-any.whl

lamindb_setup/__init__.py

@@ -33,7 +33,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.79.0"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.80.1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 import sys as _sys

lamindb_setup/_init_instance.py

@@ -415,9 +415,10 @@ def infer_instance_name(
     if storage == "create-s3":
         raise ValueError("pass name to init if storage = 'create-s3'")
     storage_path = UPath(storage)
+    # not sure if name is ever ""
     if storage_path.name != "":
         name = storage_path.name
     else:
         # dedicated treatment of bucket names
-        name = storage_path._url.netloc
+        name = storage_path.drive
     return name.lower()

lamindb_setup/_setup_user.py

@@ -48,14 +48,14 @@ def load_user(email: str | None = None, handle: str | None = None) -> UserSettin
 
 
 def login(
-    user: str | None = None, *, key: str | None = None, api_key: str | None = None
+    user: str | None = None, *, api_key: str | None = None, key: str | None = None
 ) -> None:
     """Log in user.
 
     Args:
         user: handle or email
-        key: API key
-        api_key: Beta API key
+        api_key: API key
+        key: legacy API key
     """
     if user is None and api_key is None:
         if "LAMIN_API_KEY" in os.environ:

lamindb_setup/core/_aws_credentials.py

@@ -15,9 +15,7 @@ HOSTED_REGIONS = [
 ]
 lamin_env = os.getenv("LAMIN_ENV")
 if lamin_env is None or lamin_env == "prod":
-    hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
-    hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
-    HOSTED_BUCKETS = tuple(hosted_buckets_list)
+    HOSTED_BUCKETS = tuple([f"s3://lamin-{region}" for region in HOSTED_REGIONS])
 else:
     HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
 
@@ -29,6 +27,11 @@ def _keep_trailing_slash(path_str: str):
 AWS_CREDENTIALS_EXPIRATION = 11 * 60 * 60  # refresh credentials after 11 hours
 
 
+# set anon=True for these buckets if credentials fail for a public bucket
+# to be expanded
+PUBLIC_BUCKETS = ("cellxgene-data-public",)
+
+
 class AWSCredentialsManager:
     def __init__(self):
         self._credentials_cache = {}
@@ -38,7 +41,15 @@ class AWSCredentialsManager:
         # this is cached so will be resued with the connection initialized
         fs = S3FileSystem(cache_regions=True)
         fs.connect()
-        self.anon = fs.session._credentials is None
+        self.anon: bool = fs.session._credentials is None
+        self.anon_public: bool | None = None
+        if not self.anon:
+            try:
+                # use lamindata public bucket for this test
+                fs.call_s3("head_bucket", Bucket="lamindata")
+                self.anon_public = False
+            except Exception as e:
+                self.anon_public = isinstance(e, PermissionError)
 
     def _find_root(self, path_str: str) -> str | None:
         roots = self._credentials_cache.keys()
@@ -73,6 +84,8 @@ class AWSCredentialsManager:
                 anon = False
             else:
                 anon = self.anon
+                if not anon and self.anon_public and path.drive in PUBLIC_BUCKETS:
+                    anon = True
             connection_options = {"anon": anon}
         else:
             connection_options = credentials
@@ -132,7 +145,7 @@ class AWSCredentialsManager:
                         root = "/".join(path.path.rstrip("/").split("/")[:2])
                     else:
                         # write the bucket for everything else
-                        root = path._url.netloc
+                        root = path.drive
                     root = "s3://" + root
                 self._set_cached_credentials(_keep_trailing_slash(root), credentials)
 

lamindb_setup/core/_hub_client.py

@@ -10,6 +10,8 @@ from pydantic_settings import BaseSettings
 from supabase import Client, create_client  # type: ignore
 from supabase.lib.client_options import ClientOptions
 
+from ._settings_save import save_user_settings
+
 
 class Connector(BaseSettings):
     url: str
@@ -129,14 +131,17 @@ def call_with_fallback_auth(
 
     for renew_token, fallback_env in [(False, False), (True, False), (False, True)]:
         try:
-            if renew_token:
-                logger.warning(
-                    "renewing expired lamin token: call `lamin login <your-handle>` to avoid this"
-                )
             client = connect_hub_with_auth(
                 renew_token=renew_token, fallback_env=fallback_env
             )
             result = callable(**kwargs, client=client)
+            # we update access_token here
+            # because at this point the call has been successfully resolved
+            if renew_token:
+                from lamindb_setup import settings
+
+                # here settings.user contains an updated access_token
+                save_user_settings(settings.user)
             break
         # we use Exception here as the ways in which the client fails upon 401
         # are not consistent and keep changing

lamindb_setup/core/_settings.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import sys
 from typing import TYPE_CHECKING
 
 from appdirs import AppDirs
@@ -161,8 +162,20 @@ class SetupSettings:
         cache_dir.mkdir(parents=True, exist_ok=True)
         return cache_dir
 
+    @property
+    def paths(self) -> type[SetupPaths]:
+        """Convert cloud paths to lamidb local paths.
+
+        Use `settings.paths.cloud_to_local_no_update`
+        or `settings.paths.cloud_to_local`.
+        """
+        return SetupPaths
+
     def __repr__(self) -> str:
         """Rich string representation."""
+        # do not show current setting representation when building docs
+        if "sphinx" in sys.modules:
+            return object.__repr__(self)
         repr = self.user.__repr__()
         repr += f"\nAuto-connect in Python: {self.auto_connect}\n"
         repr += f"Private Django API: {self.private_django_api}\n"
@@ -174,6 +187,38 @@ class SetupSettings:
         return repr
 
 
+class SetupPaths:
+    """A static class for conversion of cloud paths to lamindb local paths."""
+
+    @staticmethod
+    def cloud_to_local_no_update(
+        filepath: UPathStr, cache_key: str | None = None
+    ) -> UPath:
+        """Local (or local cache) filepath from filepath without synchronization."""
+        # cache_key is ignored if filepath is a string or a local path
+        # ignores a mere string even if it represents a cloud path
+        if isinstance(filepath, UPath) and not isinstance(filepath, LocalPathClasses):
+            # settings is defined further in this file
+            local_filepath = settings.cache_dir / (
+                filepath.path if cache_key is None else cache_key
+            )
+        else:
+            local_filepath = filepath
+        return UPath(local_filepath)
+
+    @staticmethod
+    def cloud_to_local(
+        filepath: UPathStr, cache_key: str | None = None, **kwargs
+    ) -> UPath:
+        """Local (or local cache) filepath from filepath."""
+        # cache_key is ignored in cloud_to_local_no_update if filepath is local or a string
+        local_filepath = SetupPaths.cloud_to_local_no_update(filepath, cache_key)
+        if isinstance(filepath, UPath) and not isinstance(filepath, LocalPathClasses):
+            local_filepath.parent.mkdir(parents=True, exist_ok=True)
+            filepath.synchronize(local_filepath, **kwargs)
+        return local_filepath
+
+
 def get_env_name():
     if "LAMIN_ENV" in os.environ:
         return os.environ["LAMIN_ENV"]

lamindb_setup/core/_settings_storage.py

@@ -322,24 +322,20 @@ class StorageSettings:
         self, filepath: UPathStr, cache_key: str | None = None, **kwargs
     ) -> UPath:
         """Local (or local cache) filepath from filepath."""
-        # cache_key is ignored in cloud_to_local_no_update if filepath is local
-        local_filepath = self.cloud_to_local_no_update(filepath, cache_key)
-        if isinstance(filepath, UPath) and not isinstance(filepath, LocalPathClasses):
-            local_filepath.parent.mkdir(parents=True, exist_ok=True)
-            filepath.synchronize(local_filepath, **kwargs)
-        return local_filepath
+        from lamindb_setup import settings
+
+        return settings.paths.cloud_to_local(
+            filepath=filepath, cache_key=cache_key, **kwargs
+        )
 
     def cloud_to_local_no_update(
         self, filepath: UPathStr, cache_key: str | None = None
     ) -> UPath:
-        # cache_key is ignored if filepath is local
-        if isinstance(filepath, UPath) and not isinstance(filepath, LocalPathClasses):
-            local_filepath = self.cache_dir / (
-                filepath.path if cache_key is None else cache_key
-            )
-        else:
-            local_filepath = filepath
-        return UPath(local_filepath)
+        from lamindb_setup import settings
+
+        return settings.paths.cloud_to_local_no_update(
+            filepath=filepath, cache_key=cache_key
+        )
 
     def key_to_filepath(self, filekey: UPathStr) -> UPath:
         """Cloud or local filepath from filekey."""

lamindb_setup/core/_settings_user.py

@@ -24,9 +24,9 @@ class UserSettings:
     email: str | None = None
     """User email."""
     api_key: str | None = None
-    """Beta API key."""
+    """API key."""
     password: str | None = None
-    """API key or legacy password."""
+    """legacy API key or legacy password."""
     access_token: str | None = None
     """User access token."""
     uid: str = "null"

lamindb_setup/core/upath.py

@@ -8,14 +8,14 @@ from collections import defaultdict
 from datetime import datetime, timezone
 from functools import partial
 from itertools import islice
-from pathlib import Path, PurePosixPath
+from pathlib import Path, PosixPath, PurePosixPath, WindowsPath
 from typing import TYPE_CHECKING, Any, Literal
 
 import fsspec
 from lamin_utils import logger
 from upath import UPath
 from upath.implementations.cloud import CloudPath, S3Path  # keep CloudPath!
-from upath.implementations.local import LocalPath, PosixUPath, WindowsUPath
+from upath.implementations.local import LocalPath
 
 from ._aws_credentials import HOSTED_BUCKETS, get_aws_credentials_manager
 from .hashing import HASH_LENGTH, b16_to_b64, hash_md5s_from_dir
@@ -23,7 +23,7 @@ from .hashing import HASH_LENGTH, b16_to_b64, hash_md5s_from_dir
 if TYPE_CHECKING:
     from .types import UPathStr
 
-LocalPathClasses = (PosixUPath, WindowsUPath, LocalPath)
+LocalPathClasses = (PosixPath, WindowsPath, LocalPath)
 
 # also see https://gist.github.com/securifera/e7eed730cbe1ce43d0c29d7cd2d582f4
 #    ".gz" is not listed here as it typically occurs with another suffix
@@ -291,21 +291,25 @@ def upload_from(
         callback = ProgressCallback(local_path.name, "uploading")
         kwargs["callback"] = callback
 
+    source: str | list[str]
+    destination: str | list[str]
     if local_path_is_dir and not create_folder:
-        source = [f for f in local_path.rglob("*") if f.is_file()]
-        destination = [str(self / f.relative_to(local_path)) for f in source]
-        source = [str(f) for f in source]  # type: ignore
+        source = [f.as_posix() for f in local_path.rglob("*") if f.is_file()]
+        destination = fsspec.utils.other_paths(
+            source, self.as_posix(), exists=False, flatten=False
+        )
     else:
-        source = str(local_path)  # type: ignore
-        destination = str(self)  # type: ignore
+        source = local_path.as_posix()
+        destination = self.as_posix()
 
     # the below lines are to avoid s3fs triggering create_bucket in upload if
     # dirs are present it allows to avoid permission error
     # would be easier to just
     if self.protocol == "s3" and local_path_is_dir and create_folder:
-        bucket = self._url.netloc
+        bucket = self.drive
         if bucket not in self.fs.dircache:
             self.fs.dircache[bucket] = [{}]
+            assert isinstance(destination, str)
             if not destination.endswith(TRAILING_SEP):  # type: ignore
                 destination += "/"
             cleanup_cache = True
@@ -399,8 +403,8 @@ def synchronize(
             callback.set_size(len(files))
             origin_file_keys = []
             for file, stat in callback.wrap(files.items()):
-                file_key = PurePosixPath(file).relative_to(self.path)
-                origin_file_keys.append(file_key.as_posix())
+                file_key = PurePosixPath(file).relative_to(self.path).as_posix()
+                origin_file_keys.append(file_key)
                 timestamp = stat[modified_key].timestamp()
 
                 origin = f"{self.protocol}://{file}"
@@ -615,11 +619,11 @@ def to_url(upath):
     if upath.protocol != "s3":
         raise ValueError("The provided UPath must be an S3 path.")
     key = "/".join(upath.parts[1:])
-    bucket = upath._url.netloc
+    bucket = upath.drive
     if bucket == "scverse-spatial-eu-central-1":
         region = "eu-central-1"
     elif f"s3://{bucket}" not in HOSTED_BUCKETS:
-        response = upath.fs.call_s3("head_bucket", Bucket=upath._url.netloc)
+        response = upath.fs.call_s3("head_bucket", Bucket=bucket)
         headers = response["ResponseMetadata"]["HTTPHeaders"]
         region = headers.get("x-amz-bucket-region")
     else:
@@ -763,13 +767,9 @@ def check_storage_is_empty(
         if raise_error
         else "consider deleting them"
     )
-    hint = "'_is_initialized'"
-    if n_offset_objects == 2:
-        hint += " & SQLite file"
-    hint += " ignored"
     message = (
-        f"Storage {directory_string} contains {n_objects - n_offset_objects} objects "
-        f"({hint}) - {ask_for_deletion}"
+        f"Storage '{directory_string}' contains {n_objects - n_offset_objects} objects"
+        f" - {ask_for_deletion}"
     )
     if n_diff > 0:
         if raise_error:

{lamindb_setup-0.79.0.dist-info → lamindb_setup-0.80.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.79.0
+Version: 0.80.1
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.9

{lamindb_setup-0.79.0.dist-info → lamindb_setup-0.80.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-lamindb_setup/__init__.py,sha256=UEOTdiNVX6I8dLRxhPxeKjfNRyjzd2Kua0notBhVObM,1714
+lamindb_setup/__init__.py,sha256=kc1LSkxZIe6SeFBsho6Xg6WyYxKcsUYoLNgQlQSRnxo,1714
 lamindb_setup/_cache.py,sha256=1XnM-V_KprbjpgPY7Bg3FYn53Iz_2_fEgcMOaSdKKbg,1332
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=6cSfpmVOSgU7YiVHfJpBTGTQ7rrnwunt1pJT_jkgNM8,3196
@@ -9,39 +9,39 @@ lamindb_setup/_django.py,sha256=DWUTjjVhEViX0S-zIkeqQgKovWqVgWMl4Y0ANwlA3Pw,1505
 lamindb_setup/_entry_points.py,sha256=Hs2oJQOCTaGUdWn-1mufM6qUZr9W_EJ_Oc3f0_Vc0Yw,616
 lamindb_setup/_exportdb.py,sha256=43g77-tH-vAlTn8ig1mMD9-KXLKvxUeDLaq0gVu3l-c,2114
 lamindb_setup/_importdb.py,sha256=yYYShzUajTsR-cTW4CZ-UNDWZY2uE5PAgNbp-wn8Ogc,1874
-lamindb_setup/_init_instance.py,sha256=6Db289T2A2464KqW0NsfSKg59zrhfny9RZxBgb9kPVs,14340
+lamindb_setup/_init_instance.py,sha256=zJbkP22h4Yif_VNooL06FFoHqH1EU45CqiLwF8FUaKs,14368
 lamindb_setup/_migrate.py,sha256=x_b4k4XRfLSD-EEFMc324yK6DIK7goW33wUytbIWlNs,8917
 lamindb_setup/_register_instance.py,sha256=alQuYp2f8Ct8xvRC1gt8p_HZ0tqCd3gZD3kiPBLPpsI,1269
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
 lamindb_setup/_schema_metadata.py,sha256=49wDLbhRhatDeADjHOPfQURt65lf_OZ7lob1ZiAT_ac,13773
 lamindb_setup/_set_managed_storage.py,sha256=4tDxXQMt8Gw028uY3vIQxZQ7qBNXhQMc8saarNK_Z-s,2043
-lamindb_setup/_setup_user.py,sha256=LB1MuhMZRpkCJBjvM7U656r1QnYFioJx7QQwxsf-CqY,4584
+lamindb_setup/_setup_user.py,sha256=-g7Xj6510BDyM8kuqAsVBZFwehlhBa_uWBSV1rPeuM8,4586
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
 lamindb_setup/core/__init__.py,sha256=BxIVMX5HQq8oZ1OuY_saUEJz5Tdd7gaCPngxVu5iou4,417
-lamindb_setup/core/_aws_credentials.py,sha256=uKMQO9q42Hnepz8aj3RxwLKDWUJx8pNOYrFnnNh5X40,5325
+lamindb_setup/core/_aws_credentials.py,sha256=E8yanKOq-idaIz5y9D9SazHaYJKX69VLK3pn7jl82do,5815
 lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
 lamindb_setup/core/_deprecated.py,sha256=3qxUI1dnDlSeR0BYrv7ucjqRBEojbqotPgpShXs4KF8,2520
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
-lamindb_setup/core/_hub_client.py,sha256=P11Ma0nDsBHlKcnyOmq75dHDzYYJfmfpTePJ-0348Wo,6117
+lamindb_setup/core/_hub_client.py,sha256=cN19XbZmvLCxL_GKdOcKbedNRL7kR47vmLmA--NMv-U,6306
 lamindb_setup/core/_hub_core.py,sha256=eUxRz9iJj6RA5-MWgQqqZYAU-di5LQDamRZn6t-VOiM,19838
 lamindb_setup/core/_hub_crud.py,sha256=eZErpq9t1Cp2ULBSi457ekrcqfesw4Y6IJgaqyrINMY,5276
 lamindb_setup/core/_hub_utils.py,sha256=08NwQsb53-tXa_pr-f0tPTN0FeeVf_i1p3dEbEWD0F4,3016
 lamindb_setup/core/_private_django_api.py,sha256=KIn43HOhiRjkbTbddyJqv-WNTTa1bAizbM1tWXoXPBg,2869
-lamindb_setup/core/_settings.py,sha256=ZIcQ3aLNRsAkoaCmse9QuMc2atDWkiraApZn0ZMOWwQ,6128
+lamindb_setup/core/_settings.py,sha256=mpGsSb98UsBedLsW2RuowZ17EP2tI2XRGPztqrJtrV4,7952
 lamindb_setup/core/_settings_instance.py,sha256=ajcq9zRNE598tTqyMkMqaEOubVfFeE998DPtbgyzK3A,18801
 lamindb_setup/core/_settings_load.py,sha256=5OpghcbkrK9KBM_0Iu-61FTI76UbOpPkkJpUittXS-w,4098
 lamindb_setup/core/_settings_save.py,sha256=rxGxgaK5i9exKqSJERQQyY1WZio20meoQJoYXlVW-1w,3138
-lamindb_setup/core/_settings_storage.py,sha256=u9CyOFxtV7M2Cs_L4k3dUVg3_fsl8SqLwBioMu06Acc,12310
+lamindb_setup/core/_settings_storage.py,sha256=15B7taJF1zxJ1_qAb67NuXkTFvO2TRTWMt6KTzDf1mw,11875
 lamindb_setup/core/_settings_store.py,sha256=WcsgOmgnu9gztcrhp-N4OONNZyxICHV8M0HdJllTaEo,2219
-lamindb_setup/core/_settings_user.py,sha256=vmh8-SDJqlF4zRqAH2loS21_TEYWX4tiG-grQys6-0c,1474
+lamindb_setup/core/_settings_user.py,sha256=iz0MqFLKXqm8LYx_CHmr02_oNvYWFLIxKkJLdpS5W08,1476
 lamindb_setup/core/_setup_bionty_sources.py,sha256=o2L5Ww8TKgSqJtL4cGUcpJwLNYxA9BZgddhCMCu_E2g,3428
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=i6TrT7HG0lqliPvZTlsZ_uplPaqhPBbabyfeR32SkA8,7107
 lamindb_setup/core/django.py,sha256=E4U9nUlV2kHd-G5v6iSdFGAAWixlQDxOFwMwOMG9xfw,3864
 lamindb_setup/core/exceptions.py,sha256=4NpLUNUIfXYVTFX2FvLZF8RW34exk2Vn2X3G4YhnTRg,276
 lamindb_setup/core/hashing.py,sha256=bkuvZyAuC7-Y_qZumJd_rybF-upJ5J3KxnKiymRUifw,3148
 lamindb_setup/core/types.py,sha256=zJii2le38BJUmsNVvzDrbzGYr0yaeb-9Rw9IKmsBr3k,523
-lamindb_setup/core/upath.py,sha256=EPLLm62q-Y3hZzd-286cynFMttXKDNXNOKL3_QGkeug,27215
-lamindb_setup-0.79.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-0.79.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb_setup-0.79.0.dist-info/METADATA,sha256=sMW1HnGk3llbrrA1BIvx_bAa6xmnMqTYRjnDTlfaTS4,1743
-lamindb_setup-0.79.0.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=q6WvpdXO_-Ajl5qjU4CIf0Q1ZbYxA0Po54LG9iHYn28,27151
+lamindb_setup-0.80.1.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-0.80.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb_setup-0.80.1.dist-info/METADATA,sha256=kMZ0UUKGc3sxehsLkXFPedNRYpo_VNNHKA-iu7I5W4I,1743
+lamindb_setup-0.80.1.dist-info/RECORD,,