lamindb_setup 0.80.0__py3-none-any.whl → 0.80.1__py3-none-any.whl

lamindb_setup/__init__.py

@@ -33,7 +33,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.80.0"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.80.1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 import sys as _sys

lamindb_setup/_init_instance.py

@@ -415,9 +415,10 @@ def infer_instance_name(
     if storage == "create-s3":
         raise ValueError("pass name to init if storage = 'create-s3'")
     storage_path = UPath(storage)
+    # not sure if name is ever ""
     if storage_path.name != "":
         name = storage_path.name
     else:
         # dedicated treatment of bucket names
-        name = storage_path._url.netloc
+        name = storage_path.drive
     return name.lower()

lamindb_setup/core/_aws_credentials.py

@@ -15,9 +15,7 @@ HOSTED_REGIONS = [
 ]
 lamin_env = os.getenv("LAMIN_ENV")
 if lamin_env is None or lamin_env == "prod":
-    hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
-    hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
-    HOSTED_BUCKETS = tuple(hosted_buckets_list)
+    HOSTED_BUCKETS = tuple([f"s3://lamin-{region}" for region in HOSTED_REGIONS])
 else:
     HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
 
@@ -29,6 +27,11 @@ def _keep_trailing_slash(path_str: str):
 AWS_CREDENTIALS_EXPIRATION = 11 * 60 * 60  # refresh credentials after 11 hours
 
 
+# set anon=True for these buckets if credentials fail for a public bucket
+# to be expanded
+PUBLIC_BUCKETS = ("cellxgene-data-public",)
+
+
 class AWSCredentialsManager:
     def __init__(self):
         self._credentials_cache = {}
@@ -38,7 +41,15 @@ class AWSCredentialsManager:
         # this is cached so will be resued with the connection initialized
         fs = S3FileSystem(cache_regions=True)
         fs.connect()
-        self.anon = fs.session._credentials is None
+        self.anon: bool = fs.session._credentials is None
+        self.anon_public: bool | None = None
+        if not self.anon:
+            try:
+                # use lamindata public bucket for this test
+                fs.call_s3("head_bucket", Bucket="lamindata")
+                self.anon_public = False
+            except Exception as e:
+                self.anon_public = isinstance(e, PermissionError)
 
     def _find_root(self, path_str: str) -> str | None:
         roots = self._credentials_cache.keys()
@@ -73,6 +84,8 @@ class AWSCredentialsManager:
                 anon = False
             else:
                 anon = self.anon
+                if not anon and self.anon_public and path.drive in PUBLIC_BUCKETS:
+                    anon = True
             connection_options = {"anon": anon}
         else:
             connection_options = credentials
@@ -132,7 +145,7 @@ class AWSCredentialsManager:
                         root = "/".join(path.path.rstrip("/").split("/")[:2])
                     else:
                         # write the bucket for everything else
-                        root = path._url.netloc
+                        root = path.drive
                     root = "s3://" + root
                 self._set_cached_credentials(_keep_trailing_slash(root), credentials)
 

lamindb_setup/core/_settings.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import sys
 from typing import TYPE_CHECKING
 
 from appdirs import AppDirs
@@ -172,6 +173,9 @@ class SetupSettings:
 
     def __repr__(self) -> str:
         """Rich string representation."""
+        # do not show current setting representation when building docs
+        if "sphinx" in sys.modules:
+            return object.__repr__(self)
         repr = self.user.__repr__()
         repr += f"\nAuto-connect in Python: {self.auto_connect}\n"
         repr += f"Private Django API: {self.private_django_api}\n"

lamindb_setup/core/upath.py

@@ -8,14 +8,14 @@ from collections import defaultdict
 from datetime import datetime, timezone
 from functools import partial
 from itertools import islice
-from pathlib import Path, PurePosixPath
+from pathlib import Path, PosixPath, PurePosixPath, WindowsPath
 from typing import TYPE_CHECKING, Any, Literal
 
 import fsspec
 from lamin_utils import logger
 from upath import UPath
 from upath.implementations.cloud import CloudPath, S3Path  # keep CloudPath!
-from upath.implementations.local import LocalPath, PosixUPath, WindowsUPath
+from upath.implementations.local import LocalPath
 
 from ._aws_credentials import HOSTED_BUCKETS, get_aws_credentials_manager
 from .hashing import HASH_LENGTH, b16_to_b64, hash_md5s_from_dir
@@ -23,7 +23,7 @@ from .hashing import HASH_LENGTH, b16_to_b64, hash_md5s_from_dir
 if TYPE_CHECKING:
     from .types import UPathStr
 
-LocalPathClasses = (PosixUPath, WindowsUPath, LocalPath)
+LocalPathClasses = (PosixPath, WindowsPath, LocalPath)
 
 # also see https://gist.github.com/securifera/e7eed730cbe1ce43d0c29d7cd2d582f4
 #    ".gz" is not listed here as it typically occurs with another suffix
@@ -291,21 +291,25 @@ def upload_from(
         callback = ProgressCallback(local_path.name, "uploading")
         kwargs["callback"] = callback
 
+    source: str | list[str]
+    destination: str | list[str]
     if local_path_is_dir and not create_folder:
-        source = [f for f in local_path.rglob("*") if f.is_file()]
-        destination = [str(self / f.relative_to(local_path)) for f in source]
-        source = [str(f) for f in source]  # type: ignore
+        source = [f.as_posix() for f in local_path.rglob("*") if f.is_file()]
+        destination = fsspec.utils.other_paths(
+            source, self.as_posix(), exists=False, flatten=False
+        )
     else:
-        source = str(local_path)  # type: ignore
-        destination = str(self)  # type: ignore
+        source = local_path.as_posix()
+        destination = self.as_posix()
 
     # the below lines are to avoid s3fs triggering create_bucket in upload if
     # dirs are present it allows to avoid permission error
     # would be easier to just
     if self.protocol == "s3" and local_path_is_dir and create_folder:
-        bucket = self._url.netloc
+        bucket = self.drive
         if bucket not in self.fs.dircache:
             self.fs.dircache[bucket] = [{}]
+            assert isinstance(destination, str)
             if not destination.endswith(TRAILING_SEP):  # type: ignore
                 destination += "/"
             cleanup_cache = True
@@ -399,8 +403,8 @@ def synchronize(
             callback.set_size(len(files))
             origin_file_keys = []
             for file, stat in callback.wrap(files.items()):
-                file_key = PurePosixPath(file).relative_to(self.path)
-                origin_file_keys.append(file_key.as_posix())
+                file_key = PurePosixPath(file).relative_to(self.path).as_posix()
+                origin_file_keys.append(file_key)
                 timestamp = stat[modified_key].timestamp()
 
                 origin = f"{self.protocol}://{file}"
@@ -615,11 +619,11 @@ def to_url(upath):
     if upath.protocol != "s3":
         raise ValueError("The provided UPath must be an S3 path.")
     key = "/".join(upath.parts[1:])
-    bucket = upath._url.netloc
+    bucket = upath.drive
     if bucket == "scverse-spatial-eu-central-1":
         region = "eu-central-1"
     elif f"s3://{bucket}" not in HOSTED_BUCKETS:
-        response = upath.fs.call_s3("head_bucket", Bucket=upath._url.netloc)
+        response = upath.fs.call_s3("head_bucket", Bucket=bucket)
         headers = response["ResponseMetadata"]["HTTPHeaders"]
         region = headers.get("x-amz-bucket-region")
     else:

{lamindb_setup-0.80.0.dist-info → lamindb_setup-0.80.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.80.0
+Version: 0.80.1
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.9

{lamindb_setup-0.80.0.dist-info → lamindb_setup-0.80.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-lamindb_setup/__init__.py,sha256=hCbASHI6nzxl7Xre0B3sTSQaCJ5mWkjI2xSF2odlJ8Q,1714
+lamindb_setup/__init__.py,sha256=kc1LSkxZIe6SeFBsho6Xg6WyYxKcsUYoLNgQlQSRnxo,1714
 lamindb_setup/_cache.py,sha256=1XnM-V_KprbjpgPY7Bg3FYn53Iz_2_fEgcMOaSdKKbg,1332
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=6cSfpmVOSgU7YiVHfJpBTGTQ7rrnwunt1pJT_jkgNM8,3196
@@ -9,7 +9,7 @@ lamindb_setup/_django.py,sha256=DWUTjjVhEViX0S-zIkeqQgKovWqVgWMl4Y0ANwlA3Pw,1505
 lamindb_setup/_entry_points.py,sha256=Hs2oJQOCTaGUdWn-1mufM6qUZr9W_EJ_Oc3f0_Vc0Yw,616
 lamindb_setup/_exportdb.py,sha256=43g77-tH-vAlTn8ig1mMD9-KXLKvxUeDLaq0gVu3l-c,2114
 lamindb_setup/_importdb.py,sha256=yYYShzUajTsR-cTW4CZ-UNDWZY2uE5PAgNbp-wn8Ogc,1874
-lamindb_setup/_init_instance.py,sha256=6Db289T2A2464KqW0NsfSKg59zrhfny9RZxBgb9kPVs,14340
+lamindb_setup/_init_instance.py,sha256=zJbkP22h4Yif_VNooL06FFoHqH1EU45CqiLwF8FUaKs,14368
 lamindb_setup/_migrate.py,sha256=x_b4k4XRfLSD-EEFMc324yK6DIK7goW33wUytbIWlNs,8917
 lamindb_setup/_register_instance.py,sha256=alQuYp2f8Ct8xvRC1gt8p_HZ0tqCd3gZD3kiPBLPpsI,1269
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
@@ -18,7 +18,7 @@ lamindb_setup/_set_managed_storage.py,sha256=4tDxXQMt8Gw028uY3vIQxZQ7qBNXhQMc8sa
 lamindb_setup/_setup_user.py,sha256=-g7Xj6510BDyM8kuqAsVBZFwehlhBa_uWBSV1rPeuM8,4586
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
 lamindb_setup/core/__init__.py,sha256=BxIVMX5HQq8oZ1OuY_saUEJz5Tdd7gaCPngxVu5iou4,417
-lamindb_setup/core/_aws_credentials.py,sha256=uKMQO9q42Hnepz8aj3RxwLKDWUJx8pNOYrFnnNh5X40,5325
+lamindb_setup/core/_aws_credentials.py,sha256=E8yanKOq-idaIz5y9D9SazHaYJKX69VLK3pn7jl82do,5815
 lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
 lamindb_setup/core/_deprecated.py,sha256=3qxUI1dnDlSeR0BYrv7ucjqRBEojbqotPgpShXs4KF8,2520
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
@@ -27,7 +27,7 @@ lamindb_setup/core/_hub_core.py,sha256=eUxRz9iJj6RA5-MWgQqqZYAU-di5LQDamRZn6t-VO
 lamindb_setup/core/_hub_crud.py,sha256=eZErpq9t1Cp2ULBSi457ekrcqfesw4Y6IJgaqyrINMY,5276
 lamindb_setup/core/_hub_utils.py,sha256=08NwQsb53-tXa_pr-f0tPTN0FeeVf_i1p3dEbEWD0F4,3016
 lamindb_setup/core/_private_django_api.py,sha256=KIn43HOhiRjkbTbddyJqv-WNTTa1bAizbM1tWXoXPBg,2869
-lamindb_setup/core/_settings.py,sha256=Iv2FUz1l92V4AcuyvRsYlJxoY1oB-UB4gAaA6QgWioE,7792
+lamindb_setup/core/_settings.py,sha256=mpGsSb98UsBedLsW2RuowZ17EP2tI2XRGPztqrJtrV4,7952
 lamindb_setup/core/_settings_instance.py,sha256=ajcq9zRNE598tTqyMkMqaEOubVfFeE998DPtbgyzK3A,18801
 lamindb_setup/core/_settings_load.py,sha256=5OpghcbkrK9KBM_0Iu-61FTI76UbOpPkkJpUittXS-w,4098
 lamindb_setup/core/_settings_save.py,sha256=rxGxgaK5i9exKqSJERQQyY1WZio20meoQJoYXlVW-1w,3138
@@ -40,8 +40,8 @@ lamindb_setup/core/django.py,sha256=E4U9nUlV2kHd-G5v6iSdFGAAWixlQDxOFwMwOMG9xfw,
 lamindb_setup/core/exceptions.py,sha256=4NpLUNUIfXYVTFX2FvLZF8RW34exk2Vn2X3G4YhnTRg,276
 lamindb_setup/core/hashing.py,sha256=bkuvZyAuC7-Y_qZumJd_rybF-upJ5J3KxnKiymRUifw,3148
 lamindb_setup/core/types.py,sha256=zJii2le38BJUmsNVvzDrbzGYr0yaeb-9Rw9IKmsBr3k,523
-lamindb_setup/core/upath.py,sha256=DJ0S3JufnCmG6EAGy2eqTp5rVX3Oqw-bUlh9-9ipKYI,27091
-lamindb_setup-0.80.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-0.80.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb_setup-0.80.0.dist-info/METADATA,sha256=IBWXhrRyGmMeYdgByW0TJV2h70cSVssuxA1Do6cmfLM,1743
-lamindb_setup-0.80.0.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=q6WvpdXO_-Ajl5qjU4CIf0Q1ZbYxA0Po54LG9iHYn28,27151
+lamindb_setup-0.80.1.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-0.80.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb_setup-0.80.1.dist-info/METADATA,sha256=kMZ0UUKGc3sxehsLkXFPedNRYpo_VNNHKA-iu7I5W4I,1743
+lamindb_setup-0.80.1.dist-info/RECORD,,