lamindb_setup 0.76.7__py2.py3-none-any.whl → 0.77.0__py2.py3-none-any.whl

lamindb_setup/core/_aws_credentials.py

@@ -1,151 +1,151 @@
-from __future__ import annotations
-
-import os
-import time
-
-from upath.implementations.cloud import S3Path
-
-HOSTED_REGIONS = [
-    "eu-central-1",
-    "eu-west-2",
-    "us-east-1",
-    "us-east-2",
-    "us-west-1",
-    "us-west-2",
-]
-lamin_env = os.getenv("LAMIN_ENV")
-if lamin_env is None or lamin_env == "prod":
-    hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
-    hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
-    HOSTED_BUCKETS = tuple(hosted_buckets_list)
-else:
-    HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
-
-
-def _keep_trailing_slash(path_str: str):
-    return path_str if path_str[-1] == "/" else path_str + "/"
-
-
-AWS_CREDENTIALS_EXPIRATION = 11 * 60 * 60  # refresh credentials after 11 hours
-
-
-class AWSCredentialsManager:
-    def __init__(self):
-        self._credentials_cache = {}
-
-        from s3fs import S3FileSystem
-
-        # this is cached so will be resued with the connection initialized
-        fs = S3FileSystem(cache_regions=True)
-        fs.connect()
-        self.anon = fs.session._credentials is None
-
-    def _find_root(self, path_str: str) -> str | None:
-        roots = self._credentials_cache.keys()
-        if path_str in roots:
-            return path_str
-        roots = sorted(roots, key=len, reverse=True)
-        for root in roots:
-            if path_str.startswith(root):
-                return root
-        return None
-
-    def _is_active(self, root: str) -> bool:
-        return (
-            time.time() - self._credentials_cache[root]["time"]
-        ) < AWS_CREDENTIALS_EXPIRATION
-
-    def _set_cached_credentials(self, root: str, credentials: dict):
-        if root not in self._credentials_cache:
-            self._credentials_cache[root] = {}
-        self._credentials_cache[root]["credentials"] = credentials
-        self._credentials_cache[root]["time"] = time.time()
-
-    def _get_cached_credentials(self, root: str) -> dict:
-        return self._credentials_cache[root]["credentials"]
-
-    def _path_inject_options(self, path: S3Path, credentials: dict) -> S3Path:
-        if credentials == {}:
-            # credentials were specified manually for the path
-            if "anon" in path.storage_options:
-                anon = path.storage_options["anon"]
-            elif path.fs.key is not None and path.fs.secret is not None:
-                anon = False
-            else:
-                anon = self.anon
-            connection_options = {"anon": anon}
-        else:
-            connection_options = credentials
-
-        if "cache_regions" in path.storage_options:
-            cache_regions = path.storage_options["cache_regions"]
-        else:
-            cache_regions = True
-
-        return S3Path(path, cache_regions=cache_regions, **connection_options)
-
-    def enrich_path(self, path: S3Path, access_token: str | None = None) -> S3Path:
-        # trailing slash is needed to avoid returning incorrect results
-        # with .startswith
-        # for example s3://lamindata-eu should not receive cache for s3://lamindata
-        path_str = _keep_trailing_slash(path.as_posix())
-        root = self._find_root(path_str)
-
-        if root is not None:
-            set_cache = False
-            credentials = self._get_cached_credentials(root)
-
-            if access_token is not None:
-                set_cache = True
-            elif credentials != {}:
-                # update credentials
-                if not self._is_active(root):
-                    set_cache = True
-        else:
-            set_cache = True
-
-        if set_cache:
-            from ._hub_core import access_aws
-            from ._settings import settings
-
-            if settings.user.handle != "anonymous" or access_token is not None:
-                storage_root_info = access_aws(path_str, access_token=access_token)
-            else:
-                storage_root_info = {"credentials": {}, "accessibility": {}}
-
-            accessibility = storage_root_info["accessibility"]
-            is_managed = accessibility.get("is_managed", False)
-            if is_managed:
-                credentials = storage_root_info["credentials"]
-            else:
-                credentials = {}
-
-            if access_token is None:
-                if "storage_root" in accessibility:
-                    root = accessibility["storage_root"]
-                # just to be safe
-                root = None if root == "" else root
-                if root is None:
-                    # heuristic
-                    # do not write the first level for the known hosted buckets
-                    if path_str.startswith(HOSTED_BUCKETS):
-                        root = "/".join(path.path.rstrip("/").split("/")[:2])
-                    else:
-                        # write the bucket for everything else
-                        root = path._url.netloc
-                    root = "s3://" + root
-                self._set_cached_credentials(_keep_trailing_slash(root), credentials)
-
-        return self._path_inject_options(path, credentials)
-
-
-_aws_credentials_manager: AWSCredentialsManager | None = None
-
-
-def get_aws_credentials_manager() -> AWSCredentialsManager:
-    global _aws_credentials_manager
-
-    if _aws_credentials_manager is None:
-        _aws_credentials_manager = AWSCredentialsManager()
-
-    return _aws_credentials_manager
+from __future__ import annotations
+
+import os
+import time
+
+from upath.implementations.cloud import S3Path
+
+HOSTED_REGIONS = [
+    "eu-central-1",
+    "eu-west-2",
+    "us-east-1",
+    "us-east-2",
+    "us-west-1",
+    "us-west-2",
+]
+lamin_env = os.getenv("LAMIN_ENV")
+if lamin_env is None or lamin_env == "prod":
+    hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
+    hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
+    HOSTED_BUCKETS = tuple(hosted_buckets_list)
+else:
+    HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
+
+
+def _keep_trailing_slash(path_str: str):
+    return path_str if path_str[-1] == "/" else path_str + "/"
+
+
+AWS_CREDENTIALS_EXPIRATION = 11 * 60 * 60  # refresh credentials after 11 hours
+
+
+class AWSCredentialsManager:
+    def __init__(self):
+        self._credentials_cache = {}
+
+        from s3fs import S3FileSystem
+
+        # this is cached so will be resued with the connection initialized
+        fs = S3FileSystem(cache_regions=True)
+        fs.connect()
+        self.anon = fs.session._credentials is None
+
+    def _find_root(self, path_str: str) -> str | None:
+        roots = self._credentials_cache.keys()
+        if path_str in roots:
+            return path_str
+        roots = sorted(roots, key=len, reverse=True)
+        for root in roots:
+            if path_str.startswith(root):
+                return root
+        return None
+
+    def _is_active(self, root: str) -> bool:
+        return (
+            time.time() - self._credentials_cache[root]["time"]
+        ) < AWS_CREDENTIALS_EXPIRATION
+
+    def _set_cached_credentials(self, root: str, credentials: dict):
+        if root not in self._credentials_cache:
+            self._credentials_cache[root] = {}
+        self._credentials_cache[root]["credentials"] = credentials
+        self._credentials_cache[root]["time"] = time.time()
+
+    def _get_cached_credentials(self, root: str) -> dict:
+        return self._credentials_cache[root]["credentials"]
+
+    def _path_inject_options(self, path: S3Path, credentials: dict) -> S3Path:
+        if credentials == {}:
+            # credentials were specified manually for the path
+            if "anon" in path.storage_options:
+                anon = path.storage_options["anon"]
+            elif path.fs.key is not None and path.fs.secret is not None:
+                anon = False
+            else:
+                anon = self.anon
+            connection_options = {"anon": anon}
+        else:
+            connection_options = credentials
+
+        if "cache_regions" in path.storage_options:
+            cache_regions = path.storage_options["cache_regions"]
+        else:
+            cache_regions = True
+
+        return S3Path(path, cache_regions=cache_regions, **connection_options)
+
+    def enrich_path(self, path: S3Path, access_token: str | None = None) -> S3Path:
+        # trailing slash is needed to avoid returning incorrect results
+        # with .startswith
+        # for example s3://lamindata-eu should not receive cache for s3://lamindata
+        path_str = _keep_trailing_slash(path.as_posix())
+        root = self._find_root(path_str)
+
+        if root is not None:
+            set_cache = False
+            credentials = self._get_cached_credentials(root)
+
+            if access_token is not None:
+                set_cache = True
+            elif credentials != {}:
+                # update credentials
+                if not self._is_active(root):
+                    set_cache = True
+        else:
+            set_cache = True
+
+        if set_cache:
+            from ._hub_core import access_aws
+            from ._settings import settings
+
+            if settings.user.handle != "anonymous" or access_token is not None:
+                storage_root_info = access_aws(path_str, access_token=access_token)
+            else:
+                storage_root_info = {"credentials": {}, "accessibility": {}}
+
+            accessibility = storage_root_info["accessibility"]
+            is_managed = accessibility.get("is_managed", False)
+            if is_managed:
+                credentials = storage_root_info["credentials"]
+            else:
+                credentials = {}
+
+            if access_token is None:
+                if "storage_root" in accessibility:
+                    root = accessibility["storage_root"]
+                # just to be safe
+                root = None if root == "" else root
+                if root is None:
+                    # heuristic
+                    # do not write the first level for the known hosted buckets
+                    if path_str.startswith(HOSTED_BUCKETS):
+                        root = "/".join(path.path.rstrip("/").split("/")[:2])
+                    else:
+                        # write the bucket for everything else
+                        root = path._url.netloc
+                    root = "s3://" + root
+                self._set_cached_credentials(_keep_trailing_slash(root), credentials)
+
+        return self._path_inject_options(path, credentials)
+
+
+_aws_credentials_manager: AWSCredentialsManager | None = None
+
+
+def get_aws_credentials_manager() -> AWSCredentialsManager:
+    global _aws_credentials_manager
+
+    if _aws_credentials_manager is None:
+        _aws_credentials_manager = AWSCredentialsManager()
+
+    return _aws_credentials_manager

lamindb_setup/core/_aws_storage.py

@@ -1,48 +1,48 @@
-from __future__ import annotations
-
-
-def get_location(ip="ipinfo.io"):
-    import requests  # type: ignore
-
-    response = requests.get(f"http://{ip}/json").json()
-    loc = response["loc"].split(",")
-    return {"latitude": float(loc[0]), "longitude": float(loc[1])}
-
-
-def haversine(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
-    import math
-
-    R = 6371  # Radius of the Earth in kilometers
-    dLat = math.radians(lat2 - lat1)
-    dLon = math.radians(lon2 - lon1)
-    a = math.sin(dLat / 2) * math.sin(dLat / 2) + math.cos(
-        math.radians(lat1)
-    ) * math.cos(math.radians(lat2)) * math.sin(dLon / 2) * math.sin(dLon / 2)
-    c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
-    distance = R * c
-    return distance
-
-
-def find_closest_aws_region() -> str:
-    aws_region_locations = {
-        "us-east-1": {"latitude": 39.0, "longitude": -77.0},  # Northern Virginia
-        "us-east-2": {"latitude": 40.0, "longitude": -83.0},  # Ohio
-        "us-west-1": {"latitude": 37.77, "longitude": -122.41},  # Northern California
-        "us-west-2": {"latitude": 45.52, "longitude": -122.68},  # Oregon
-        "eu-central-1": {"latitude": 50.11, "longitude": 8.68},  # Frankfurt
-        "eu-west-2": {"latitude": 51.51, "longitude": -0.13},  # London, UK
-    }
-    your_location = get_location()
-    closest_region = ""
-    min_distance = float("inf")
-    for region in aws_region_locations:
-        region_location = aws_region_locations[region]
-        distance = haversine(
-            your_location["latitude"],
-            your_location["longitude"],
-            region_location["latitude"],
-            region_location["longitude"],
-        )
-        if distance < min_distance:
-            closest_region, min_distance = region, distance
-    return closest_region
+from __future__ import annotations
+
+
+def get_location(ip="ipinfo.io"):
+    import requests  # type: ignore
+
+    response = requests.get(f"http://{ip}/json").json()
+    loc = response["loc"].split(",")
+    return {"latitude": float(loc[0]), "longitude": float(loc[1])}
+
+
+def haversine(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
+    import math
+
+    R = 6371  # Radius of the Earth in kilometers
+    dLat = math.radians(lat2 - lat1)
+    dLon = math.radians(lon2 - lon1)
+    a = math.sin(dLat / 2) * math.sin(dLat / 2) + math.cos(
+        math.radians(lat1)
+    ) * math.cos(math.radians(lat2)) * math.sin(dLon / 2) * math.sin(dLon / 2)
+    c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+    distance = R * c
+    return distance
+
+
+def find_closest_aws_region() -> str:
+    aws_region_locations = {
+        "us-east-1": {"latitude": 39.0, "longitude": -77.0},  # Northern Virginia
+        "us-east-2": {"latitude": 40.0, "longitude": -83.0},  # Ohio
+        "us-west-1": {"latitude": 37.77, "longitude": -122.41},  # Northern California
+        "us-west-2": {"latitude": 45.52, "longitude": -122.68},  # Oregon
+        "eu-central-1": {"latitude": 50.11, "longitude": 8.68},  # Frankfurt
+        "eu-west-2": {"latitude": 51.51, "longitude": -0.13},  # London, UK
+    }
+    your_location = get_location()
+    closest_region = ""
+    min_distance = float("inf")
+    for region in aws_region_locations:
+        region_location = aws_region_locations[region]
+        distance = haversine(
+            your_location["latitude"],
+            your_location["longitude"],
+            region_location["latitude"],
+            region_location["longitude"],
+        )
+        if distance < min_distance:
+            closest_region, min_distance = region, distance
+    return closest_region

lamindb_setup/core/_deprecated.py

@@ -1,55 +1,55 @@
-from __future__ import annotations
-
-# BSD 3-Clause License
-# Copyright (c) 2017-2018 P. Angerer, F. Alexander Wolf, Theis Lab
-# All rights reserved.
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-# * Redistributions of source code must retain the above copyright notice, this
-#   list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above copyright notice,
-#   this list of conditions and the following disclaimer in the documentation
-#   and/or other materials provided with the distribution.
-# * Neither the name of the copyright holder nor the names of its
-#   contributors may be used to endorse or promote products derived from
-#   this software without specific prior written permission.
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import warnings
-from functools import wraps
-
-
-def deprecated(new_name: str):
-    """Deprecated.
-
-    This is a decorator which can be used to mark functions
-    as deprecated. It will result in a warning being emitted
-    when the function is used.
-    """
-
-    def decorator(func):
-        @wraps(func)
-        def new_func(*args, **kwargs):
-            # turn off filter
-            warnings.simplefilter("always", DeprecationWarning)
-            warnings.warn(
-                f"Use {new_name} instead of {func.__name__}, "
-                f"{func.__name__} will be removed in the future.",
-                category=DeprecationWarning,
-                stacklevel=2,
-            )
-            warnings.simplefilter("default", DeprecationWarning)  # reset filter
-            return func(*args, **kwargs)
-
-        setattr(new_func, "__deprecated", True)
-        return new_func
-
-    return decorator
+from __future__ import annotations
+
+# BSD 3-Clause License
+# Copyright (c) 2017-2018 P. Angerer, F. Alexander Wolf, Theis Lab
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import warnings
+from functools import wraps
+
+
+def deprecated(new_name: str):
+    """Deprecated.
+
+    This is a decorator which can be used to mark functions
+    as deprecated. It will result in a warning being emitted
+    when the function is used.
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def new_func(*args, **kwargs):
+            # turn off filter
+            warnings.simplefilter("always", DeprecationWarning)
+            warnings.warn(
+                f"Use {new_name} instead of {func.__name__}, "
+                f"{func.__name__} will be removed in the future.",
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+            warnings.simplefilter("default", DeprecationWarning)  # reset filter
+            return func(*args, **kwargs)
+
+        setattr(new_func, "__deprecated", True)
+        return new_func
+
+    return decorator

lamindb_setup/core/_docs.py

@@ -1,14 +1,14 @@
-from __future__ import annotations
-
-from textwrap import dedent
-
-
-def doc_args(*args):
-    """Pass arguments to docstrings."""
-
-    def dec(obj):
-        obj.__orig_doc__ = obj.__doc__
-        obj.__doc__ = dedent(obj.__doc__).format(*args)
-        return obj
-
-    return dec
+from __future__ import annotations
+
+from textwrap import dedent
+
+
+def doc_args(*args):
+    """Pass arguments to docstrings."""
+
+    def dec(obj):
+        obj.__orig_doc__ = obj.__doc__
+        obj.__doc__ = dedent(obj.__doc__).format(*args)
+        return obj
+
+    return dec