lamindb_setup 0.78.0__py3-none-any.whl

lamindb_setup/core/_aws_credentials.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import os
+import time
+
+from upath.implementations.cloud import S3Path
+
+HOSTED_REGIONS = [
+    "eu-central-1",
+    "eu-west-2",
+    "us-east-1",
+    "us-east-2",
+    "us-west-1",
+    "us-west-2",
+]
+lamin_env = os.getenv("LAMIN_ENV")
+if lamin_env is None or lamin_env == "prod":
+    hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
+    hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
+    HOSTED_BUCKETS = tuple(hosted_buckets_list)
+else:
+    HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
+
+
+def _keep_trailing_slash(path_str: str):
+    return path_str if path_str[-1] == "/" else path_str + "/"
+
+
+AWS_CREDENTIALS_EXPIRATION = 11 * 60 * 60  # refresh credentials after 11 hours
+
+
+class AWSCredentialsManager:
+    def __init__(self):
+        self._credentials_cache = {}
+
+        from s3fs import S3FileSystem
+
+        # this is cached so will be resued with the connection initialized
+        fs = S3FileSystem(cache_regions=True)
+        fs.connect()
+        self.anon = fs.session._credentials is None
+
+    def _find_root(self, path_str: str) -> str | None:
+        roots = self._credentials_cache.keys()
+        if path_str in roots:
+            return path_str
+        roots = sorted(roots, key=len, reverse=True)
+        for root in roots:
+            if path_str.startswith(root):
+                return root
+        return None
+
+    def _is_active(self, root: str) -> bool:
+        return (
+            time.time() - self._credentials_cache[root]["time"]
+        ) < AWS_CREDENTIALS_EXPIRATION
+
+    def _set_cached_credentials(self, root: str, credentials: dict):
+        if root not in self._credentials_cache:
+            self._credentials_cache[root] = {}
+        self._credentials_cache[root]["credentials"] = credentials
+        self._credentials_cache[root]["time"] = time.time()
+
+    def _get_cached_credentials(self, root: str) -> dict:
+        return self._credentials_cache[root]["credentials"]
+
+    def _path_inject_options(self, path: S3Path, credentials: dict) -> S3Path:
+        if credentials == {}:
+            # credentials were specified manually for the path
+            if "anon" in path.storage_options:
+                anon = path.storage_options["anon"]
+            elif path.fs.key is not None and path.fs.secret is not None:
+                anon = False
+            else:
+                anon = self.anon
+            connection_options = {"anon": anon}
+        else:
+            connection_options = credentials
+
+        if "cache_regions" in path.storage_options:
+            cache_regions = path.storage_options["cache_regions"]
+        else:
+            cache_regions = True
+
+        return S3Path(path, cache_regions=cache_regions, **connection_options)
+
+    def enrich_path(self, path: S3Path, access_token: str | None = None) -> S3Path:
+        # trailing slash is needed to avoid returning incorrect results
+        # with .startswith
+        # for example s3://lamindata-eu should not receive cache for s3://lamindata
+        path_str = _keep_trailing_slash(path.as_posix())
+        root = self._find_root(path_str)
+
+        if root is not None:
+            set_cache = False
+            credentials = self._get_cached_credentials(root)
+
+            if access_token is not None:
+                set_cache = True
+            elif credentials != {}:
+                # update credentials
+                if not self._is_active(root):
+                    set_cache = True
+        else:
+            set_cache = True
+
+        if set_cache:
+            from ._hub_core import access_aws
+            from ._settings import settings
+
+            if settings.user.handle != "anonymous" or access_token is not None:
+                storage_root_info = access_aws(path_str, access_token=access_token)
+            else:
+                storage_root_info = {"credentials": {}, "accessibility": {}}
+
+            accessibility = storage_root_info["accessibility"]
+            is_managed = accessibility.get("is_managed", False)
+            if is_managed:
+                credentials = storage_root_info["credentials"]
+            else:
+                credentials = {}
+
+            if access_token is None:
+                if "storage_root" in accessibility:
+                    root = accessibility["storage_root"]
+                # just to be safe
+                root = None if root == "" else root
+                if root is None:
+                    # heuristic
+                    # do not write the first level for the known hosted buckets
+                    if path_str.startswith(HOSTED_BUCKETS):
+                        root = "/".join(path.path.rstrip("/").split("/")[:2])
+                    else:
+                        # write the bucket for everything else
+                        root = path._url.netloc
+                    root = "s3://" + root
+                self._set_cached_credentials(_keep_trailing_slash(root), credentials)
+
+        return self._path_inject_options(path, credentials)
+
+
+_aws_credentials_manager: AWSCredentialsManager | None = None
+
+
+def get_aws_credentials_manager() -> AWSCredentialsManager:
+    global _aws_credentials_manager
+
+    if _aws_credentials_manager is None:
+        _aws_credentials_manager = AWSCredentialsManager()
+
+    return _aws_credentials_manager

lamindb_setup/core/_aws_storage.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+
+def get_location(ip="ipinfo.io"):
+    import requests  # type: ignore
+
+    response = requests.get(f"http://{ip}/json").json()
+    loc = response["loc"].split(",")
+    return {"latitude": float(loc[0]), "longitude": float(loc[1])}
+
+
+def haversine(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
+    import math
+
+    R = 6371  # Radius of the Earth in kilometers
+    dLat = math.radians(lat2 - lat1)
+    dLon = math.radians(lon2 - lon1)
+    a = math.sin(dLat / 2) * math.sin(dLat / 2) + math.cos(
+        math.radians(lat1)
+    ) * math.cos(math.radians(lat2)) * math.sin(dLon / 2) * math.sin(dLon / 2)
+    c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+    distance = R * c
+    return distance
+
+
+def find_closest_aws_region() -> str:
+    aws_region_locations = {
+        "us-east-1": {"latitude": 39.0, "longitude": -77.0},  # Northern Virginia
+        "us-east-2": {"latitude": 40.0, "longitude": -83.0},  # Ohio
+        "us-west-1": {"latitude": 37.77, "longitude": -122.41},  # Northern California
+        "us-west-2": {"latitude": 45.52, "longitude": -122.68},  # Oregon
+        "eu-central-1": {"latitude": 50.11, "longitude": 8.68},  # Frankfurt
+        "eu-west-2": {"latitude": 51.51, "longitude": -0.13},  # London, UK
+    }
+    your_location = get_location()
+    closest_region = ""
+    min_distance = float("inf")
+    for region in aws_region_locations:
+        region_location = aws_region_locations[region]
+        distance = haversine(
+            your_location["latitude"],
+            your_location["longitude"],
+            region_location["latitude"],
+            region_location["longitude"],
+        )
+        if distance < min_distance:
+            closest_region, min_distance = region, distance
+    return closest_region

lamindb_setup/core/_deprecated.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+# BSD 3-Clause License
+# Copyright (c) 2017-2018 P. Angerer, F. Alexander Wolf, Theis Lab
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import warnings
+from functools import wraps
+
+
+def deprecated(new_name: str):
+    """Deprecated.
+
+    This is a decorator which can be used to mark functions
+    as deprecated. It will result in a warning being emitted
+    when the function is used.
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def new_func(*args, **kwargs):
+            # turn off filter
+            warnings.simplefilter("always", DeprecationWarning)
+            warnings.warn(
+                f"Use {new_name} instead of {func.__name__}, "
+                f"{func.__name__} will be removed in the future.",
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+            warnings.simplefilter("default", DeprecationWarning)  # reset filter
+            return func(*args, **kwargs)
+
+        setattr(new_func, "__deprecated", True)
+        return new_func
+
+    return decorator

lamindb_setup/core/_docs.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from textwrap import dedent
+
+
+def doc_args(*args):
+    """Pass arguments to docstrings."""
+
+    def dec(obj):
+        obj.__orig_doc__ = obj.__doc__
+        obj.__doc__ = dedent(obj.__doc__).format(*args)
+        return obj
+
+    return dec

lamindb_setup/core/_hub_client.py

@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+import json
+import os
+from urllib.request import urlretrieve
+
+from gotrue.errors import AuthUnknownError
+from lamin_utils import logger
+from pydantic_settings import BaseSettings
+from supabase import Client, create_client  # type: ignore
+from supabase.lib.client_options import ClientOptions
+
+
+class Connector(BaseSettings):
+    url: str
+    key: str
+
+
+def load_fallback_connector() -> Connector:
+    url = "https://lamin-site-assets.s3.amazonaws.com/connector.env"
+    connector_file, _ = urlretrieve(url)
+    connector = Connector(_env_file=connector_file)
+    return connector
+
+
+PROD_URL = "https://hub.lamin.ai"
+PROD_ANON_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImxhZXNhdW1tZHlkbGxwcGdmY2h1Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2NTY4NDA1NTEsImV4cCI6MTk3MjQxNjU1MX0.WUeCRiun0ExUxKIv5-CtjF6878H8u26t0JmCWx3_2-c"
+
+
+class Environment:
+    def __init__(self, fallback: bool = False):
+        lamin_env = os.getenv("LAMIN_ENV")
+        if lamin_env is None:
+            lamin_env = "prod"
+        # set public key
+        if lamin_env == "prod":
+            if not fallback:
+                url = PROD_URL
+                key = PROD_ANON_KEY
+            else:
+                connector = load_fallback_connector()
+                url = connector.url
+                key = connector.key
+        elif lamin_env == "staging":
+            url = "https://amvrvdwndlqdzgedrqdv.supabase.co"
+            key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImFtdnJ2ZHduZGxxZHpnZWRycWR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2NzcxNTcxMzMsImV4cCI6MTk5MjczMzEzM30.Gelt3dQEi8tT4j-JA36RbaZuUvxRnczvRr3iyRtzjY0"
+        elif lamin_env == "staging-test":
+            url = "https://iugyyajllqftbpidapak.supabase.co"
+            key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Iml1Z3l5YWpsbHFmdGJwaWRhcGFrIiwicm9sZSI6ImFub24iLCJpYXQiOjE2OTQyMjYyODMsImV4cCI6MjAwOTgwMjI4M30.s7B0gMogFhUatMSwlfuPJ95kWhdCZMn1ROhZ3t6Og90"
+        elif lamin_env == "prod-test":
+            url = "https://xtdacpwiqwpbxsatoyrv.supabase.co"
+            key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Inh0ZGFjcHdpcXdwYnhzYXRveXJ2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2OTQyMjYxNDIsImV4cCI6MjAwOTgwMjE0Mn0.Dbi27qujTt8Ei9gfp9KnEWTYptE5KUbZzEK6boL46k4"
+        else:
+            url = os.environ["SUPABASE_API_URL"]
+            key = os.environ["SUPABASE_ANON_KEY"]
+        self.lamin_env: str = lamin_env
+        self.supabase_api_url: str = url
+        self.supabase_anon_key: str = key
+
+
+# runs ~0.5s
+def connect_hub(
+    fallback_env: bool = False, client_options: ClientOptions | None = None
+) -> Client:
+    env = Environment(fallback=fallback_env)
+    if client_options is None:
+        client_options = ClientOptions(auto_refresh_token=False)
+    return create_client(env.supabase_api_url, env.supabase_anon_key, client_options)
+
+
+def connect_hub_with_auth(
+    fallback_env: bool = False,
+    renew_token: bool = False,
+    access_token: str | None = None,
+) -> Client:
+    hub = connect_hub(fallback_env=fallback_env)
+    if access_token is None:
+        from lamindb_setup import settings
+
+        if renew_token:
+            settings.user.access_token = get_access_token(
+                settings.user.email, settings.user.password, settings.user.api_key
+            )
+        access_token = settings.user.access_token
+    hub.postgrest.auth(access_token)
+    hub.functions.set_auth(access_token)
+    return hub
+
+
+# runs ~0.5s
+def get_access_token(
+    email: str | None = None, password: str | None = None, api_key: str | None = None
+):
+    hub = connect_hub()
+    try:
+        if api_key is not None:
+            auth_response = hub.functions.invoke(
+                "get-jwt-v1",
+                invoke_options={"body": {"api_key": api_key}},
+            )
+            return json.loads(auth_response)["accessToken"]
+        auth_response = hub.auth.sign_in_with_password(
+            {
+                "email": email,
+                "password": password,
+            }
+        )
+        return auth_response.session.access_token
+    finally:
+        hub.auth.sign_out(options={"scope": "local"})
+
+
+def call_with_fallback_auth(
+    callable,
+    **kwargs,
+):
+    access_token = kwargs.pop("access_token", None)
+
+    if access_token is not None:
+        try:
+            client = connect_hub_with_auth(access_token=access_token)
+            result = callable(**kwargs, client=client)
+        finally:
+            try:
+                client.auth.sign_out(options={"scope": "local"})
+            except NameError:
+                pass
+        return result
+
+    for renew_token, fallback_env in [(False, False), (True, False), (False, True)]:
+        try:
+            if renew_token:
+                logger.warning(
+                    "renewing expired lamin token: call `lamin login <your-handle>` to avoid this"
+                )
+            client = connect_hub_with_auth(
+                renew_token=renew_token, fallback_env=fallback_env
+            )
+            result = callable(**kwargs, client=client)
+            break
+        # we use Exception here as the ways in which the client fails upon 401
+        # are not consistent and keep changing
+        # because we ultimately raise the error, it's OK I'd say
+        except Exception as e:
+            if fallback_env:
+                raise e
+        finally:
+            try:
+                client.auth.sign_out(options={"scope": "local"})
+            except NameError:
+                pass
+    return result
+
+
+def call_with_fallback(
+    callable,
+    **kwargs,
+):
+    for fallback_env in [False, True]:
+        try:
+            client = connect_hub(fallback_env=fallback_env)
+            result = callable(**kwargs, client=client)
+            break
+        except AuthUnknownError as e:
+            if fallback_env:
+                raise e
+        finally:
+            try:
+                # in case there was sign in
+                client.auth.sign_out(options={"scope": "local"})
+            except NameError:
+                pass
+    return result