lamindb_setup 1.11.0__py3-none-any.whl → 1.12.1__py3-none-any.whl

lamindb_setup/__init__.py

@@ -35,9 +35,15 @@ Modules & settings:
 
 """
 
-__version__ = "1.11.0"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.12.1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
+import warnings
+
+# ignore for now, remove this after supabase upgrade in the deps
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="supabase")
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="supafunc")
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="postgrest")
 
 from packaging import version as packaging_version
 

lamindb_setup/_connect_instance.py

@@ -104,6 +104,7 @@ def _connect_instance(
     db: str | None = None,
     raise_permission_error: bool = True,
     use_root_db_user: bool = False,
+    use_proxy_db: bool = False,
     access_token: str | None = None,
 ) -> InstanceSettings:
     settings_file = instance_settings_file(name, owner)
@@ -118,8 +119,7 @@ def _connect_instance(
             if db is not None and isettings.dialect == "postgresql":
                 isettings._db = db
     if make_hub_request:
-        # the following will return a string if the instance does not exist
-        # on the hub
+        # the following will return a string if the instance does not exist on the hub
         # do not call hub if the user is anonymous
         if owner != "anonymous":
             hub_result = connect_instance_hub(
@@ -127,6 +127,7 @@ def _connect_instance(
                 name=name,
                 access_token=access_token,
                 use_root_db_user=use_root_db_user,
+                use_proxy_db=use_proxy_db,
             )
         else:
             hub_result = "anonymous-user"
@@ -161,10 +162,10 @@ def _connect_instance(
                 schema_id=None
                 if (schema_id := instance_result["schema_id"]) is None
                 else UUID(schema_id),
-                fine_grained_access=bool(
-                    instance_result["fine_grained_access"]
-                ),  # can be None
-                db_permissions=instance_result.get("db_permissions", None),
+                fine_grained_access=bool(instance_result["fine_grained_access"]),
+                db_permissions=instance_result.get("db_permissions", None)
+                if not use_root_db_user
+                else "write",
             )
         else:
             if hub_result != "anonymous-user":
@@ -192,8 +193,7 @@ def reset_django_module_variables():
     #     import lamindb as ln
     #     ln.connect(...)
     #
-    # Then it will **not** work and the `ln` variable becomes stale and hold a reference
-    # to the old classes
+    # Then it will **not** work and the `ln` variable becomes stale and hold a reference to the old classes
     # Other functions that dynamically import are no problem because the variables
     # are automatically refreshed when the function runs the next time after ln.connect() was called
     logger.debug("resetting django module variables")
@@ -242,11 +242,15 @@ def reset_django_module_variables():
                 continue
 
 
-def _connect_cli(instance: str, use_root_db_user: bool = False) -> None:
+def _connect_cli(
+    instance: str, use_root_db_user: bool = False, use_proxy_db: bool = False
+) -> None:
     from lamindb_setup import settings as settings_
 
     owner, name = get_owner_name_from_identifier(instance)
-    isettings = _connect_instance(owner, name, use_root_db_user=use_root_db_user)
+    isettings = _connect_instance(
+        owner, name, use_root_db_user=use_root_db_user, use_proxy_db=use_proxy_db
+    )
     isettings._persist(write_to_disk=True)
     if not isettings.is_on_hub or isettings._is_cloud_sqlite:
         # there are two reasons to call the full-blown connect
@@ -259,6 +263,36 @@ def _connect_cli(instance: str, use_root_db_user: bool = False) -> None:
     return None
 
 
+def validate_connection_state(
+    owner: str, name: str, use_root_db_user: bool = False
+) -> None:
+    from django.db import connection
+
+    if (
+        settings._instance_exists
+        and f"{owner}/{name}" == settings.instance.slug
+        # below is to ensure that if another process interferes
+        # we don't use the in-memory mock database
+        # could be made more specific by checking whether the django
+        # configured database is the same as the one in settings
+        and connection.settings_dict["NAME"] != ":memory:"
+        and not use_root_db_user  # always re-connect for root db user
+    ):
+        logger.important(
+            f"doing nothing, already connected lamindb: {settings.instance.slug}"
+        )
+        return None
+    else:
+        if settings._instance_exists and settings.instance.slug != "none/none":
+            import lamindb as ln
+
+            if ln.context.transform is not None:
+                raise CannotSwitchDefaultInstance(
+                    "Cannot switch default instance while `ln.track()` is live: call `ln.finish()`"
+                )
+        reset_django()
+
+
 @unlock_cloud_sqlite_upon_exception(ignore_prev_locker=True)
 def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
     """Connect to an instance.
@@ -274,6 +308,7 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
     # validate kwargs
     valid_kwargs = {
         "use_root_db_user",
+        "use_proxy_db",
         "_db",
         "_write_settings",
         "_raise_not_found_error",
@@ -284,15 +319,18 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
     for kwarg in kwargs:
         if kwarg not in valid_kwargs:
             raise TypeError(f"connect() got unexpected keyword argument '{kwarg}'")
-    isettings: InstanceSettings = None  # type: ignore
-    # _db is still needed because it is called in init
+
     use_root_db_user: bool = kwargs.get("use_root_db_user", False)
+    use_proxy_db = kwargs.get("use_proxy_db", False)
+    # _db is still needed because it is called in init
     _db: str | None = kwargs.get("_db", None)
     _write_settings: bool = kwargs.get("_write_settings", False)
     _raise_not_found_error: bool = kwargs.get("_raise_not_found_error", True)
     _reload_lamindb: bool = kwargs.get("_reload_lamindb", True)
     _test: bool = kwargs.get("_test", False)
 
+    isettings: InstanceSettings = None  # type: ignore
+
     access_token: str | None = None
     _user: UserSettings | None = kwargs.get("_user", None)
     if _user is not None:
@@ -317,36 +355,11 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
             if _db is not None and isettings.dialect == "postgresql":
                 isettings._db = _db
         else:
-            from django.db import connection
-
             owner, name = get_owner_name_from_identifier(instance)
             if _check_instance_setup() and not _test:
-                if (
-                    settings._instance_exists
-                    and f"{owner}/{name}" == settings.instance.slug
-                    # below is to ensure that if another process interferes
-                    # we don't use the in-memory mock database
-                    # could be made more specific by checking whether the django
-                    # configured database is the same as the one in settings
-                    and connection.settings_dict["NAME"] != ":memory:"
-                    and not use_root_db_user  # always re-connect for root db user
-                ):
-                    logger.important(
-                        f"doing nothing, already connected lamindb: {settings.instance.slug}"
-                    )
-                    return None
-                else:
-                    if (
-                        settings._instance_exists
-                        and settings.instance.slug != "none/none"
-                    ):
-                        import lamindb as ln
-
-                        if ln.context.transform is not None:
-                            raise CannotSwitchDefaultInstance(
-                                "Cannot switch default instance while `ln.track()` is live: call `ln.finish()`"
-                            )
-                    reset_django()
+                validate_connection_state(
+                    owner, name, use_root_db_user=use_root_db_user
+                )
             elif (
                 _write_settings
                 and settings._instance_exists
@@ -362,6 +375,7 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
                     db=_db,
                     access_token=access_token,
                     use_root_db_user=use_root_db_user,
+                    use_proxy_db=use_proxy_db,
                 )
             except InstanceNotFoundError as e:
                 if _raise_not_found_error:
@@ -416,7 +430,7 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
     return None
 
 
-def get_owner_name_from_identifier(identifier: str):
+def get_owner_name_from_identifier(identifier: str) -> tuple[str, str]:
     if "/" in identifier:
         if identifier.startswith("https://lamin.ai/"):
             identifier = identifier.replace("https://lamin.ai/", "")

lamindb_setup/_delete.py

@@ -21,6 +21,8 @@ if TYPE_CHECKING:
 
 
 def delete_cache(isettings: InstanceSettings):
+    if isettings.storage is None:
+        return
     # avoid init of root
     root = isettings.storage._root_init
     if not isinstance(root, LocalPathClasses):
@@ -40,7 +42,7 @@ def delete_by_isettings(isettings: InstanceSettings) -> None:
     if settings_file.exists():
         settings_file.unlink()
     delete_cache(isettings)
-    if isettings.dialect == "sqlite":
+    if isettings.dialect == "sqlite" and isettings.storage is not None:
         try:
             if isettings._sqlite_file.exists():
                 isettings._sqlite_file.unlink()

lamindb_setup/_init_instance.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import importlib
 import os
 import uuid
-from pathlib import Path
 from typing import TYPE_CHECKING, Literal
 from uuid import UUID
 
@@ -17,12 +16,13 @@ from ._silence_loggers import silence_loggers
 from .core import InstanceSettings
 from .core._docs import doc_args
 from .core._settings import settings
-from .core._settings_instance import check_is_instance_remote, is_local_db_url
+from .core._settings_instance import check_is_instance_remote
 from .core._settings_storage import StorageSettings, init_storage
 from .core.upath import UPath
 from .errors import CannotSwitchDefaultInstance
 
 if TYPE_CHECKING:
+    from lamindb.models import Storage
     from pydantic import PostgresDsn
 
     from .core._settings_user import UserSettings
@@ -50,7 +50,7 @@ def get_schema_module_name(module_name, raise_import_error: bool = True) -> str
     return None
 
 
-def register_storage_in_instance(ssettings: StorageSettings):
+def register_storage_in_instance(ssettings: StorageSettings) -> Storage:
     from lamindb.models import Storage
 
     # how do we ensure that this function is only called passing
@@ -70,7 +70,7 @@ def register_storage_in_instance(ssettings: StorageSettings):
     return storage
 
 
-def register_user(usettings: UserSettings, update_user: bool = True):
+def register_user(usettings: UserSettings, update_user: bool = True) -> None:
     from lamindb.models import User
 
     if not update_user and User.objects.filter(uid=usettings.uid).exists():
@@ -92,7 +92,9 @@ def register_user(usettings: UserSettings, update_user: bool = True):
         pass
 
 
-def register_initial_records(isettings: InstanceSettings, usettings: UserSettings):
+def register_initial_records(
+    isettings: InstanceSettings, usettings: UserSettings
+) -> None:
     """Register space, user & storage in DB."""
     from django.db.utils import OperationalError
     from lamindb.models import Branch, Space
@@ -246,6 +248,15 @@ def init(
     See Also:
         Init an instance for via the CLI, see `here <https://docs.lamin.ai/cli#init>`__.
     """
+    from ._check_setup import _check_instance_setup
+    from ._connect_instance import (
+        reset_django_module_variables,
+        validate_connection_state,
+    )
+    from .core._hub_core import init_instance_hub
+
+    silence_loggers()
+
     isettings = None
     ssettings = None
 
@@ -262,22 +273,6 @@ def init(
     access_token: str | None = None if _user is None else _user.access_token
 
     try:
-        silence_loggers()
-        from ._check_setup import _check_instance_setup
-
-        if _check_instance_setup() and not _test:
-            from lamindb_setup.core.django import reset_django
-
-            if settings._instance_exists:
-                raise CannotSwitchDefaultInstance(
-                    "Cannot init new instance after connecting to an existing instance."
-                )
-            reset_django()
-        elif _write_settings:
-            disconnect(mute=True)
-        from ._connect_instance import reset_django_module_variables
-        from .core._hub_core import init_instance_hub
-
         name_str, instance_id, instance_state, _ = validate_init_args(
             storage=storage,
             name=name,
@@ -289,6 +284,10 @@ def init(
         )
         if instance_state == "connected":
             return None
+        if _check_instance_setup() and not _test:
+            validate_connection_state(user_handle, name_str)
+        elif _write_settings:
+            disconnect(mute=True)
         isettings = InstanceSettings(
             id=instance_id,  # type: ignore
             owner=user_handle,
@@ -379,10 +378,8 @@ def load_from_isettings(
     else:
         # when loading, django is already set up
         #
-        # only register user if the instance is connected
-        # for the first time in an environment
-        # this is our best proxy for that the user might not
-        # yet be registered
+        # only register user if the instance is connected for the first time in an environment
+        # this is our best proxy for that the user might not yet be registered
         if not isettings._get_settings_file().exists():
             # do not try to update the user on fine grained access instances
             # this is blocked anyways, only select and insert are allowed

lamindb_setup/_migrate.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-import requests  # type: ignore
+import httpx
 from django.db import connection
 from django.db.migrations.loader import MigrationLoader
 from lamin_utils import logger
@@ -146,7 +146,7 @@ class migrate:
                 logger.warning(
                     "clearing instance cache in hub; if this fails, re-run with latest lamindb version"
                 )
-                requests.delete(
+                httpx.delete(
                     f"{settings.instance.api_url}/cache/instances/{settings.instance._id.hex}",
                     headers={"Authorization": f"Bearer {settings.user.access_token}"},
                 )

lamindb_setup/_schema_metadata.py

@@ -71,9 +71,9 @@ def _synchronize_schema(client: Client) -> tuple[bool, UUID, dict]:
         .eq("id", settings.instance._id.hex)
         .execute()
     )
-    assert (
-        len(instance_response.data) == 1
-    ), f"schema of instance {settings.instance._id.hex} could not be updated with schema {schema_uuid.hex}"
+    assert len(instance_response.data) == 1, (
+        f"schema of instance {settings.instance._id.hex} could not be updated with schema {schema_uuid.hex}"
+    )
 
     return is_new, schema_uuid, schema
 

lamindb_setup/core/_aws_options.py

@@ -29,8 +29,7 @@ def _keep_trailing_slash(path_str: str) -> str:
 AWS_CREDENTIALS_EXPIRATION: int = 11 * 60 * 60  # refresh credentials after 11 hours
 
 
-# set anon=True for these buckets if credentials fail for a public bucket
-# to be expanded
+# set anon=True for these buckets if credentials fail for a public bucket to be expanded
 PUBLIC_BUCKETS: tuple[str, ...] = ("cellxgene-data-public", "bionty-assets")
 
 
@@ -155,8 +154,7 @@ class AWSOptionsManager:
                 # this option is needed for correct uploads to R2
                 path = UPath(path, fixed_upload_size=True)
             return path
-        # trailing slash is needed to avoid returning incorrect results
-        # with .startswith
+        # trailing slash is needed to avoid returning incorrect results with .startswith
         # for example s3://lamindata-eu should not receive cache for s3://lamindata
         path_str = _keep_trailing_slash(path.as_posix())
         root = self._find_root(path_str)

lamindb_setup/core/_aws_storage.py

@@ -1,12 +1,11 @@
 from __future__ import annotations
 
+import httpx
 from lamin_utils import logger
 
 
 def get_location(ip="ipinfo.io"):
-    import requests  # type: ignore
-
-    response = requests.get(f"http://{ip}/json").json()
+    response = httpx.get(f"http://{ip}/json").json()
     loc = response["loc"].split(",")
     return {"latitude": float(loc[0]), "longitude": float(loc[1])}
 

lamindb_setup/core/_hub_client.py

@@ -2,10 +2,13 @@ from __future__ import annotations
 
 import json
 import os
+from contextlib import contextmanager
+from datetime import datetime
 from typing import Literal
 from urllib.request import urlretrieve
 
-from httpx import HTTPTransport
+import httpx
+from httpx_retries import Retry, RetryTransport
 from lamin_utils import logger
 from pydantic_settings import BaseSettings
 from supabase import Client, create_client  # type: ignore
@@ -61,7 +64,17 @@ class Environment:
         self.supabase_anon_key: str = key
 
 
-DEFAULT_TIMEOUT = 20
+DEFAULT_TIMEOUT = 12
+
+
+# needed to log retries
+class LogRetry(Retry):
+    def increment(self):
+        new = super().increment()
+        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        # new.attempts_made is the 1-based retry count
+        logger.warning(f"{now} HTTP retry attempt {new.attempts_made}/{new.total}")
+        return new
 
 
 # runs ~0.5s
@@ -78,11 +91,33 @@ def connect_hub(
     client = create_client(env.supabase_api_url, env.supabase_anon_key, client_options)
     # needed to enable retries for http requests in supabase
     # these are separate clients and need separate transports
-    # retries are done only in case an httpx.ConnectError or an httpx.ConnectTimeout occurs
-    transport_kwargs = {"verify": True, "http2": True, "retries": 2}
-    client.auth._http_client._transport = HTTPTransport(**transport_kwargs)
-    client.functions._client._transport = HTTPTransport(**transport_kwargs)
-    client.postgrest.session._transport = HTTPTransport(**transport_kwargs)
+    transports = []
+    for _ in range(2):
+        transports.append(
+            RetryTransport(
+                retry=LogRetry(total=2, backoff_factor=0.2),
+                transport=httpx.HTTPTransport(verify=True, http2=True),
+            )
+        )
+    client.auth._http_client._transport = transports[0]
+    client.postgrest.session._transport = transports[1]
+    # POST is not retryable by default, but for our functions it should be safe to retry
+    client.functions._client._transport = RetryTransport(
+        retry=LogRetry(
+            total=2,
+            backoff_factor=0.2,
+            allowed_methods=[
+                "HEAD",
+                "GET",
+                "PUT",
+                "DELETE",
+                "OPTIONS",
+                "TRACE",
+                "POST",
+            ],
+        ),
+        transport=httpx.HTTPTransport(verify=True, http2=True),
+    )
     return client
 
 
@@ -140,17 +175,18 @@ def call_with_fallback_auth(
     access_token = kwargs.pop("access_token", None)
 
     if access_token is not None:
+        client = None
         try:
             client = connect_hub_with_auth(access_token=access_token)
             result = callable(**kwargs, client=client)
         finally:
-            try:
+            if client is not None:
                 client.auth.sign_out(options={"scope": "local"})
-            except NameError:
-                pass
+
         return result
 
     for renew_token, fallback_env in [(False, False), (True, False), (False, True)]:
+        client = None
         try:
             client = connect_hub_with_auth(
                 renew_token=renew_token, fallback_env=fallback_env
@@ -171,10 +207,9 @@ def call_with_fallback_auth(
             if fallback_env:
                 raise e
         finally:
-            try:
+            if client is not None:
                 client.auth.sign_out(options={"scope": "local"})
-            except NameError:
-                pass
+
     return result
 
 
@@ -183,6 +218,7 @@ def call_with_fallback(
     **kwargs,
 ):
     for fallback_env in [False, True]:
+        client = None
         try:
             client = connect_hub(fallback_env=fallback_env)
             result = callable(**kwargs, client=client)
@@ -191,25 +227,32 @@ def call_with_fallback(
             if fallback_env:
                 raise e
         finally:
-            try:
+            if client is not None:
                 # in case there was sign in
                 client.auth.sign_out(options={"scope": "local"})
-            except NameError:
-                pass
     return result
 
 
-def requests_client():
-    # local is used in tests
-    if os.environ.get("LAMIN_ENV", "prod") == "local":
-        from fastapi.testclient import TestClient
-        from laminhub_rest.main import app
-
-        return TestClient(app)
-
-    import requests  # type: ignore
+@contextmanager
+def httpx_client():
+    client = None
+    try:
+        # local is used in tests
+        if os.environ.get("LAMIN_ENV", "prod") == "local":
+            from fastapi.testclient import TestClient
+            from laminhub_rest.main import app
 
-    return requests
+            client = TestClient(app)
+        else:
+            transport = RetryTransport(
+                retry=LogRetry(total=2, backoff_factor=0.2),
+                transport=httpx.HTTPTransport(verify=True, http2=True),
+            )
+            client = httpx.Client(transport=transport)
+        yield client
+    finally:
+        if client is not None:
+            client.close()
 
 
 def request_with_auth(
@@ -219,30 +262,27 @@ def request_with_auth(
     renew_token: bool = True,
     **kwargs,
 ):
-    requests = requests_client()
-
     headers = kwargs.pop("headers", {})
     headers["Authorization"] = f"Bearer {access_token}"
-
-    make_request = getattr(requests, method)
     timeout = kwargs.pop("timeout", DEFAULT_TIMEOUT)
 
-    response = make_request(url, headers=headers, timeout=timeout, **kwargs)
-    status_code = response.status_code
-    # update access_token and try again if failed
-    if not (200 <= status_code < 300) and renew_token:
-        from lamindb_setup import settings
-
-        logger.debug(f"{method} {url} failed: {status_code} {response.text}")
+    with httpx_client() as client:
+        make_request = getattr(client, method)
+        response = make_request(url, headers=headers, timeout=timeout, **kwargs)
+        status_code = response.status_code
+        # update access_token and try again if failed
+        if not (200 <= status_code < 300) and renew_token:
+            from lamindb_setup import settings
 
-        access_token = get_access_token(
-            settings.user.email, settings.user.password, settings.user.api_key
-        )
+            logger.debug(f"{method} {url} failed: {status_code} {response.text}")
 
-        settings.user.access_token = access_token
-        save_user_settings(settings.user)
+            access_token = get_access_token(
+                settings.user.email, settings.user.password, settings.user.api_key
+            )
 
-        headers["Authorization"] = f"Bearer {access_token}"
+            settings.user.access_token = access_token
+            save_user_settings(settings.user)
 
-        response = make_request(url, headers=headers, timeout=timeout, **kwargs)
+            headers["Authorization"] = f"Bearer {access_token}"
+            response = make_request(url, headers=headers, timeout=timeout, **kwargs)
     return response

lamindb_setup/core/_hub_core.py

@@ -36,7 +36,6 @@ from ._hub_utils import (
 from ._settings import settings
 from ._settings_instance import InstanceSettings
 from ._settings_storage import StorageSettings, base62, instance_uid_from_uuid
-from .hashing import hash_and_encode_as_b62
 
 if TYPE_CHECKING:
     from supabase import Client  # type: ignore
@@ -194,6 +193,8 @@ def _select_storage_by_settings(
 
 
 def _select_storage_or_parent(path: str, client: Client) -> dict | None:
+    # add get=True when we upgrade supabase
+    # because otherwise it uses POST which is not retryable
     result = client.rpc("existing_root_or_child", {"_path": path}).execute().data
     if result["root"] is None:
         return None
@@ -253,8 +254,7 @@ def _init_storage_hub(
     from lamindb_setup import settings
 
     created_by = settings.user._uuid if created_by is None else created_by
-    # storage roots are always stored without the trailing slash in the SQL
-    # database
+    # storage roots are always stored without the trailing slash in the SQL database
     root = ssettings.root_as_str
     if _select_storage_by_settings(ssettings, update_uid=True, client=client):
         return "hub-record-retrieved"
@@ -285,8 +285,7 @@ def _init_storage_hub(
         "is_default": is_default,
         "space_id": space_id.hex if space_id is not None else None,
     }
-    # TODO: add error message for violated unique constraint
-    # on root & description
+    # TODO: add error message for violated unique constraint on root & description
     client.table("storage").upsert(fields).execute()
     ssettings._uuid_ = id
     return "hub-record-created"
@@ -341,7 +340,7 @@ def _delete_instance(
             )
             # gate storage and instance deletion on empty storage location for
             # normally auth.get_session() doesn't have access_token
-            # so this block is useless i think (Sergei)
+            # so this block is useless I think (Sergei)
             # the token is received from user settings inside create_path
             # might be needed in the hub though
             if client.auth.get_session() is not None:
@@ -425,6 +424,7 @@ def _connect_instance_hub(
     owner: str,  # account_handle
     name: str,  # instance_name
     use_root_db_user: bool,
+    use_proxy_db: bool,
     client: Client,
 ) -> tuple[dict, dict] | str:
     response = client.functions.invoke(
@@ -456,7 +456,7 @@ def _connect_instance_hub(
                 )
     # no instance found, check why is that
     if response == b"{}":
-        # try the via single requests, will take more time
+        # try via separate requests, will take more time
         account = select_account_by_handle(owner, client)
         if account is None:
             return "account-not-exists"
@@ -500,12 +500,26 @@ def _connect_instance_hub(
                     db_user["name" if fine_grained_access else "db_user_name"],
                     db_user["password" if fine_grained_access else "db_user_password"],
                 )
+
+        if use_proxy_db:
+            host = instance.get("proxy_host", None)
+            assert host is not None, (
+                "Database proxy host is not available, please do not pass 'use_proxy_db'."
+            )
+            port = instance.get("proxy_port", None)
+            assert port is not None, (
+                "Database proxy port is not available, please do not pass 'use_proxy_db'."
+            )
+        else:
+            host = instance["db_host"]
+            port = instance["db_port"]
+
         db_dsn = LaminDsn.build(
             scheme=instance["db_scheme"],
             user=db_user_name if db_user_name is not None else "none",
             password=db_user_password if db_user_password is not None else "none",
-            host=instance["db_host"],
-            port=instance["db_port"],
+            host=host,
+            port=port,
             database=instance["db_database"],
         )
         instance["db"] = db_dsn
@@ -519,6 +533,7 @@ def connect_instance_hub(
     name: str,  # instance_name
     access_token: str | None = None,
     use_root_db_user: bool = False,
+    use_proxy_db: bool = False,
 ) -> tuple[dict, dict] | str:
     from ._settings import settings
 
@@ -528,6 +543,7 @@ def connect_instance_hub(
             owner=owner,
             name=name,
             use_root_db_user=use_root_db_user,
+            use_proxy_db=use_proxy_db,
             access_token=access_token,
         )
     else:
@@ -536,6 +552,7 @@ def connect_instance_hub(
             owner=owner,
             name=name,
             use_root_db_user=use_root_db_user,
+            use_proxy_db=use_proxy_db,
         )
 
 
@@ -610,7 +627,7 @@ def access_db(
     else:
         renew_token = False
     # local is used in tests
-    url = f"/access_v2/instances/{instance_id}/db_token"
+    url = f"/instances/{instance_id}/db_token"
     if os.environ.get("LAMIN_ENV", "prod") != "local":
         if instance_api_url is None:
             raise RuntimeError(

lamindb_setup/core/_settings.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import os
 import sys
-import warnings
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -226,8 +225,8 @@ class SetupSettings:
     def private_django_api(self) -> bool:
         """Turn internal Django API private to clean up the API (default `False`).
 
-        This patches your local pip-installed django installation. You can undo
-        the patch by setting this back to `False`.
+        This patches your local pip-installed django installation.
+        You can undo the patch by setting this back to `False`.
         """
         return self._private_django_api_path.exists()
 

lamindb_setup/core/_settings_instance.py

@@ -242,8 +242,8 @@ class InstanceSettings:
     def storage(self) -> StorageSettings:
         """Default storage of instance.
 
-        For a cloud instance, this is cloud storage. For a local instance, this
-        is a local directory.
+        For a cloud instance, this is cloud storage.
+        For a local instance, this is a local directory.
         """
         return self._storage  # type: ignore
 
@@ -350,8 +350,23 @@ class InstanceSettings:
 
         Use this URL for API calls related to this instance.
         """
+        if "LAMIN_API_URL" in os.environ:
+            return os.environ["LAMIN_API_URL"]
         return self._api_url
 
+    @property
+    def ui_url(self) -> str | None:
+        """URL for UI.
+
+        Use this URL for accessing the UI related to this instance.
+        """
+        if self.api_url is None:
+            return None
+        if "lamin.ai" in self.api_url:
+            return "https://lamin.ai"
+        else:
+            return self.api_url.replace("/api", "")
+
     @property
     def available_spaces(self) -> dict | None:
         """Available spaces with roles for instances fine-grained permissions.
@@ -461,7 +476,7 @@ class InstanceSettings:
                 )
             lock_msg += (
                 " The instance will be automatically unlocked after"
-                f" {int(EXPIRATION_TIME/3600/24)}d of no activity."
+                f" {int(EXPIRATION_TIME / 3600 / 24)}d of no activity."
             )
             raise InstanceLockedException(lock_msg)
 
@@ -501,8 +516,6 @@ class InstanceSettings:
 
     @property
     def _is_cloud_sqlite(self) -> bool:
-        # can we make this a private property, Sergei?
-        # as it's not relevant to the user
         """Is this a cloud instance with sqlite db."""
         return self.dialect == "sqlite" and self.storage.type_is_cloud
 
@@ -530,8 +543,8 @@ class InstanceSettings:
     def is_on_hub(self) -> bool:
         """Is this instance on the hub?
 
-        Can only reliably establish if user has access to the instance. Will
-        return `False` in case the instance isn't found.
+        Can only reliably establish if user has access to the instance.
+        Will return `False` in case the instance isn't found.
         """
         if self._is_on_hub is None:
             from ._hub_client import call_with_fallback_auth

lamindb_setup/core/_settings_save.py

@@ -41,9 +41,9 @@ def save_settings(
 ):
     with open(settings_file, "w") as f:
         for store_key, type_ in type_hints.items():
-            if type_ == Optional[str]:
+            if type_ == Optional[str]:  # noqa: UP045
                 type_ = str
-            if type_ == Optional[bool]:
+            if type_ == Optional[bool]:  # noqa: UP045
                 type_ = bool
             if "__" not in store_key:
                 if store_key == "model_config":

lamindb_setup/core/_settings_store.py

@@ -63,19 +63,19 @@ def system_settings_file():
 
 
 class InstanceSettingsStore(BaseSettings):
-    api_url: Optional[str] = None
+    api_url: str | None = None
     owner: str
     name: str
     storage_root: str
-    storage_region: Optional[str]  # take old type annotations here because pydantic
-    db: Optional[str]  # doesn't like new types on 3.9 even with future annotations
-    schema_str: Optional[str]
-    schema_id: Optional[str] = None
+    storage_region: str | None  # take old type annotations here because pydantic
+    db: str | None  # doesn't like new types on 3.9 even with future annotations
+    schema_str: str | None
+    schema_id: str | None = None
     fine_grained_access: bool = False
-    db_permissions: Optional[str] = None
+    db_permissions: str | None = None
     id: str
-    git_repo: Optional[str]
-    keep_artifacts_local: Optional[bool]
+    git_repo: str | None
+    keep_artifacts_local: bool | None
     model_config = SettingsConfigDict(env_prefix="lamindb_instance_", env_file=".env")
 
 
@@ -83,7 +83,7 @@ class UserSettingsStore(BaseSettings):
     email: str
     password: str
     access_token: str
-    api_key: Optional[str] = None
+    api_key: str | None = None
     uid: str
     uuid: str
     handle: str

lamindb_setup/core/upath.py

@@ -181,7 +181,7 @@ def print_hook(size: int, value: int, objectname: str, action: str):
         progress_in_percent = 100.0
     else:
         progress_in_percent = (value / size) * 100
-    out = f"... {action} {objectname}:" f" {min(progress_in_percent, 100):4.1f}%"
+    out = f"... {action} {objectname}: {min(progress_in_percent, 100):4.1f}%"
     if "NBPRJ_TEST_NBPATH" not in os.environ:
         end = "\n" if progress_in_percent >= 100 else "\r"
         print(out, end=end)
@@ -660,7 +660,7 @@ def view_tree(
         skip_suffixes: Skip directories with these suffixes.
 
     Examples:
-        >>> dir_path = ln.core.datasets.generate_cell_ranger_files(
+        >>> dir_path = ln.examples.datasets.generate_cell_ranger_files(
         >>>     "sample_001", ln.settings.storage
         >>> )
         >>> ln.UPath(dir_path).view_tree()
@@ -723,15 +723,12 @@ def to_url(upath):
 # Why aren't we subclassing?
 #
 # The problem is that UPath defines a type system of paths
-# Its __new__ method returns instances of different subclasses rather than a
-# UPath object
+# Its __new__ method returns instances of different subclasses rather than a UPath object
 # If we create a custom subclass naively, subclasses of the parent UPath won't
 # be subclasses of our custom subclass
-# This makes life really hard in type checks involving local to cloud
-# comparisons, etc.
+# This makes life really hard in type checks involving local to cloud comparisons, etc.
 # Hence, we extend the existing UPath and amend the docs
-# Some of this might end up in the original UPath implementation over time,
-# we'll see.
+# Some of this might end up in the original UPath implementation over time, we'll see.
 
 
 # add custom functions
@@ -903,7 +900,7 @@ def create_path(path: UPathStr, access_token: str | None = None) -> UPath:
             storage_options["client_kwargs"] = client_kwargs
         # see download_to for the reason
         if "use_listings_cache" not in upath.storage_options:
-            storage_options["use_listings_cache"] = True
+            storage_options["use_listings_cache"] = True  # type: ignore
         if len(storage_options) > 0:
             return UPath(upath, **storage_options)
     return upath
@@ -1014,7 +1011,7 @@ def check_storage_is_empty(
         if raise_error
         else "consider deleting them"
     )
-    message = f"'{directory_string}' contains {n_diff} objects" f" - {ask_for_deletion}"
+    message = f"'{directory_string}' contains {n_diff} objects - {ask_for_deletion}"
     if n_diff > 0:
         if raise_error:
             raise StorageNotEmpty(message) from None

{lamindb_setup-1.11.0.dist-info → lamindb_setup-1.12.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.11.0
+Version: 1.12.1
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10
@@ -10,10 +10,11 @@ Requires-Dist: django>=5.1,<5.2
 Requires-Dist: dj_database_url>=1.3.0,<3.0.0
 Requires-Dist: pydantic-settings
 Requires-Dist: platformdirs<5.0.0
+Requires-Dist: httpx_retries<1.0.0
 Requires-Dist: requests
 Requires-Dist: universal_pathlib==0.2.6
 Requires-Dist: botocore<2.0.0
-Requires-Dist: supabase>=2.8.1,<=2.15.0
+Requires-Dist: supabase>=2.8.1,<=2.16.0
 Requires-Dist: gotrue<=2.12.0
 Requires-Dist: storage3!=0.11.2; python_version < '3.11'
 Requires-Dist: pyjwt<3.0.0

{lamindb_setup-1.11.0.dist-info → lamindb_setup-1.12.1.dist-info}/RECORD

@@ -1,19 +1,19 @@
-lamindb_setup/__init__.py,sha256=iSKoEHzHwBMbBer5p_FuUPHJndK2eX3pfc6avRWqSaY,2783
+lamindb_setup/__init__.py,sha256=lH8RfaY2XMMqOwAr4mTa8B-FS1rwxNo3aslU6S7g2wg,3112
 lamindb_setup/_cache.py,sha256=pGvDNVHGx4HWr_6w5ajqEJOdysmaGc6F221qFnXkT-k,2747
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=ToKMxsUq8dQBQh8baOrNVlSb1iC8h4zTg5dV8wMu0W4,6760
-lamindb_setup/_connect_instance.py,sha256=BWQplcUNZ-hCLDwB1xUPtBrvcmLAl9-5-I5yfKc2tXE,17536
-lamindb_setup/_delete.py,sha256=4kS-_nQrV5xMvZE3BGCNEEGCboyGmqqDMXlckF0GxSk,5780
+lamindb_setup/_connect_instance.py,sha256=OSOGZEj_9JsLvZT9JsXGvbC96DY4akmbAOuqUKyjjiE,17717
+lamindb_setup/_delete.py,sha256=KS3r-xGFuDmAbzPUy-9JR-YnPShYdaHjDRQrAmXQ0qM,5863
 lamindb_setup/_disconnect.py,sha256=FT8EpCm5XXDdhDH7QtAnkO3KPatq2HqT9VXGNjgJDbk,1232
 lamindb_setup/_django.py,sha256=uIQflpkp8l3axyPaKURlk3kacgpElVP5KOKmFxYSMGk,1454
 lamindb_setup/_entry_points.py,sha256=sKwXPX9xjOotoAjvgkU5LBwjjHLWVkh0ZGdiSsrch9k,522
 lamindb_setup/_exportdb.py,sha256=QLjoH4dEwqa01A12naKaDPglCCzl2_VLKWFfJRE_uSg,2113
 lamindb_setup/_importdb.py,sha256=fKv9ev5OOj_-bmzC8XZ1GxOcjIjI486yrHSHDWQrJeI,1874
-lamindb_setup/_init_instance.py,sha256=5FRECcB_wuQ9pFkQctooyI6fJV0ItpVaBOHlL5EjfIk,14990
-lamindb_setup/_migrate.py,sha256=O84O2QLzLrXOK6ogeK4Nxtt-M2oLPUuZ8lKYo2pCcgU,10795
+lamindb_setup/_init_instance.py,sha256=8ejD6zjV0eF7KR-DvnmDAVJb9Ty0hjaPtIkFbyLDvA0,14806
+lamindb_setup/_migrate.py,sha256=aOWE13LJOW55mC4QiYeCS5bJGSTRsRZPpUYz6e_xoFs,10773
 lamindb_setup/_register_instance.py,sha256=RdUZxZWHLdbvdNZWpF8e0UWROb_T0cStWbzc5yUw34I,1047
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
-lamindb_setup/_schema_metadata.py,sha256=VWO03tYBoU_rZF2eI2a5HSMPzpDS4kCE0Ul3fpvnSTA,15132
+lamindb_setup/_schema_metadata.py,sha256=At_EAE9mMzMJIJ1mfiOZYXVgBaXRkWUW6a3fLz5Z_lY,15132
 lamindb_setup/_set_managed_storage.py,sha256=y5YQASsWNYVWUYeLgh3N2YBETYP7mBtbpxe3X_Vgb5I,2699
 lamindb_setup/_setup_user.py,sha256=DapdzT3u0f5LN5W9W9A6PWw-n8ejcJciQtHN9b5lidA,5889
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
@@ -21,21 +21,21 @@ lamindb_setup/errors.py,sha256=qZTfSL0rpbY8AIG-Z4-3-_EbLW5zyo2CFEJrVU02-3A,1863
 lamindb_setup/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lamindb_setup/types.py,sha256=XlXLb4nmbc68uBj5Hp3xpDRezYGJIBZv6jAAqqN0p10,614
 lamindb_setup/core/__init__.py,sha256=5M4A6CVHBO_T5Rr9MeLaPW3WTk4-y00cgRYEgUJVU5U,410
-lamindb_setup/core/_aws_options.py,sha256=UbwnaEX2KcZBDOa2W2NbH62cNrKJQ78Hrn7zcszhQvU,8092
-lamindb_setup/core/_aws_storage.py,sha256=ofPTHXvF97I9eUCHxj5RPpUUqAcNV0VsPWpyKMcshOI,2030
+lamindb_setup/core/_aws_options.py,sha256=SadUhcLCRtvsy3Qvx6799Iv4_CJkb1rwWB6d5GIixHc,8080
+lamindb_setup/core/_aws_storage.py,sha256=QEtV-riQrwfivcwqHnXBbkJ-9YyNEXL4fLoCmOHZ1BI,2003
 lamindb_setup/core/_deprecated.py,sha256=M3vpM4fZPOncxY2qsXQAPeaEph28xWdv7tYaueaUyAA,2554
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
-lamindb_setup/core/_hub_client.py,sha256=Pl3sEG2rasdJp4Rh5HNY0VsPVls-nfY0OxD5QzrYF9A,8678
-lamindb_setup/core/_hub_core.py,sha256=6VNeREgFGjJ6nMNBlhbylV28oHZeoT0BlXL6dXzn9LU,26628
+lamindb_setup/core/_hub_client.py,sha256=J0x43at0zb0yWP-RoT2lyqaHV66ewUP3OiYVYQCjxe8,9974
+lamindb_setup/core/_hub_core.py,sha256=axnNugfAehXIB_GLJoJE2zQfsAfPNEw5UELnUvTLWHY,27279
 lamindb_setup/core/_hub_crud.py,sha256=j6516H82kLjFUNPqFGUINbDw9YbofMgjxadGzYb0OS4,6362
 lamindb_setup/core/_hub_utils.py,sha256=6dyDGyzYFgVfR_lE3VN3CP1jGp98gxPtr-T91PAP05U,2687
 lamindb_setup/core/_private_django_api.py,sha256=By63l3vIEtK1pq246FhHq3tslxsaTJGKm5VakYluWp4,2656
-lamindb_setup/core/_settings.py,sha256=s2wB0vMeR0D36YCtwMSbyfYj6R05cQc0JYO1SH89bqc,14033
-lamindb_setup/core/_settings_instance.py,sha256=P2O2RWOSx2OUuU2nJBaD8FV6EIZYvUGGqTeVabIBKsA,23325
+lamindb_setup/core/_settings.py,sha256=0nz3HKnBuXdDY4R2UJQts3ZVC7vROpsBAxWIgZNz800,14017
+lamindb_setup/core/_settings_instance.py,sha256=oLRH_BlQ6EshZtpXZ9v4qk_2bUZDG5Km4qjqpII3EPA,23675
 lamindb_setup/core/_settings_load.py,sha256=j20cy3J56ZBHLDfB2A8oKjekNetMNsy0_W3eWD36pWI,5161
-lamindb_setup/core/_settings_save.py,sha256=XZx-vow7BT6y3JpRBB2UOJp2vwc7jOGea4wSgOPqjPU,3262
+lamindb_setup/core/_settings_save.py,sha256=jh412jXIAbIYvnSoW9riBFePRAa4vmPm-ScYD0smlnw,3292
 lamindb_setup/core/_settings_storage.py,sha256=pyU25hP5rQYjVe0tFPR8P6TzAYzu1NpT-PIbXoxfV18,15348
-lamindb_setup/core/_settings_store.py,sha256=QmeWIGdIyq7UmjfHiEB_0xRD8hY-8-ZR2WntIKfwTKI,2714
+lamindb_setup/core/_settings_store.py,sha256=ykJeBA9IODK4G_jrfBE9pb0c1xkfePkARPpb306DT08,2687
 lamindb_setup/core/_settings_user.py,sha256=gFfyMf-738onbh1Mf4wsmLlenQJPtjQfpUgKnOlqc2o,1453
 lamindb_setup/core/_setup_bionty_sources.py,sha256=ox3X-SHiHa2lNPSWjwZhINypbLacX6kGwH6hVVrSFZc,1505
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=H_CTUCjURFXwD1cCtV_Jn0_60iztZTkaesLLXIBgIxc,7204
@@ -43,8 +43,8 @@ lamindb_setup/core/django.py,sha256=kV8W3WZy5Rkhn4rDJv2GNoq8JYvX_8dLBHhDRZdSgwE,
 lamindb_setup/core/exceptions.py,sha256=qjMzqy_uzPA7mCOdnoWnS_fdA6OWbdZGftz-YYplrY0,84
 lamindb_setup/core/hashing.py,sha256=Y8Uc5uSGTfU6L2R_gb5w8DdHhGRog7RnkK-e9FEMjPY,3680
 lamindb_setup/core/types.py,sha256=T7NwspfRHgIIpYsXDcApks8jkOlGeGRW-YbVLB7jNIo,67
-lamindb_setup/core/upath.py,sha256=5mrAeAyYWXn1zJlwQmBM-NfXhpi2S-YsbEkXo_UESk0,36017
-lamindb_setup-1.11.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-1.11.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-lamindb_setup-1.11.0.dist-info/METADATA,sha256=gvtfSb-3uOq8Vl0buE_koqO1zGu2B22kmcBNjmrChbQ,1804
-lamindb_setup-1.11.0.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=uk3LpDA7Jbk1GzUb8hCsxByg5cMYTjPusIvwyXe8g3Y,36023
+lamindb_setup-1.12.1.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-1.12.1.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
+lamindb_setup-1.12.1.dist-info/METADATA,sha256=RzkBLCAlLcUzJtjzWutWxXI75Q81ZQuuVYWtHwvZXcw,1839
+lamindb_setup-1.12.1.dist-info/RECORD,,