lamindb_setup 1.8.1__tar.gz → 1.8.3__tar.gz

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.8.1
+Version: 1.8.3
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/docs/hub-cloud/06-connect-hosted-instance.ipynb

@@ -16,6 +16,7 @@
     "import pytest\n",
     "import shutil\n",
     "import lamindb_setup as ln_setup\n",
+    "from lamindb_setup._connect_instance import _connect_cli\n",
     "from lamindb_setup.core.upath import UPath\n",
     "from lamindb_setup.core._hub_core import delete_instance\n",
     "from lamindb_setup.core._hub_client import connect_hub_with_auth\n",
@@ -31,7 +32,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln_setup.connect(f\"testuser1/{instance_name}\")"
+    "_connect_cli(f\"testuser1/{instance_name}\")  # cover here for cloud sqlite"
    ]
   },
   {
@@ -223,7 +224,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.10.16"
   }
  },
  "nbformat": 4,

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/docs/hub-cloud/07-keep-artifacts-local.ipynb

@@ -48,7 +48,7 @@
     "    ln_setup.settings.instance.local_storage\n",
     "assert (\n",
     "    error.exconly()\n",
-    "    == \"ValueError: `keep_artifacts_local` is not enabled for this instance.\"\n",
+    "    == \"ValueError: `keep_artifacts_local` is False, switch via: ln.setup.settings.instance.keep_artifacts_local = True\"\n",
     ")"
    ]
   },
@@ -58,12 +58,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln_setup.settings.instance._keep_artifacts_local = True\n",
+    "ln_setup.settings.instance.keep_artifacts_local = True\n",
     "with pytest.raises(ValueError) as error:\n",
     "    ln_setup.settings.instance.local_storage\n",
-    "assert (\n",
-    "    error.exconly()\n",
-    "    == \"ValueError: No storage location found in current environment: create one via, e.g., ln.Storage(root='/dir/our_shared_dir', host='our-server-123').save()\"\n",
+    "assert error.exconly().startswith(\n",
+    "    \"ValueError: No local storage location found in current environment:\"\n",
     ")"
    ]
   },

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/docs/hub-prod/test-cloud-sync.ipynb

@@ -229,7 +229,7 @@
     ")\n",
     "assert dir_sync_local == settings.cache_dir / \"dir_cache/key\"\n",
     "\n",
-    "assert dir_sync.synchronize(dir_sync_local, just_check=True)\n",
+    "assert dir_sync.synchronize_to(dir_sync_local, just_check=True)\n",
     "assert not dir_sync_local.exists()"
    ]
   },
@@ -240,7 +240,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert dir_sync.synchronize(dir_sync_local, just_check=False)\n",
+    "assert dir_sync.synchronize_to(dir_sync_local, just_check=False)\n",
     "assert dir_sync_local.is_dir()\n",
     "assert num_files(dir_sync_local) == 2"
    ]
@@ -308,6 +308,28 @@
     "http_local.unlink()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7e610d67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# test download_to\n",
+    "http_path.download_to(http_local, print_progress=True)\n",
+    "assert not http_path.synchronize_to(http_local)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1e293d0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "http_local.unlink()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "574c3f95",
@@ -323,7 +345,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "dir_sync_local = settings.paths.cloud_to_local(dir_sync)"
+    "dir_sync_local = settings.paths.cloud_to_local(dir_sync)\n",
+    "assert not dir_sync.synchronize_to(dir_sync_local)"
    ]
   },
   {
@@ -459,7 +482,7 @@
    "source": [
     "dir_sync_local = settings.paths.cloud_to_local(dir_sync)\n",
     "\n",
-    "assert num_files(dir_sync_local) == 2\n",
+    "assert num_files(dir_sync_local) == 2, list(dir_sync_local.rglob(\"*\"))\n",
     "assert not local_file_new.exists()\n",
     "assert not local_file_new_parent.exists()\n",
     "\n",
@@ -759,7 +782,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert hf_path.synchronize(hf_path_local, just_check=True)\n",
+    "assert hf_path.synchronize_to(hf_path_local, just_check=True)\n",
     "assert not hf_path_local.exists()"
    ]
   },
@@ -770,7 +793,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert hf_path.synchronize(hf_path_local)\n",
+    "assert hf_path.synchronize_to(hf_path_local)\n",
     "assert hf_path_local.is_file()"
    ]
   },
@@ -793,7 +816,7 @@
    "source": [
     "hf_path = UPath(\"hf://datasets/Koncopd/lamindb-test@main/does_not_exist.file\")\n",
     "with pytest.raises(FileNotFoundError):\n",
-    "    hf_path.synchronize(UPath(\"./does_not_exist.file\"), error_no_origin=True)"
+    "    hf_path.synchronize_to(UPath(\"./does_not_exist.file\"), error_no_origin=True)"
    ]
   },
   {

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.8.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.8.3"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/lamindb_setup/_check_setup.py

@@ -147,6 +147,9 @@ def _check_instance_setup(from_module: str | None = None) -> bool:
             else:
                 django_lamin.setup_django(isettings)
                 logger.important(f"connected lamindb: {isettings.slug}")
+                settings._instance_settings = (
+                    isettings  # update of local storage location
+                )
         return django_lamin.IS_SETUP
     else:
         if from_module is not None and settings.auto_connect:

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/lamindb_setup/_connect_instance.py

@@ -188,13 +188,32 @@ def _connect_instance(
     return isettings
 
 
+def _connect_cli(instance: str) -> None:
+    from lamindb_setup import settings as settings_
+
+    settings_.auto_connect = True
+    owner, name = get_owner_name_from_identifier(instance)
+    isettings = _connect_instance(owner, name)
+    isettings._persist(write_to_disk=True)
+    if not isettings.is_on_hub or isettings._is_cloud_sqlite:
+        # there are two reasons to call the full-blown connect
+        # (1) if the instance is not on the hub, we need to register
+        # potential users through register_user()
+        # (2) if the instance is cloud sqlite, we need to lock it
+        connect(_write_settings=False, _reload_lamindb=False)
+    else:
+        logger.important(f"connected lamindb: {isettings.slug}")
+    return None
+
+
 @unlock_cloud_sqlite_upon_exception(ignore_prev_locker=True)
 def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
     """Connect to an instance.
 
     Args:
         instance: Pass a slug (`account/name`) or URL (`https://lamin.ai/account/name`).
-            If `None`, looks for an environment variable `LAMIN_CURRENT_INSTANCE` to get the instance identifier. If it doesn't find this variable, it connects to the instance that was connected with `lamin connect` through the CLI.
+            If `None`, looks for an environment variable `LAMIN_CURRENT_INSTANCE` to get the instance identifier.
+            If it doesn't find this variable, it connects to the instance that was connected with `lamin connect` through the CLI.
     """
     # validate kwargs
     valid_kwargs = {

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/lamindb_setup/core/_aws_options.py

@@ -172,11 +172,7 @@ class AWSOptionsManager:
             from ._hub_core import access_aws
             from ._settings import settings
 
-            if settings.user.handle != "anonymous" or access_token is not None:
-                storage_root_info = access_aws(path_str, access_token=access_token)
-            else:
-                storage_root_info = {"credentials": {}, "accessibility": {}}
-
+            storage_root_info = access_aws(path_str, access_token=access_token)
             accessibility = storage_root_info["accessibility"]
             is_managed = accessibility.get("is_managed", False)
             if is_managed:

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/lamindb_setup/core/_hub_client.py

@@ -5,6 +5,7 @@ import os
 from typing import Literal
 from urllib.request import urlretrieve
 
+from httpx import HTTPTransport
 from lamin_utils import logger
 from pydantic_settings import BaseSettings
 from supabase import Client, create_client  # type: ignore
@@ -60,20 +61,29 @@ class Environment:
         self.supabase_anon_key: str = key
 
 
+DEFAULT_TIMEOUT = 20
+
+
 # runs ~0.5s
 def connect_hub(
     fallback_env: bool = False, client_options: ClientOptions | None = None
 ) -> Client:
     env = Environment(fallback=fallback_env)
     if client_options is None:
-        # function_client_timeout=5 by default
-        # increase to avoid rare timeouts for edge functions
         client_options = ClientOptions(
             auto_refresh_token=False,
-            function_client_timeout=30,
-            postgrest_client_timeout=20,
+            function_client_timeout=DEFAULT_TIMEOUT,
+            postgrest_client_timeout=DEFAULT_TIMEOUT,
         )
-    return create_client(env.supabase_api_url, env.supabase_anon_key, client_options)
+    client = create_client(env.supabase_api_url, env.supabase_anon_key, client_options)
+    # needed to enable retries for http requests in supabase
+    # these are separate clients and need separate transports
+    # retries are done only in case an httpx.ConnectError or an httpx.ConnectTimeout occurs
+    transport_kwargs = {"verify": True, "http2": True, "retries": 2}
+    client.auth._http_client._transport = HTTPTransport(**transport_kwargs)
+    client.functions._client._transport = HTTPTransport(**transport_kwargs)
+    client.postgrest.session._transport = HTTPTransport(**transport_kwargs)
+    return client
 
 
 def connect_hub_with_auth(
@@ -210,11 +220,16 @@ def request_with_auth(
     headers["Authorization"] = f"Bearer {access_token}"
 
     make_request = getattr(requests, method)
-    response = make_request(url, headers=headers, **kwargs)
-    # upate access_token and try again if failed
-    if response.status_code != 200 and renew_token:
+    timeout = kwargs.pop("timeout", DEFAULT_TIMEOUT)
+
+    response = make_request(url, headers=headers, timeout=timeout, **kwargs)
+    status_code = response.status_code
+    # update access_token and try again if failed
+    if not (200 <= status_code < 300) and renew_token:
         from lamindb_setup import settings
 
+        logger.debug(f"{method} {url} failed: {status_code} {response.text}")
+
         access_token = get_access_token(
             settings.user.email, settings.user.password, settings.user.api_key
         )
@@ -224,5 +239,5 @@ def request_with_auth(
 
         headers["Authorization"] = f"Bearer {access_token}"
 
-        response = make_request(url, headers=headers, **kwargs)
+        response = make_request(url, headers=headers, timeout=timeout, **kwargs)
     return response

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/lamindb_setup/core/_hub_core.py

@@ -460,9 +460,9 @@ def access_aws(storage_root: str, access_token: str | None = None) -> dict[str,
         storage_root_info = call_with_fallback_auth(
             _access_aws, storage_root=storage_root, access_token=access_token
         )
-        return storage_root_info
     else:
-        raise RuntimeError("Can only get access to AWS if authenticated.")
+        storage_root_info = call_with_fallback(_access_aws, storage_root=storage_root)
+    return storage_root_info
 
 
 def _access_aws(*, storage_root: str, client: Client) -> dict[str, dict]:
@@ -497,8 +497,8 @@ def access_db(
     instance_slug: str
     instance_api_url: str | None
     if (
-        "LAMIN_TEST_DB_TOKEN" in os.environ
-        and (env_db_token := os.environ["LAMIN_TEST_DB_TOKEN"]) != ""
+        "LAMIN_DB_TOKEN" in os.environ
+        and (env_db_token := os.environ["LAMIN_DB_TOKEN"]) != ""
     ):
         return env_db_token
 
@@ -531,11 +531,13 @@ def access_db(
         url = instance_api_url + url
 
     response = request_with_auth(url, "get", access_token, renew_token)  # type: ignore
-    response_json = response.json()
-    if response.status_code != 200:
+    status_code = response.status_code
+    if not (200 <= status_code < 300):
         raise PermissionError(
-            f"Fine-grained access to {instance_slug} failed: {response_json}"
+            f"Fine-grained access to {instance_slug} failed: {status_code} {response.text}"
         )
+
+    response_json = response.json()
     if "token" not in response_json:
         raise RuntimeError("The response of access_db does not contain a db token.")
     return response_json["token"]

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/lamindb_setup/core/_settings.py

@@ -350,7 +350,7 @@ class SetupPaths:
         local_filepath = SetupPaths.cloud_to_local_no_update(filepath, cache_key)
         if not isinstance(filepath, LocalPathClasses):
             local_filepath.parent.mkdir(parents=True, exist_ok=True)
-            filepath.synchronize(local_filepath, **kwargs)  # type: ignore
+            filepath.synchronize_to(local_filepath, **kwargs)  # type: ignore
         return local_filepath
 
 

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/lamindb_setup/core/_settings_instance.py

@@ -32,7 +32,7 @@ if TYPE_CHECKING:
 
     from ._settings_user import UserSettings
 
-LOCAL_STORAGE_MESSAGE = "No storage location found in current environment: create one via, e.g., ln.Storage(root='/dir/our_shared_dir', host='our-server-123').save()"
+LOCAL_STORAGE_MESSAGE = "No local storage location found in current environment: defaulting to cloud storage"
 
 
 def sanitize_git_repo_url(repo_url: str) -> str:
@@ -104,8 +104,20 @@ class InstanceSettings:
         for attr in attrs:
             value = getattr(self, attr)
             if attr == "storage":
-                representation += f"\n - storage root: {value.root_as_str}"
-                representation += f"\n - storage region: {value.region}"
+                if self.keep_artifacts_local:
+                    import lamindb as ln
+
+                    self._local_storage = ln.setup.settings.instance._local_storage
+                if self._local_storage is not None:
+                    value_local = self.local_storage
+                    representation += f"\n - local storage: {value_local.root_as_str} ({value_local.region})"
+                    representation += (
+                        f"\n - cloud storage: {value.root_as_str} ({value.region})"
+                    )
+                else:
+                    representation += (
+                        f"\n - storage: {value.root_as_str} ({value.region})"
+                    )
             elif attr == "db":
                 if self.dialect != "sqlite":
                     model = LaminDsnModel(db=value)
@@ -156,9 +168,17 @@ class InstanceSettings:
         found = []
         for record in all_local_records:
             root_path = Path(record.root)
-            if root_path.exists():
+            try:
+                root_path_exists = root_path.exists()
+            except PermissionError:
+                continue
+            if root_path_exists:
                 marker_path = root_path / STORAGE_UID_FILE_KEY
-                if not marker_path.exists():
+                try:
+                    marker_path_exists = marker_path.exists()
+                except PermissionError:
+                    continue
+                if not marker_path_exists:
                     legacy_filepath = root_path / LEGACY_STORAGE_UID_FILE_KEY
                     if legacy_filepath.exists():
                         logger.warning(
@@ -182,8 +202,9 @@ class InstanceSettings:
             if len(found) > 1:
                 found_display = "\n - ".join([f"{record.root}" for record in found])
                 logger.important(f"found locations:\n - {found_display}")
+            record = found[0]
             logger.important(f"defaulting to local storage: {record.root}")
-            return StorageSettings(record.root)
+            return StorageSettings(record.root, region=record.region)
         elif not mute_warning:
             start = LOCAL_STORAGE_MESSAGE[0].lower()
             logger.warning(f"{start}{LOCAL_STORAGE_MESSAGE[1:]}")
@@ -193,15 +214,19 @@ class InstanceSettings:
     def keep_artifacts_local(self) -> bool:
         """Default to keeping artifacts local.
 
-        Enable this optional setting for cloud instances on lamin.ai.
-
         Guide: :doc:`faq/keep-artifacts-local`
         """
         return self._keep_artifacts_local
 
+    @keep_artifacts_local.setter
+    def keep_artifacts_local(self, value: bool):
+        if not isinstance(value, bool):
+            raise ValueError("keep_artifacts_local must be a boolean value.")
+        self._keep_artifacts_local = value
+
     @property
     def storage(self) -> StorageSettings:
-        """Default storage.
+        """Default storage of instance.
 
         For a cloud instance, this is cloud storage. For a local instance, this
         is a local directory.
@@ -210,14 +235,16 @@ class InstanceSettings:
 
     @property
     def local_storage(self) -> StorageSettings:
-        """An additional local storage location.
+        """An alternative default local storage location in the current environment.
 
-        Is only available if :attr:`keep_artifacts_local` is enabled.
+        Serves as the default storage location if :attr:`keep_artifacts_local` is enabled.
 
         Guide: :doc:`faq/keep-artifacts-local`
         """
-        if not self._keep_artifacts_local:
-            raise ValueError("`keep_artifacts_local` is not enabled for this instance.")
+        if not self.keep_artifacts_local:
+            raise ValueError(
+                "`keep_artifacts_local` is False, switch via: ln.setup.settings.instance.keep_artifacts_local = True"
+            )
         if self._local_storage is None:
             self._local_storage = self._search_local_root()
         if self._local_storage is None:
@@ -235,7 +262,7 @@ class InstanceSettings:
             local_root, host = local_root_host
 
         local_root = Path(local_root)
-        if not self._keep_artifacts_local:
+        if not self.keep_artifacts_local:
             raise ValueError("`keep_artifacts_local` is not enabled for this instance.")
         local_storage = self._search_local_root(
             local_root=StorageSettings(local_root).root_as_str, mute_warning=True
@@ -370,7 +397,7 @@ class InstanceSettings:
                 self._check_sqlite_lock()
             sqlite_file = self._sqlite_file
             cache_file = self.storage.cloud_to_local_no_update(sqlite_file)
-            sqlite_file.synchronize(cache_file, print_progress=True)  # type: ignore
+            sqlite_file.synchronize_to(cache_file, print_progress=True)  # type: ignore
 
     def _check_sqlite_lock(self):
         if not self._cloud_sqlite_locker.has_lock:

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/lamindb_setup/core/_settings_storage.py

@@ -19,7 +19,13 @@ from ._aws_options import (
 from ._aws_storage import find_closest_aws_region
 from ._deprecated import deprecated
 from .hashing import hash_and_encode_as_b62
-from .upath import LocalPathClasses, UPath, _split_path_query, create_path
+from .upath import (
+    LocalPathClasses,
+    UPath,
+    _split_path_query,
+    create_path,
+    get_storage_region,
+)
 
 if TYPE_CHECKING:
     from lamindb_setup.types import StorageType, UPathStr
@@ -43,50 +49,6 @@ def instance_uid_from_uuid(instance_id: UUID) -> str:
     return hash_and_encode_as_b62(instance_id.hex)[:12]
 
 
-def get_storage_region(path: UPathStr) -> str | None:
-    path_str = str(path)
-    if path_str.startswith("s3://"):
-        import botocore.session
-        from botocore.config import Config
-        from botocore.exceptions import ClientError
-
-        # check for endpoint_url in storage options if upath
-        if isinstance(path, UPath):
-            endpoint_url = path.storage_options.get("endpoint_url", None)
-        else:
-            endpoint_url = None
-        path_part = path_str.replace("s3://", "")
-        # check for endpoint_url in the path string
-        if "?" in path_part:
-            assert endpoint_url is None
-            path_part, query = _split_path_query(path_part)
-            endpoint_url = query.get("endpoint_url", [None])[0]
-        bucket = path_part.split("/")[0]
-        session = botocore.session.get_session()
-        credentials = session.get_credentials()
-        if credentials is None or credentials.access_key is None:
-            config = Config(signature_version=botocore.session.UNSIGNED)
-        else:
-            config = None
-        s3_client = session.create_client(
-            "s3", endpoint_url=endpoint_url, config=config
-        )
-        try:
-            response = s3_client.head_bucket(Bucket=bucket)
-        except ClientError as exc:
-            response = getattr(exc, "response", {})
-            if response.get("Error", {}).get("Code") == "404":
-                raise exc
-        region = (
-            response.get("ResponseMetadata", {})
-            .get("HTTPHeaders", {})
-            .get("x-amz-bucket-region", None)
-        )
-    else:
-        region = None
-    return region
-
-
 def get_storage_type(root_as_str: str) -> StorageType:
     import fsspec
 

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/lamindb_setup/core/_settings_user.py

@@ -39,7 +39,7 @@ class UserSettings:
     def __repr__(self) -> str:
         """Rich string representation."""
         representation = "Current user:"
-        attrs = ["handle", "email", "uid"]
+        attrs = ["handle", "uid"]
         for attr in attrs:
             value = getattr(self, attr)
             representation += f"\n - {attr}: {value}"

{lamindb_setup-1.8.1 → lamindb_setup-1.8.3}/lamindb_setup/core/django.py

@@ -247,7 +247,7 @@ def setup_django(
     IS_SETUP = True
 
     if isettings.keep_artifacts_local:
-        isettings._search_local_root()
+        isettings._local_storage = isettings._search_local_root()
 
 
 # THIS IS NOT SAFE