lamindb_setup 0.77.6__tar.gz → 0.78.0__tar.gz

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/PKG-INFO

@@ -1,8 +1,9 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.77.6
+Version: 0.78.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: lnschema_core>=0.51.0
 Requires-Dist: lamin_utils>=0.3.3
@@ -11,7 +12,7 @@ Requires-Dist: dj_database_url>=1.3.0,<3.0.0
 Requires-Dist: pydantic-settings
 Requires-Dist: appdirs<2.0.0
 Requires-Dist: requests
-Requires-Dist: universal_pathlib==0.2.2
+Requires-Dist: universal_pathlib==0.2.5
 Requires-Dist: botocore<2.0.0
 Requires-Dist: gotrue>=2.1.0,<2.9.0
 Requires-Dist: supabase==2.2.1

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/docs/hub-cloud/02-connect-local-instance.ipynb

@@ -19,10 +19,10 @@
    },
    "outputs": [],
    "source": [
-    "!lamin load --unload\n",
+    "!lamin disconnect\n",
     "!lamin delete --force mydata\n",
     "!lamin init --storage mydata\n",
-    "!lamin load --unload"
+    "!lamin disconnect"
    ]
   },
   {

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/docs/hub-cloud/03-add-managed-storage.ipynb

@@ -21,7 +21,7 @@
    },
    "outputs": [],
    "source": [
-    "!lamin load --unload\n",
+    "!lamin disconnect\n",
     "!lamin login testuser1\n",
     "!lamin delete --force test-add-managed-storage\n",
     "!docker stop pgtest && docker rm pgtest"

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/docs/hub-prod/test-cache-management.ipynb

@@ -15,7 +15,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import os\n",
     "import lamindb_setup as ln_setup\n",
+    "from lamindb_setup._cache import set_cache_dir\n",
     "from lamindb_setup.core._settings_store import system_storage_settings_file\n",
     "\n",
     "ln_setup.login(\"testuser2\")"
@@ -67,8 +69,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "save_cache_dir = ln_setup.settings.storage.cache_dir\n",
-    "set_cache_dir = save_cache_dir.parent / \"Cache1\""
+    "old_cache_dir = ln_setup.settings.cache_dir\n",
+    "new_cache_dir = old_cache_dir.parent / \"Cache1\""
    ]
   },
   {
@@ -78,7 +80,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln_setup.settings.storage.cache_dir = set_cache_dir"
+    "set_cache_dir(new_cache_dir)"
    ]
   },
   {
@@ -94,21 +96,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "df1a1396",
+   "id": "c1958c5b",
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert ln_setup.settings.storage.cache_dir != save_cache_dir"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "5b5ca8a8",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "assert ln_setup.settings.storage.cache_dir == set_cache_dir"
+    "assert ln_setup.settings.cache_dir == new_cache_dir"
    ]
   },
   {
@@ -118,7 +110,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert set_cache_dir.exists()"
+    "assert new_cache_dir.exists()"
    ]
   },
   {
@@ -128,7 +120,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert set_cache_dir in ln_setup.settings.instance._sqlite_file_local.parents"
+    "assert new_cache_dir in ln_setup.settings.instance._sqlite_file_local.parents"
    ]
   },
   {
@@ -156,53 +148,35 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert not any(set_cache_dir.iterdir())"
+    "assert not any(new_cache_dir.iterdir())"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "14ffa755",
-   "metadata": {},
-   "source": [
-    "Check that the cache dir is preserved between`close` and `load`."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "a05bb5a2",
+   "id": "e2aeabb4",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "# ln_setup.connect(\"test-cache\")  # this isn't possible"
+    "Setting to `\"null\"` or `None` restores the default directory."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "4f1c8dbe",
+   "id": "95aec7fd",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# assert ln_setup.settings.storage.cache_dir == set_cache_dir"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "e2aeabb4",
-   "metadata": {},
-   "source": [
-    "Setting to `\"null\"` or `None` restores the default directory."
+    "set_cache_dir(\"null\")"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "95aec7fd",
+   "id": "f1923e4d",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# ln_setup.settings.storage.cache_dir = None"
+    "assert ln_setup.settings.cache_dir == old_cache_dir"
    ]
   },
   {
@@ -212,7 +186,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# assert ln_setup.settings.storage.cache_dir == save_cache_dir"
+    "set_cache_dir(None)"
    ]
   },
   {
@@ -222,7 +196,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# assert ln_setup.settings.storage.cache_dir != set_cache_dir"
+    "assert ln_setup.settings.cache_dir == old_cache_dir"
    ]
   },
   {
@@ -248,7 +222,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!lamin cache"
+    "exit_status = os.system(\"lamin cache get\")\n",
+    "assert exit_status == 0"
    ]
   },
   {
@@ -258,7 +233,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!lamin cache set \"null\""
+    "exit_status = os.system(\"lamin cache set 'null'\")\n",
+    "assert exit_status == 0"
    ]
   },
   {
@@ -297,7 +273,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.9.17"
   }
  },
  "nbformat": 4,

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/docs/hub-prod/test-cloud-sync.ipynb

@@ -18,7 +18,7 @@
     "import os\n",
     "\n",
     "instance_name = f\"test-sqlite-sync\"\n",
-    "!lamin load {instance_name}\n",
+    "!lamin connect {instance_name}\n",
     "!yes | lamin delete {instance_name}"
    ]
   },
@@ -130,8 +130,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert settings.storage.cloud_to_local_no_update(dir_sync) == settings.storage.cache_dir / f\"lamindb-ci/{instance_name}/dir_sync\"\n",
-    "assert settings.storage.cloud_to_local_no_update(dir_sync, cache_key=\"dir_cache/key\") == settings.storage.cache_dir / \"dir_cache/key\""
+    "assert settings.storage.cloud_to_local_no_update(dir_sync) == settings.cache_dir / f\"lamindb-ci/{instance_name}/dir_sync\"\n",
+    "assert settings.storage.cloud_to_local_no_update(dir_sync, cache_key=\"dir_cache/key\") == settings.cache_dir / \"dir_cache/key\""
    ]
   },
   {
@@ -159,7 +159,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert dir_sync_local == settings.storage.cache_dir / \"dir_cache/key\"\n",
+    "assert dir_sync_local == settings.cache_dir / \"dir_cache/key\"\n",
     "assert dir_sync_local.is_dir()\n",
     "assert num_files(dir_sync_local) == 2"
    ]
@@ -312,8 +312,8 @@
    "source": [
     "time.sleep(1)\n",
     "cloud_file = dir_sync / \"file1\"\n",
-    "# update cloud timestamp, exist_ok=False needed due to truncate=not exist_ok in upath\n",
-    "cloud_file.touch(exist_ok=False)  \n",
+    "# update cloud timestamp\n",
+    "cloud_file.fs.touch(cloud_file.as_posix(), truncate=True)  \n",
     "\n",
     "assert cloud_file.modified.timestamp() > local_file_new.stat().st_mtime"
    ]

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/docs/hub-prod/test-init-load-local-anonymously.ipynb

@@ -63,7 +63,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!lamin load --unload\n",
+    "!lamin disconnect\n",
     "!lamin init --storage ./test-anonymous-init --schema bionty"
    ]
   },

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/docs/hub-prod/test-insufficient-user-info.ipynb

@@ -13,7 +13,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!lamin load --unload"
+    "!lamin disconnect"
    ]
   },
   {

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/lamindb_setup/__init__.py

@@ -33,7 +33,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.77.6"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.78.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 import sys as _sys

lamindb_setup-0.78.0/lamindb_setup/_cache.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import shutil
+
+from lamin_utils import logger
+
+from .core._settings_save import save_system_storage_settings
+
+
+def clear_cache_dir():
+    from lamindb_setup import close, settings
+
+    if settings.instance._is_cloud_sqlite:
+        logger.warning(
+            "Closing the current instance to update the cloud sqlite database."
+        )
+        close()
+
+    cache_dir = settings.cache_dir
+    shutil.rmtree(cache_dir)
+    cache_dir.mkdir()
+    logger.success("The cache directory was cleared.")
+
+
+def get_cache_dir():
+    from lamindb_setup import settings
+
+    return settings.cache_dir.as_posix()
+
+
+def set_cache_dir(cache_dir: str):
+    from lamindb_setup.core._settings import (
+        DEFAULT_CACHE_DIR,
+        _process_cache_path,
+        settings,
+    )
+
+    old_cache_dir = settings.cache_dir
+    new_cache_dir = _process_cache_path(cache_dir)
+    if new_cache_dir is None:
+        new_cache_dir = DEFAULT_CACHE_DIR
+    if new_cache_dir != old_cache_dir:
+        shutil.copytree(old_cache_dir, new_cache_dir, dirs_exist_ok=True)
+        shutil.rmtree(old_cache_dir)
+        logger.info("The current cache directory was moved to the specified location")
+    new_cache_dir = new_cache_dir.resolve()
+    save_system_storage_settings(new_cache_dir)
+    settings._cache_dir = new_cache_dir

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/lamindb_setup/_check_setup.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import importlib as il
 import os
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 
 from lamin_utils import logger
 
@@ -42,9 +42,9 @@ def _get_current_instance_settings() -> InstanceSettings | None:
         except Exception as e:
             # user will get more detailed traceback once they run the CLI
             logger.error(
-                "Current instance cannot be reached, unload it: `lamin load --unload`\n"
+                "Current instance cannot be reached, disconnect from it: `lamin disconnect`\n"
                 "Alternatively, init or load a connectable instance on the"
-                " command line: `lamin load <instance>` or `lamin init <...>`"
+                " command line: `lamin connect <instance>` or `lamin init <...>`"
             )
             raise e
         return isettings

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/lamindb_setup/_connect_instance.py

@@ -11,7 +11,7 @@ from ._check_setup import _check_instance_setup
 from ._close import close as close_instance
 from ._init_instance import MESSAGE_NO_MULTIPLE_INSTANCE, load_from_isettings
 from ._silence_loggers import silence_loggers
-from .core._hub_core import connect_instance as connect_instance_from_hub
+from .core._hub_core import connect_instance_hub
 from .core._hub_utils import (
     LaminDsn,
     LaminDsnModel,
@@ -128,7 +128,7 @@ def _connect_instance(
         # on the hub
         # do not call hub if the user is anonymous
         if owner != "anonymous":
-            hub_result = connect_instance_from_hub(
+            hub_result = connect_instance_hub(
                 owner=owner, name=name, access_token=access_token
             )
         else:
@@ -160,6 +160,10 @@ def _connect_instance(
                 git_repo=instance_result["git_repo"],
                 keep_artifacts_local=bool(instance_result["keep_artifacts_local"]),
                 is_on_hub=True,
+                api_url=instance_result["api_url"],
+                schema_id=None
+                if (schema_id := instance_result["schema_id"]) is None
+                else UUID(schema_id),
             )
         else:
             if hub_result != "anonymous-user":
@@ -256,7 +260,7 @@ def connect(slug: str, **kwargs) -> str | tuple | None:
                 logger.warning(
                     "SQLite file does not exist in the cloud, but exists locally:"
                     f" {isettings._sqlite_file_local}\nTo push the file to the cloud,"
-                    " call: lamin load --unload"
+                    " call: lamin disconnect"
                 )
             elif _raise_not_found_error:
                 raise SystemExit(msg)

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/lamindb_setup/_delete.py

@@ -24,7 +24,7 @@ def delete_cache(isettings: InstanceSettings):
     # avoid init of root
     root = isettings.storage._root_init
     if not isinstance(root, LocalPathClasses):
-        cache_dir = isettings.storage.cache_dir / root
+        cache_dir = settings.cache_dir / root.path
         if cache_dir.exists():
             shutil.rmtree(cache_dir)
 

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/lamindb_setup/_django.py

@@ -1,7 +1,5 @@
 from __future__ import annotations
 
-from typing import Optional
-
 from .core._settings import settings
 from .core.django import setup_django
 

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/lamindb_setup/_init_instance.py

@@ -309,7 +309,7 @@ def init(
             isettings._cloud_sqlite_locker.lock()
             logger.warning(
                 "locked instance (to unlock and push changes to the cloud SQLite file,"
-                " call: lamin load --unload)"
+                " call: lamin disconnect)"
             )
         if register_on_hub and isettings.dialect != "sqlite":
             from ._schema_metadata import update_schema_in_hub

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/lamindb_setup/_setup_user.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import os
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING
 
 from lamin_utils import logger
 

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/lamindb_setup/core/_hub_core.py

@@ -325,69 +325,7 @@ def _init_instance(
     logger.important(f"go to: https://lamin.ai/{isettings.owner}/{isettings.name}")
 
 
-def connect_instance(
-    *,
-    owner: str,  # account_handle
-    name: str,  # instance_name
-    access_token: str | None = None,
-) -> tuple[dict, dict] | str:
-    from ._settings import settings
-
-    if settings.user.handle != "anonymous" or access_token is not None:
-        return call_with_fallback_auth(
-            _connect_instance, owner=owner, name=name, access_token=access_token
-        )
-    else:
-        return call_with_fallback(_connect_instance, owner=owner, name=name)
-
-
-def _connect_instance(
-    *,
-    owner: str,  # account_handle
-    name: str,  # instance_name
-    client: Client,
-) -> tuple[dict, dict] | str:
-    instance_account_storage = select_instance_by_owner_name(owner, name, client)
-    if instance_account_storage is None:
-        # try the via single requests, will take more time
-        account = select_account_by_handle(owner, client)
-        if account is None:
-            return "account-not-exists"
-        instance = select_instance_by_name(account["id"], name, client)
-        if instance is None:
-            return "instance-not-found"
-        # get default storage
-        storage = select_default_storage_by_instance_id(instance["id"], client)
-        if storage is None:
-            return "default-storage-does-not-exist-on-hub"
-    else:
-        account = instance_account_storage.pop("account")
-        storage = instance_account_storage.pop("storage")
-        instance = instance_account_storage
-    # check if is postgres instance
-    # this used to be a check for `instance["db"] is not None` in earlier versions
-    # removed this on 2022-10-25 and can remove from the hub probably for lamindb 1.0
-    if instance["db_scheme"] is not None:
-        db_user = select_db_user_by_instance(instance["id"], client)
-        if db_user is None:
-            name, password = "none", "none"
-        else:
-            name, password = db_user["db_user_name"], db_user["db_user_password"]
-        # construct dsn from instance and db_account fields
-        db_dsn = LaminDsn.build(
-            scheme=instance["db_scheme"],
-            user=name,
-            password=password,
-            host=instance["db_host"],
-            port=instance["db_port"],
-            database=instance["db_database"],
-        )
-        instance["db"] = db_dsn
-    check_whether_migrations_in_sync(instance["lamindb_version"])
-    return instance, storage  # type: ignore
-
-
-def _connect_instance_new(
+def _connect_instance_hub(
     owner: str,  # account_handle
     name: str,  # instance_name
     client: Client,
@@ -439,10 +377,11 @@ def _connect_instance_new(
             database=instance["db_database"],
         )
         instance["db"] = db_dsn
+    check_whether_migrations_in_sync(instance["lamindb_version"])
     return instance, storage  # type: ignore
 
 
-def connect_instance_new(
+def connect_instance_hub(
     *,
     owner: str,  # account_handle
     name: str,  # instance_name
@@ -452,10 +391,10 @@ def connect_instance_new(
 
     if settings.user.handle != "anonymous" or access_token is not None:
         return call_with_fallback_auth(
-            _connect_instance_new, owner=owner, name=name, access_token=access_token
+            _connect_instance_hub, owner=owner, name=name, access_token=access_token
         )
     else:
-        return call_with_fallback(_connect_instance_new, owner=owner, name=name)
+        return call_with_fallback(_connect_instance_hub, owner=owner, name=name)
 
 
 def access_aws(storage_root: str, access_token: str | None = None) -> dict[str, dict]:

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/lamindb_setup/core/_hub_utils.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, ClassVar, Optional
+from typing import Any, ClassVar
 from urllib.parse import urlparse, urlunparse
 
 from pydantic import BaseModel, Field, GetCoreSchemaHandler

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/lamindb_setup/core/_settings.py

@@ -3,17 +3,37 @@ from __future__ import annotations
 import os
 from typing import TYPE_CHECKING
 
+from appdirs import AppDirs
+
 from ._settings_load import (
     load_instance_settings,
     load_or_create_user_settings,
+    load_system_storage_settings,
 )
 from ._settings_store import current_instance_settings_file, settings_dir
+from .upath import LocalPathClasses, UPath
 
 if TYPE_CHECKING:
     from pathlib import Path
 
     from lamindb_setup.core import InstanceSettings, StorageSettings, UserSettings
 
+    from .types import UPathStr
+
+
+DEFAULT_CACHE_DIR = UPath(AppDirs("lamindb", "laminlabs").user_cache_dir)
+
+
+def _process_cache_path(cache_path: UPathStr | None):
+    if cache_path is None or cache_path == "null":
+        return None
+    cache_dir = UPath(cache_path)
+    if not isinstance(cache_dir, LocalPathClasses):
+        raise ValueError("cache dir should be a local path.")
+    if cache_dir.exists() and not cache_dir.is_dir():
+        raise ValueError("cache dir should be a directory.")
+    return cache_dir
+
 
 class SetupSettings:
     """Setup settings."""
@@ -29,6 +49,8 @@ class SetupSettings:
     _auto_connect_path: Path = settings_dir / "auto_connect"
     _private_django_api_path: Path = settings_dir / "private_django_api"
 
+    _cache_dir: Path | None = None
+
     @property
     def _instance_settings_path(self) -> Path:
         return current_instance_settings_file()
@@ -123,11 +145,28 @@ class SetupSettings:
         except SystemExit:
             return False
 
+    @property
+    def cache_dir(self) -> UPath:
+        """Cache root, a local directory to cache cloud files."""
+        if "LAMIN_CACHE_DIR" in os.environ:
+            cache_dir = UPath(os.environ["LAMIN_CACHE_DIR"])
+        elif self._cache_dir is None:
+            cache_path = load_system_storage_settings().get("lamindb_cache_path", None)
+            cache_dir = _process_cache_path(cache_path)
+            if cache_dir is None:
+                cache_dir = DEFAULT_CACHE_DIR
+            self._cache_dir = cache_dir
+        else:
+            cache_dir = self._cache_dir
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        return cache_dir
+
     def __repr__(self) -> str:
         """Rich string representation."""
         repr = self.user.__repr__()
         repr += f"\nAuto-connect in Python: {self.auto_connect}\n"
         repr += f"Private Django API: {self.private_django_api}\n"
+        repr += f"Cache directory: {self.cache_dir}\n"
         if self._instance_exists:
             repr += self.instance.__repr__()
         else:

{lamindb_setup-0.77.6 → lamindb_setup-0.78.0}/lamindb_setup/core/_settings_instance.py

@@ -56,6 +56,8 @@ class InstanceSettings:
         schema: str | None = None,  # comma-separated string of schema names
         git_repo: str | None = None,  # a git repo URL
         is_on_hub: bool | None = None,  # initialized from hub
+        api_url: str | None = None,
+        schema_id: UUID | None = None,
         _locker_user: UserSettings | None = None,  # user to lock for if cloud sqlite
     ):
         from ._hub_utils import validate_db_arg
@@ -73,6 +75,9 @@ class InstanceSettings:
         self._keep_artifacts_local = keep_artifacts_local
         self._storage_local: StorageSettings | None = None
         self._is_on_hub = is_on_hub
+        # private, needed for writing instance settings
+        self._api_url = api_url
+        self._schema_id = schema_id
         # if None then settings.user is used
         self._locker_user = _locker_user
 
@@ -306,7 +311,7 @@ class InstanceSettings:
         if self._is_cloud_sqlite:
             logger.warning(
                 "updating local SQLite & locking cloud SQLite (sync back & unlock:"
-                " lamin load --unload)"
+                " lamin disconnect)"
             )
             if lock_cloud_sqlite:
                 self._cloud_sqlite_locker.lock()