lamindb_setup 0.69.5__tar.gz → 0.70.0__tar.gz

{lamindb_setup-0.69.5 → lamindb_setup-0.70.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.69.5
+Version: 0.70.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <laminlabs@gmail.com>
 Description-Content-Type: text/markdown

{lamindb_setup-0.69.5 → lamindb_setup-0.70.0}/docs/changelog.md

@@ -3,6 +3,10 @@
 <!-- prettier-ignore -->
 Name | PR | Developer | Date | Version
 --- | --- | --- | --- | ---
+✨ Introduce `local_storage` mode for cloud instances | [728](https://github.com/laminlabs/lamindb-setup/pull/728) | [falexwolf](https://github.com/falexwolf) | 2024-04-23 | 0.70.0
+🚚 Make `.uuid` attributes private | [727](https://github.com/laminlabs/lamindb-setup/pull/727) | [falexwolf](https://github.com/falexwolf) | 2024-04-23 |
+🚚 Rename `is_cloud` to `type_is_cloud` | [726](https://github.com/laminlabs/lamindb-setup/pull/726) | [falexwolf](https://github.com/falexwolf) | 2024-04-23 |
+🚸 In `.delete()`, check for data deletion & delete from hub | [725](https://github.com/laminlabs/lamindb-setup/pull/725) | [falexwolf](https://github.com/falexwolf) | 2024-04-22 |
 ✨ Upload dirs inplace | [722](https://github.com/laminlabs/lamindb-setup/pull/722) | [Koncopd](https://github.com/Koncopd) | 2024-04-20 | 0.69.5
 ♻️ Do not always read from local cache | [720](https://github.com/laminlabs/lamindb-setup/pull/720) | [falexwolf](https://github.com/falexwolf) | 2024-04-19 | 0.69.4
 ♻️ Improve suffix handling | [718](https://github.com/laminlabs/lamindb-setup/pull/718) | [falexwolf](https://github.com/falexwolf) | 2024-04-19 | 0.69.3

lamindb_setup-0.70.0/docs/hub-cloud/01-init-local-instance.ipynb

@@ -0,0 +1,114 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "# Init a local instance"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!lamin login testuser1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import lamindb_setup as ln_setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln_setup.init(storage=\"./mydata\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "\n",
+    "assert ln_setup.settings.instance.storage.type_is_cloud == False\n",
+    "assert ln_setup.settings.instance.owner == ln_setup.settings.user.handle\n",
+    "assert ln_setup.settings.instance.name == \"mydata\"\n",
+    "assert ln_setup.settings.storage.root.as_posix() == Path(\"mydata\").resolve().as_posix()\n",
+    "storage_root = ln_setup.settings.storage.root\n",
+    "assert storage_root.exists()\n",
+    "assert ln_setup.settings.storage.id is not None\n",
+    "assert (\n",
+    "    ln_setup.settings.instance.db\n",
+    "    == f\"sqlite:///{Path('./mydata').resolve().as_posix()}/{ln_setup.settings.instance.id.hex}.lndb\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln_setup.delete(\"mydata\", force=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from lamindb_setup.core._settings_store import instance_settings_file\n",
+    "\n",
+    "settings_file = instance_settings_file(\"mydata\", \"testuser1\")\n",
+    "assert not storage_root.exists()\n",
+    "assert settings_file.exists() == False"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "40d3a090f54c6569ab1632332b64b2c03c39dcf918b08424e98f38b5ae0af88f"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

lamindb_setup-0.69.5/docs/hub-cloud/02-connect-on-prem-instance.ipynb → lamindb_setup-0.70.0/docs/hub-cloud/02-connect-local-instance.ipynb

@@ -8,15 +8,7 @@
     "tags": []
    },
    "source": [
-    "# Load & delete an on-prem instance"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "f165d50c",
-   "metadata": {},
-   "source": [
-    "Here, we construct a case in which the storage location of the previous instance was moved:"
+    "# Connect to local instance"
    ]
   },
   {
@@ -28,8 +20,9 @@
    },
    "outputs": [],
    "source": [
+    "# here, we construct a case in which the storage location of the previous instance was moved\n",
     "!lamin close\n",
-    "!yes | lamin delete mydata\n",
+    "!lamin delete --force mydata\n",
     "!lamin init --storage mydata\n",
     "!rm -r ./mydata_new_loc\n",
     "!mv mydata ./mydata_new_loc\n",
@@ -99,7 +92,7 @@
    "source": [
     "from pathlib import Path\n",
     "\n",
-    "assert ln_setup.settings.instance.storage.is_cloud == False\n",
+    "assert ln_setup.settings.instance.storage.type_is_cloud == False\n",
     "assert ln_setup.settings.instance.name == \"mydata\"\n",
     "assert (\n",
     "    ln_setup.settings.instance.storage.root.as_posix()\n",
@@ -141,7 +134,7 @@
    "id": "e3701120",
    "metadata": {},
    "source": [
-    "## Delete an instance"
+    "Delete:"
    ]
   },
   {
@@ -173,7 +166,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.15"
+   "version": "3.10.13"
   },
   "vscode": {
    "interpreter": {

{lamindb_setup-0.69.5 → lamindb_setup-0.70.0}/docs/hub-cloud/03-set-storage.ipynb

@@ -182,7 +182,7 @@
    },
    "outputs": [],
    "source": [
-    "assert ln_setup.settings.storage.is_cloud\n",
+    "assert ln_setup.settings.storage.type_is_cloud\n",
     "assert ln_setup.settings.storage.root_as_str == \"s3://lamindb-ci\"\n",
     "assert ln_setup.settings.storage.region == \"us-west-1\"\n",
     "# root.fs contains the underlying fsspec filesystem\n",

{lamindb_setup-0.69.5 → lamindb_setup-0.70.0}/docs/hub-cloud/05-init-hosted-instance.ipynb

@@ -90,7 +90,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert ln_setup.settings.instance.storage.is_cloud == True\n",
+    "assert ln_setup.settings.instance.storage.type_is_cloud == True\n",
     "assert ln_setup.settings.instance.owner == ln_setup.settings.user.handle\n",
     "assert ln_setup.settings.instance.name == \"my-hosted\"\n",
     "assert ln_setup.settings.storage.root.as_posix().startswith(hosted_buckets)\n",

lamindb_setup-0.70.0/docs/hub-cloud/07-connect-hybrid-instance.ipynb

@@ -0,0 +1,86 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Connect hybrid instance"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# this shouldn't run in a notebook, but in the unit tests...\n",
+    "\n",
+    "import pytest\n",
+    "import lamindb_setup as ln_setup\n",
+    "from pathlib import Path\n",
+    "\n",
+    "ln_setup.login(\"testuser1\")\n",
+    "ln_setup.init(storage=\"s3://lamindb-ci/load_remote_instance\")\n",
+    "\n",
+    "assert ln_setup.settings.instance.name == \"load_remote_instance\"\n",
+    "assert ln_setup.settings.instance.storage.type_is_cloud\n",
+    "assert (\n",
+    "    ln_setup.settings.instance.storage.root_as_str\n",
+    "    == \"s3://lamindb-ci/load_remote_instance\"\n",
+    ")\n",
+    "assert (\n",
+    "    ln_setup.settings.instance._sqlite_file.as_posix()\n",
+    "    == f\"s3://lamindb-ci/load_remote_instance/{ln_setup.settings.instance.id.hex}.lndb\"  # noqa\n",
+    ")\n",
+    "with pytest.raises(ValueError) as error:\n",
+    "    ln_setup.settings.instance.local_storage\n",
+    "assert error.exconly() == \"ValueError: Local storage is not enabled for this instance.\"\n",
+    "ln_setup.settings.instance._local_storage_on = True\n",
+    "with pytest.raises(ValueError) as error:\n",
+    "    ln_setup.settings.instance.local_storage\n",
+    "assert (\n",
+    "    error.exconly()\n",
+    "    == \"ValueError: No local storage root found, set via\"\n",
+    "    \" `ln.setup.settings.instance.local_storage = local_root`\"\n",
+    ")\n",
+    "\n",
+    "# now set local storage location\n",
+    "ln_setup.settings.instance.local_storage = \"./my_local_storage\"\n",
+    "\n",
+    "assert (\n",
+    "    ln_setup.settings.instance.local_storage.root.as_posix()\n",
+    "    == Path(\"./my_local_storage\").resolve().as_posix()\n",
+    ")\n",
+    "assert ln_setup.settings.instance.local_storage is not None\n",
+    "with pytest.raises(ValueError) as error:\n",
+    "    ln_setup.settings.instance.local_storage = \"./my_local_storage2\"\n",
+    "assert error.exconly().startswith(\n",
+    "    \"ValueError: You already configured a local storage root for this instance in this\"\n",
+    "    \" environment:\"\n",
+    ")\n",
+    "ln_setup.delete(\"load_remote_instance\", force=True)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "py310",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

{lamindb_setup-0.69.5 → lamindb_setup-0.70.0}/docs/hub-prod/test-empty-init.ipynb

@@ -71,7 +71,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert ln_setup.settings.storage.is_cloud\n",
+    "assert ln_setup.settings.storage.type_is_cloud\n",
     "assert ln_setup.settings.storage.root_as_str == root_str\n",
     "assert ln_setup.settings.storage.region == \"us-east-1\"\n",
     "assert (\n",

{lamindb_setup-0.69.5 → lamindb_setup-0.70.0}/docs/notebooks.md

@@ -3,13 +3,14 @@
 ```{toctree}
 :maxdepth: 1
 
-hub-cloud/init-on-prem-instance
-hub-cloud/connect-on-prem-instance
+hub-cloud/init-local-instance
+hub-cloud/connect-local-instance
 hub-cloud/set-storage
 hub-cloud/test-bionty
 hub-cloud/init-hosted-instance
 hub-cloud/connect-hosted-instance
 hub-cloud/test-multi-session
+hub-cloud/connect-hybrid-instance
 
 hub-prod/test-cache-management
 hub-prod/test-empty-init

{lamindb_setup-0.69.5 → lamindb_setup-0.70.0}/lamindb_setup/__init__.py

@@ -36,7 +36,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.69.5"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.70.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

{lamindb_setup-0.69.5 → lamindb_setup-0.70.0}/lamindb_setup/_connect_instance.py

@@ -27,6 +27,17 @@ from ._migrate import check_whether_migrations_in_sync
 _TEST_FAILED_LOAD = False
 
 
+INSTANCE_NOT_FOUND_MESSAGE = (
+    "'{owner}/{name}' not found:"
+    " '{hub_result}'\nCheck your permissions:"
+    " https://lamin.ai/{owner}/{name}"
+)
+
+
+class InstanceNotFoundError(SystemExit):
+    pass
+
+
 def check_db_dsn_equal_up_to_credentials(db_dsn_hub, db_dsn_local):
     return (
         db_dsn_hub.scheme == db_dsn_local.scheme
@@ -121,6 +132,8 @@ def connect(
         make_hub_request = True
         if settings_file.exists():
             isettings = load_instance_settings(settings_file)
+            # mimic instance_result from hub
+            instance_result = {"id": isettings.id.hex}
             # skip hub request for a purely local instance
             make_hub_request = isettings.is_remote
 
@@ -139,7 +152,6 @@ def connect(
                     root=storage_result["root"],
                     region=storage_result["region"],
                     uid=storage_result["lnid"],
-                    is_hybrid=instance_result["storage_mode"] == "hybrid",
                 )
                 isettings = InstanceSettings(
                     id=UUID(instance_result["id"]),
@@ -149,27 +161,24 @@ def connect(
                     db=db_updated,
                     schema=instance_result["schema_str"],
                     git_repo=instance_result["git_repo"],
+                    local_storage=instance_result["storage_mode"] == "hybrid",
                 )
                 check_whether_migrations_in_sync(instance_result["lamindb_version"])
             else:
-                error_message = (
-                    f"'{owner}/{name}' not loadable:"
-                    f" '{hub_result}'\nCheck your permissions:"
-                    f" https://lamin.ai/{owner}/{name}?tab=collaborators"
+                message = INSTANCE_NOT_FOUND_MESSAGE.format(
+                    owner=owner, name=name, hub_result=hub_result
                 )
                 if settings_file.exists():
                     isettings = load_instance_settings(settings_file)
                     if isettings.is_remote:
                         if _raise_not_reachable_error:
-                            raise SystemExit(error_message)
+                            raise InstanceNotFoundError(message)
                         return "instance-not-reachable"
 
                 else:
                     if _raise_not_reachable_error:
-                        raise SystemExit(error_message)
+                        raise InstanceNotFoundError(message)
                     return "instance-not-reachable"
-                # mimic instance_result from hub
-                instance_result = {"id": isettings.id.hex}
 
         if storage is not None:
             update_isettings_with_storage(isettings, storage)
@@ -255,7 +264,7 @@ def update_isettings_with_storage(
     isettings: InstanceSettings, storage: UPathStr
 ) -> None:
     ssettings = StorageSettings(storage)
-    if ssettings.is_cloud:
+    if ssettings.type_is_cloud:
         try:  # triggering ssettings.id makes a lookup in the storage table
             logger.success(f"loaded storage: {ssettings.id} / {ssettings.root_as_str}")
         except RuntimeError as e:

lamindb_setup-0.70.0/lamindb_setup/_delete.py

@@ -0,0 +1,144 @@
+import shutil
+from pathlib import Path
+from lamin_utils import logger
+from uuid import UUID
+from typing import Optional
+from .core._settings_instance import InstanceSettings
+from .core._settings_storage import StorageSettings
+from .core._settings import settings
+from .core._settings_load import load_instance_settings
+from .core._settings_store import instance_settings_file
+from .core.upath import check_storage_is_empty, hosted_buckets
+from .core._hub_core import delete_instance as delete_instance_on_hub
+from .core._hub_core import connect_instance as load_instance_from_hub
+from ._connect_instance import INSTANCE_NOT_FOUND_MESSAGE
+
+
+def delete_cache(cache_dir: Path):
+    if cache_dir is not None and cache_dir.exists():
+        shutil.rmtree(cache_dir)
+
+
+def delete_exclusion_dir(isettings: InstanceSettings) -> None:
+    exclusion_dir = isettings.storage.root / f".lamindb/_exclusion/{isettings.id.hex}"
+    if exclusion_dir.exists():
+        exclusion_dir.rmdir()
+
+
+def delete_by_isettings(isettings: InstanceSettings) -> None:
+    settings_file = isettings._get_settings_file()
+    if settings_file.exists():
+        settings_file.unlink()
+    delete_cache(isettings.storage.cache_dir)
+    if isettings.dialect == "sqlite":
+        try:
+            if isettings._sqlite_file.exists():
+                isettings._sqlite_file.unlink()
+        except PermissionError:
+            logger.warning(
+                "Did not have permission to delete SQLite file:"
+                f" {isettings._sqlite_file}"
+            )
+            pass
+    # unset the global instance settings
+    if settings._instance_exists and isettings.slug == settings.instance.slug:
+        if settings._instance_settings_path.exists():
+            settings._instance_settings_path.unlink()
+        settings._instance_settings = None
+    if isettings.storage._mark_storage_root.exists():
+        isettings.storage._mark_storage_root.unlink()
+
+
+def delete(
+    instance_name: str, force: bool = False, require_empty: bool = True
+) -> Optional[int]:
+    """Delete a LaminDB instance.
+
+    Args:
+        instance_name (str): The name of the instance to delete.
+        force (bool): Whether to skip the confirmation prompt.
+        require_empty (bool): Whether to check if the instance is empty before deleting.
+    """
+    if "/" in instance_name:
+        logger.warning(
+            "Deleting the instance of another user is currently not supported with the"
+            " CLI. Please provide only the instance name when deleting an instance ('/'"
+            " delimiter not allowed)."
+        )
+        raise ValueError("Invalid instance name: '/' delimiter not allowed.")
+    instance_slug = f"{settings.user.handle}/{instance_name}"
+    if settings._instance_exists and settings.instance.name == instance_name:
+        isettings = settings.instance
+    else:
+        settings_file = instance_settings_file(instance_name, settings.user.handle)
+        if not settings_file.exists():
+            hub_result = load_instance_from_hub(
+                owner=settings.user.handle, name=instance_name
+            )
+            if isinstance(hub_result, str):
+                message = INSTANCE_NOT_FOUND_MESSAGE.format(
+                    owner=settings.user.handle,
+                    name=instance_name,
+                    hub_result=hub_result,
+                )
+                logger.warning(message)
+                return None
+            instance_result, storage_result = hub_result
+            ssettings = StorageSettings(
+                root=storage_result["root"],
+                region=storage_result["region"],
+                uid=storage_result["lnid"],
+            )
+            isettings = InstanceSettings(
+                id=UUID(instance_result["id"]),
+                owner=settings.user.handle,
+                name=instance_name,
+                storage=ssettings,
+                local_storage=instance_result["storage_mode"] == "hybrid",
+                db=instance_result["db"] if "db" in instance_result else None,
+                schema=instance_result["schema_str"],
+                git_repo=instance_result["git_repo"],
+            )
+        else:
+            isettings = load_instance_settings(settings_file)
+    if isettings.dialect != "sqlite":
+        logger.warning(
+            f"delete() does not yet affect your Postgres database at {isettings.db}"
+        )
+    if not force:
+        valid_responses = ["y", "yes"]
+        user_input = (
+            input(f"Are you sure you want to delete instance {instance_slug}? (y/n) ")
+            .strip()
+            .lower()
+        )
+        if user_input not in valid_responses:
+            return -1
+
+    # the actual deletion process begins here
+    if isettings.dialect == "sqlite" and isettings.is_remote:
+        # delete the exlusion dir first because it's hard to count its objects
+        delete_exclusion_dir(isettings)
+    if isettings.storage.type_is_cloud and isettings.storage.root_as_str.startswith(
+        hosted_buckets
+    ):
+        if not require_empty:
+            logger.warning(
+                "hosted storage always has to be empty, ignoring `require_empty`"
+            )
+        require_empty = True
+    n_objects = check_storage_is_empty(
+        isettings.storage.root,
+        raise_error=require_empty,
+        account_for_sqlite_file=isettings.dialect == "sqlite",
+    )
+    logger.info(f"deleting instance {instance_slug}")
+    # below we can skip check_storage_is_empty() because we already called
+    # it above
+    delete_instance_on_hub(isettings.id, require_empty=False)
+    delete_by_isettings(isettings)
+    if n_objects == 0 and isettings.storage.type == "local":
+        # dir is only empty after sqlite file was delete via delete_by_isettings
+        (isettings.storage.root / ".lamindb").rmdir()
+        isettings.storage.root.rmdir()
+    return None

{lamindb_setup-0.69.5 → lamindb_setup-0.70.0}/lamindb_setup/_init_instance.py

@@ -269,7 +269,7 @@ def init(
             delete_instance_record(isettings.id)
             isettings._get_settings_file().unlink(missing_ok=True)  # type: ignore
         if ssettings is not None:
-            delete_storage_record(ssettings.uuid)  # type: ignore
+            delete_storage_record(ssettings._uuid)  # type: ignore
         raise e
     return None
 

{lamindb_setup-0.69.5 → lamindb_setup-0.70.0}/lamindb_setup/_migrate.py

@@ -16,6 +16,9 @@ def check_whether_migrations_in_sync(db_version_str: str):
         installed_version_str = metadata.version("lamindb")
     except metadata.PackageNotFoundError:
         return None
+    if db_version_str is None:
+        logger.warning("no lamindb version stored to compare with installed version")
+        return None
     installed_version = version.parse(installed_version_str)
     db_version = version.parse(db_version_str)
     if (
@@ -83,7 +86,7 @@ class migrate:
             collaborator = call_with_fallback_auth(
                 select_collaborator,
                 instance_id=instance_id_str,
-                account_id=settings.user.uuid,
+                account_id=settings.user._uuid,
             )
             if collaborator is None or collaborator["role"] != "admin":
                 raise SystemExit(

{lamindb_setup-0.69.5 → lamindb_setup-0.70.0}/lamindb_setup/_register_instance.py

@@ -17,6 +17,6 @@ def register(_test: bool = False):
     if ssettings._uid is None and _test:
         # because django isn't up, we can't get it from the database
         ssettings._uid = base62(8)
-    ssettings._uuid = init_storage_hub(ssettings)
+    ssettings._uuid_ = init_storage_hub(ssettings)
     init_instance_hub(isettings)
     isettings._persist()

{lamindb_setup-0.69.5 → lamindb_setup-0.70.0}/lamindb_setup/_setup_user.py

@@ -99,7 +99,7 @@ def login(
     user_settings.uid = user_id
     user_settings.handle = user_handle
     user_settings.name = user_name
-    user_settings.uuid = user_uuid
+    user_settings._uuid = user_uuid
     user_settings.access_token = access_token
     save_user_settings(user_settings)