lamindb_setup 1.6.1__tar.gz → 1.7.0__tar.gz

{lamindb_setup-1.6.1 → lamindb_setup-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.6.1
+Version: 1.7.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10
@@ -9,7 +9,7 @@ Requires-Dist: lamin_utils>=0.3.3
 Requires-Dist: django>=5.1,<5.2
 Requires-Dist: dj_database_url>=1.3.0,<3.0.0
 Requires-Dist: pydantic-settings
-Requires-Dist: appdirs<2.0.0
+Requires-Dist: platformdirs<5.0.0
 Requires-Dist: requests
 Requires-Dist: universal_pathlib==0.2.6
 Requires-Dist: botocore<2.0.0

{lamindb_setup-1.6.1 → lamindb_setup-1.7.0}/docs/hub-cloud/01-init-local-instance.ipynb

@@ -27,6 +27,15 @@
     "import lamindb_setup as ln_setup"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert not ln_setup.settings.is_connected"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -36,6 +45,15 @@
     "ln_setup.init(storage=\"./mydata\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert ln_setup.settings.is_connected"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -92,7 +110,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "lamindb",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -106,7 +124,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.8"
+   "version": "3.10.16"
   }
  },
  "nbformat": 4,

{lamindb_setup-1.6.1 → lamindb_setup-1.7.0}/docs/hub-cloud/02-connect-local-instance.ipynb

@@ -147,9 +147,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert ln_setup.settings._instance_settings_path.exists()\n",
-    "ln_setup.delete(\"mydata\", force=True)\n",
-    "assert not ln_setup.settings._instance_settings_path.exists()"
+    "ln_setup.delete(\"mydata\", force=True)"
    ]
   }
  ],

{lamindb_setup-1.6.1 → lamindb_setup-1.7.0}/docs/hub-cloud/03-add-managed-storage.ipynb

@@ -7,7 +7,7 @@
     "tags": []
    },
    "source": [
-    "# Add managed storage"
+    "# Add managed and referenced storage locations"
    ]
   },
   {
@@ -27,6 +27,24 @@
     "!docker stop pgtest && docker rm pgtest"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "6dcaa297",
+   "metadata": {},
+   "source": [
+    "Create a local instance that's not registered on the hub, so that we can test interplay."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "599eae6a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!lamin init --storage \"./storage-of-another-instance\""
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -65,6 +83,35 @@
     "ln_setup.init(storage=\"./storage1\", name=\"test-add-managed-storage\", db=pgurl)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "b62e84cb",
+   "metadata": {},
+   "source": [
+    "Test adding referenced storage location."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dbf195e3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import lamindb as ln\n",
+    "\n",
+    "storage = ln.Storage(root=\"./storage-of-another-instance\").save()\n",
+    "assert storage.instance_uid == \"__unknown__\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "93234f70",
+   "metadata": {},
+   "source": [
+    "Now continue with managed storage locations."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -134,7 +181,7 @@
     "assert ln_setup.settings.storage.root_as_str == f\"{Path.cwd()}/storage2\"\n",
     "assert (\n",
     "    ln_setup.settings.storage.root / \".lamindb/storage_uid.txt\"\n",
-    ").read_text() == ln_setup.settings.storage.uid\n",
+    ").read_text().splitlines()[0] == ln_setup.settings.storage.uid\n",
     "assert ln_setup.settings.storage.is_on_hub"
    ]
   },
@@ -166,7 +213,7 @@
     "assert ln_setup.settings.storage.root_as_str == f\"{Path.cwd()}/storage1\"\n",
     "assert (\n",
     "    ln_setup.settings.storage.root / \".lamindb/storage_uid.txt\"\n",
-    ").read_text() == ln_setup.settings.storage.uid\n",
+    ").read_text().splitlines()[0] == ln_setup.settings.storage.uid\n",
     "assert ln_setup.settings.storage.is_on_hub\n",
     "assert ln_setup.settings.storage.uid == storage1_uid"
    ]
@@ -199,7 +246,7 @@
     "assert ln_setup.settings.storage.root_as_str == f\"{Path.cwd()}/storage1\"\n",
     "assert (\n",
     "    ln_setup.settings.storage.root / \".lamindb/storage_uid.txt\"\n",
-    ").read_text() == ln_setup.settings.storage.uid\n",
+    ").read_text().splitlines()[0] == ln_setup.settings.storage.uid\n",
     "assert ln_setup.settings.storage.is_on_hub\n",
     "assert ln_setup.settings.storage.uid == storage1_uid"
    ]
@@ -225,7 +272,7 @@
     "assert ln_setup.settings.storage.root_as_str == f\"{Path.cwd()}/storage2\"\n",
     "assert (\n",
     "    ln_setup.settings.storage.root / \".lamindb/storage_uid.txt\"\n",
-    ").read_text() == ln_setup.settings.storage.uid\n",
+    ").read_text().splitlines()[0] == ln_setup.settings.storage.uid\n",
     "assert ln_setup.settings.storage.is_on_hub\n",
     "assert ln_setup.settings.storage.uid == storage2_uid"
    ]
@@ -264,7 +311,7 @@
     "assert ln_setup.settings.storage.region == \"us-west-1\"\n",
     "assert (\n",
     "    ln_setup.settings.storage.root / \".lamindb/storage_uid.txt\"\n",
-    ").read_text() == ln_setup.settings.storage.uid\n",
+    ").read_text().splitlines()[0] == ln_setup.settings.storage.uid\n",
     "# root.fs contains the underlying fsspec filesystem\n",
     "assert (\n",
     "    ln_setup.settings.storage.root.fs.cache_regions  # set by lamindb to True for s3 by default\n",
@@ -443,7 +490,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "py312",
    "language": "python",
    "name": "python3"
   },
@@ -457,12 +504,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.16"
-  },
-  "vscode": {
-   "interpreter": {
-    "hash": "61b4062b24dfb1010f420dad5aa3bd73a4d2af47d0ec44eafec465a35a9d7239"
-   }
+   "version": "3.12.8"
   }
  },
  "nbformat": 4,

{lamindb_setup-1.6.1 → lamindb_setup-1.7.0}/docs/hub-cloud/05-init-hosted-instance.ipynb

@@ -23,7 +23,7 @@
    "outputs": [],
    "source": [
     "import lamindb_setup as ln_setup\n",
-    "from lamindb_setup.core.upath import create_path, UPath, InstanceNotEmpty\n",
+    "from lamindb_setup.core.upath import create_path, UPath, StorageNotEmpty\n",
     "from lamindb_setup.core._hub_core import (\n",
     "    delete_instance,\n",
     "    call_with_fallback_auth,\n",
@@ -44,7 +44,7 @@
    "source": [
     "try:\n",
     "    delete_instance(f\"testuser1/{instance_name}\")\n",
-    "except InstanceNotEmpty:\n",
+    "except StorageNotEmpty:\n",
     "    instance_with_storage = call_with_fallback_auth(\n",
     "        select_instance_by_owner_name,\n",
     "        owner=\"testuser1\",\n",
@@ -137,7 +137,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "py312",
    "language": "python",
    "name": "python3"
   },
@@ -151,7 +151,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.12.8"
   }
  },
  "nbformat": 4,

{lamindb_setup-1.6.1 → lamindb_setup-1.7.0}/docs/hub-cloud/06-connect-hosted-instance.ipynb

@@ -19,7 +19,7 @@
     "from lamindb_setup.core.upath import UPath\n",
     "from lamindb_setup.core._hub_core import delete_instance\n",
     "from lamindb_setup.core._hub_client import connect_hub_with_auth\n",
-    "from lamindb_setup.core.upath import InstanceNotEmpty\n",
+    "from lamindb_setup.core.upath import StorageNotEmpty\n",
     "\n",
     "instance_name = \"my-hosted\"\n",
     "assert ln_setup.settings.user.handle == \"testuser1\""
@@ -123,7 +123,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "with pytest.raises(InstanceNotEmpty):\n",
+    "with pytest.raises(StorageNotEmpty):\n",
     "    delete_instance(f\"testuser1/{instance_name}\")"
    ]
   },

{lamindb_setup-1.6.1 → lamindb_setup-1.7.0}/docs/hub-cloud/07-keep-artifacts-local.ipynb

@@ -23,7 +23,7 @@
     "from pathlib import Path\n",
     "\n",
     "name = f\"keep-artifacts-local-setup-{os.environ['LAMIN_ENV']}\"\n",
-    "storage = (UPath(\"s3://lamindb-ci\") / name).as_posix()\n",
+    "storage = UPath(f\"s3://lamindb-ci/{name}\").as_posix()\n",
     "\n",
     "ln_setup.login(\"testuser1\")\n",
     "ln_setup.init(storage=storage)\n",
@@ -76,7 +76,7 @@
     ")\n",
     "assert (\n",
     "    ln_setup.settings.instance.storage_local.root / \".lamindb/storage_uid.txt\"\n",
-    ").read_text() == ln_setup.settings.instance.storage_local.uid\n",
+    ").read_text().splitlines()[0] == ln_setup.settings.instance.storage_local.uid\n",
     "assert ln_setup.settings.instance.storage_local is not None\n",
     "# the remote storage location is still in the regular slot\n",
     "assert ln_setup.settings.instance.storage.root.as_posix() == storage"
@@ -110,7 +110,7 @@
     ")\n",
     "assert (\n",
     "    ln_setup.settings.instance.storage_local.root / \".lamindb/storage_uid.txt\"\n",
-    ").read_text() == ln_setup.settings.instance.storage_local.uid"
+    ").read_text().splitlines()[0] == ln_setup.settings.instance.storage_local.uid"
    ]
   },
   {
@@ -177,7 +177,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "with pytest.raises(ln_setup.core.upath.InstanceNotEmpty):\n",
+    "with pytest.raises(ln_setup.errors.StorageNotEmpty):\n",
     "    ln_setup.delete(name, force=True)"
    ]
   },
@@ -227,7 +227,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "lamindb",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -241,7 +241,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.8"
+   "version": "3.10.16"
   }
  },
  "nbformat": 4,

{lamindb_setup-1.6.1 → lamindb_setup-1.7.0}/docs/hub-cloud/08-test-multi-session.ipynb

@@ -54,6 +54,15 @@
     "ln_setup.init(storage=\"./testsetup\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert ln_setup.settings.instance.slug == \"testuser1/testsetup\""
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -111,43 +120,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Let us try to init another instance in the same Python session:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from lamindb_setup._init_instance import CannotSwitchDefaultInstance\n",
-    "\n",
-    "with pytest.raises(CannotSwitchDefaultInstance):\n",
-    "    ln_setup.init(storage=\"./testsetup2\")\n",
-    "with pytest.raises(CannotSwitchDefaultInstance):\n",
-    "    ln_setup.connect(\"testsetup\")\n",
-    "with pytest.raises(RuntimeError):\n",
-    "    ln_setup.migrate.create()\n",
-    "with pytest.raises(RuntimeError):\n",
-    "    ln_setup.migrate.deploy()\n",
-    "\n",
-    "assert ln_setup.settings.instance.slug == \"testuser1/testsetup\""
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Reset `django` and connect to another instance:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ln_setup.core.django.reset_django()"
+    "Connect to another instance in the same process:"
    ]
   },
   {
@@ -182,7 +155,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "lamindb",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -196,7 +169,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.8"
+   "version": "3.10.16"
   }
  },
  "nbformat": 4,

{lamindb_setup-1.6.1 → lamindb_setup-1.7.0}/docs/hub-prod/test-cache-management.ipynb

@@ -16,9 +16,20 @@
    "outputs": [],
    "source": [
     "import os\n",
+    "from dotenv import dotenv_values\n",
+    "from platformdirs import user_config_dir\n",
+    "\n",
+    "# otherwise writing to system_settings_file() fails\n",
+    "os.environ[\"XDG_CONFIG_DIRS\"] = user_config_dir()\n",
+    "\n",
+    "import pytest\n",
     "import lamindb_setup as ln_setup\n",
     "from lamindb_setup._cache import set_cache_dir\n",
-    "from lamindb_setup.core._settings_store import system_storage_settings_file\n",
+    "from lamindb_setup.core._settings_store import (\n",
+    "    platform_user_storage_settings_file,\n",
+    "    system_settings_file,\n",
+    ")\n",
+    "from lamindb_setup.core._settings_load import load_cache_path_from_settings\n",
     "\n",
     "ln_setup.login(\"testuser2\")"
    ]
@@ -30,8 +41,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "if system_storage_settings_file().exists():\n",
-    "    system_storage_settings_file().unlink()"
+    "if platform_user_storage_settings_file().exists():\n",
+    "    platform_user_storage_settings_file().unlink()"
    ]
   },
   {
@@ -44,6 +55,16 @@
     "ln_setup.init(storage=\"s3://lamindb-ci/test-cache\", name=\"test-cache\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1f984eac",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(ln_setup.settings)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "fc410abd",
@@ -59,7 +80,54 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert not system_storage_settings_file().exists()"
+    "assert not platform_user_storage_settings_file().exists()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0fe69c33",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "initial_cache_dir = ln_setup.settings.cache_dir\n",
+    "\n",
+    "system_cache_dir = initial_cache_dir.parent / \"Cache_system\"\n",
+    "system_cache_dir.mkdir(parents=True, exist_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "90126758",
+   "metadata": {},
+   "source": [
+    "You can specify system-wide cache dir via system settings file, it has the lowest priority."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "58fe268f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "system_settings = system_settings_file()\n",
+    "system_settings.parent.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "with open(system_settings, \"w\") as f:\n",
+    "    f.write(f\"lamindb_cache_path={system_cache_dir.as_posix()}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "58de6e01",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln_setup.settings._cache_dir = None\n",
+    "\n",
+    "assert ln_setup.settings.cache_dir == system_cache_dir"
    ]
   },
   {
@@ -69,8 +137,19 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "old_cache_dir = ln_setup.settings.cache_dir\n",
-    "new_cache_dir = old_cache_dir.parent / \"Cache1\""
+    "new_cache_dir = initial_cache_dir.parent / \"Cache1\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f809a95f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with pytest.raises(ValueError) as error:\n",
+    "    set_cache_dir(\"./\")\n",
+    "assert error.exconly() == \"ValueError: A path to the cache dir should be absolute.\""
    ]
   },
   {
@@ -80,7 +159,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "set_cache_dir(new_cache_dir)"
+    "set_cache_dir(new_cache_dir)\n",
+    "\n",
+    "ln_setup.settings._cache_dir = (\n",
+    "    None  # just reset to check that the cache dir is loaded correctly\n",
+    ")"
    ]
   },
   {
@@ -90,7 +173,25 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert system_storage_settings_file().exists()"
+    "platform_user_storage_settings = platform_user_storage_settings_file()\n",
+    "\n",
+    "assert platform_user_storage_settings.exists()\n",
+    "\n",
+    "lamindb_cache_path = dotenv_values(platform_user_storage_settings)[\"lamindb_cache_path\"]\n",
+    "\n",
+    "assert lamindb_cache_path == new_cache_dir.as_posix(), lamindb_cache_path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cb75539e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loaded_cache_path = load_cache_path_from_settings()\n",
+    "\n",
+    "assert loaded_cache_path == new_cache_dir, loaded_cache_path"
    ]
   },
   {
@@ -100,7 +201,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert ln_setup.settings.cache_dir == new_cache_dir"
+    "assert ln_setup.settings.cache_dir == new_cache_dir, ln_setup.settings.cache_dir"
    ]
   },
   {
@@ -138,7 +239,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!lamin cache clear"
+    "exit_status = os.system(\"lamin cache clear\")\n",
+    "assert exit_status == 0"
    ]
   },
   {
@@ -148,7 +250,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert not any(new_cache_dir.iterdir())"
+    "cache_content = list(new_cache_dir.iterdir())\n",
+    "assert len(cache_content) == 0, cache_content"
    ]
   },
   {
@@ -176,7 +279,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert ln_setup.settings.cache_dir == old_cache_dir"
+    "assert ln_setup.settings.cache_dir == system_cache_dir"
    ]
   },
   {
@@ -196,12 +299,33 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert ln_setup.settings.cache_dir == old_cache_dir"
+    "assert ln_setup.settings.cache_dir == system_cache_dir"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1e08abf2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "system_settings.unlink()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6e86fc4a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln_setup.settings._cache_dir = None\n",
+    "assert ln_setup.settings.cache_dir == initial_cache_dir"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "a4de1b2b",
+   "id": "57c74240",
    "metadata": {},
    "source": [
     "Setting the cache dir through CLI doesn't affect the currently loaded settings because it is done in another process and the settings only check the cache dir in `settings.env` on init."
@@ -252,7 +376,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "system_storage_settings_file().unlink()\n",
+    "platform_user_storage_settings_file().unlink()\n",
     "ln_setup.delete(\"test-cache\", force=True)"
    ]
   }
@@ -273,7 +397,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.10.16"
   }
  },
  "nbformat": 4,

{lamindb_setup-1.6.1 → lamindb_setup-1.7.0}/docs/hub-prod/test-init-load-local-anonymously.ipynb

@@ -82,7 +82,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln_setup.load(\"test-anonymous-init\")\n",
+    "ln_setup.connect(\"test-anonymous-init\")\n",
     "assert ln_setup.settings.instance.name == \"test-anonymous-init\"\n",
     "assert ln_setup.settings.instance.owner == \"anonymous\""
    ]

{lamindb_setup-1.6.1 → lamindb_setup-1.7.0}/lamindb_setup/__init__.py

@@ -30,18 +30,20 @@ Modules & settings:
    settings
    core
    django
+   errors
+   types
 
 """
 
-__version__ = "1.6.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.7.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 
 from packaging import version as packaging_version
 
-from . import core
+from . import core, errors, types
 from ._check_setup import _check_instance_setup
-from ._connect_instance import connect, load
+from ._connect_instance import connect
 from ._delete import delete
 from ._disconnect import disconnect
 from ._django import django