lamindb_setup 0.71.0__tar.gz → 0.71.1__tar.gz

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.71.0
+Version: 0.71.1
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <laminlabs@gmail.com>
 Description-Content-Type: text/markdown

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/docs/changelog.md

@@ -3,6 +3,10 @@
 <!-- prettier-ignore -->
 Name | PR | Developer | Date | Version
 --- | --- | --- | --- | ---
+♻️ Extend valid suffixes to composite suffixes | [746](https://github.com/laminlabs/lamindb-setup/pull/746) | [falexwolf](https://github.com/falexwolf) | 2024-05-03 | 0.71.1
+🐛 Fix test failures | [745](https://github.com/laminlabs/lamindb-setup/pull/745) | [Koncopd](https://github.com/Koncopd) | 2024-05-03 |
+♻️ Persist keep-artifacts-local in settings.env | [743](https://github.com/laminlabs/lamindb-setup/pull/743) | [falexwolf](https://github.com/falexwolf) | 2024-05-03 |
+✅ Expand tests to multi users for managed storage | [742](https://github.com/laminlabs/lamindb-setup/pull/742) | [falexwolf](https://github.com/falexwolf) | 2024-05-03 |
 🐛 Fix delete function | [741](https://github.com/laminlabs/lamindb-setup/pull/741) | [falexwolf](https://github.com/falexwolf) | 2024-04-30 | 0.71.0
 ✨ Manage multiple storage locations with integrity | [738](https://github.com/laminlabs/lamindb-setup/pull/738) | [falexwolf](https://github.com/falexwolf) | 2024-04-30 |
 ✨ Proper progress bars for upload and download | [739](https://github.com/laminlabs/lamindb-setup/pull/739) | [Koncopd](https://github.com/Koncopd) | 2024-04-28 |

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/docs/hub-cloud/03-add-managed-storage.ipynb

@@ -38,7 +38,7 @@
     "import pytest\n",
     "from pathlib import Path\n",
     "import lamindb_setup as ln_setup\n",
-    "from lamindb_setup._add_remote_storage import add_managed_storage"
+    "from lamindb_setup._set_managed_storage import set_managed_storage"
    ]
   },
   {
@@ -84,7 +84,7 @@
    "outputs": [],
    "source": [
     "with pytest.raises(ValueError) as error:\n",
-    "    add_managed_storage(\"./storage2\")\n",
+    "    set_managed_storage(\"./storage2\")\n",
     "assert error.exconly() == \"ValueError: Can't add additional managed storage locations for instances that aren't managed through the hub.\""
    ]
   },
@@ -113,7 +113,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "add_managed_storage(\"./storage2\")"
+    "set_managed_storage(\"./storage2\")"
    ]
   },
   {
@@ -148,7 +148,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "add_managed_storage(\"./storage1\")"
+    "set_managed_storage(\"./storage1\")"
    ]
   },
   {
@@ -179,7 +179,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "add_managed_storage(\"./storage1\")"
+    "set_managed_storage(\"./storage1\")"
    ]
   },
   {
@@ -202,7 +202,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "add_managed_storage(\"./storage2\")"
+    "set_managed_storage(\"./storage2\")"
    ]
   },
   {
@@ -234,7 +234,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "add_managed_storage(\"s3://lamindb-ci/storage3\")"
+    "set_managed_storage(\"s3://lamindb-ci/storage3\")"
    ]
   },
   {
@@ -273,7 +273,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "add_managed_storage(\"s3://lamindb-ci/storage3\", cache_regions=False)"
+    "set_managed_storage(\"s3://lamindb-ci/storage3\", cache_regions=False)"
    ]
   },
   {
@@ -302,18 +302,18 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# from laminhub_rest.core.collaborator._add_collaborator import add_collaborator\n",
-    "# from lamindb_setup.core._hub_client import connect_hub_with_auth\n",
+    "from laminhub_rest.core.collaborator._add_collaborator import add_collaborator\n",
+    "from lamindb_setup.core._hub_client import connect_hub_with_auth\n",
     "\n",
-    "# admin_hub = connect_hub_with_auth()\n",
-    "# add_collaborator(\n",
-    "#     \"testuser2\",\n",
-    "#     \"testuser1\",\n",
-    "#     \"test-add-managed-storage\",\n",
-    "#     \"write\",\n",
-    "#     admin_hub,\n",
-    "# )\n",
-    "# admin_hub.auth.close()"
+    "admin_hub = connect_hub_with_auth()\n",
+    "add_collaborator(\n",
+    "    \"testuser2\",\n",
+    "    \"testuser1\",\n",
+    "    \"test-add-managed-storage\",\n",
+    "    \"write\",\n",
+    "    admin_hub,\n",
+    ")\n",
+    "admin_hub.auth.close()"
    ]
   },
   {
@@ -321,7 +321,7 @@
    "id": "e9d64cad",
    "metadata": {},
    "source": [
-    "Another user:"
+    "Sign them in and let them add another storage location:"
    ]
   },
   {
@@ -331,9 +331,28 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# ln_setup.login(\"testuser2\")\n",
-    "# add_managed_storage(\"./storage4\")\n",
-    "# assert ln_setup.settings.storage.root_as_str == f\"{Path.cwd()}/storage4\""
+    "ln_setup.login(\"testuser2\")\n",
+    "set_managed_storage(\"./storage4\")\n",
+    "assert ln_setup.settings.storage.root_as_str == f\"{Path.cwd()}/storage4\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "60ac7cb2",
+   "metadata": {},
+   "source": [
+    "Attempt to delete instance with testuser2:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6de90900",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with pytest.raises(PermissionError) as error:\n",
+    "    ln_setup.delete(\"testuser1/test-add-managed-storage\", force=True)"
    ]
   },
   {
@@ -341,7 +360,7 @@
    "id": "9d1fd197",
    "metadata": {},
    "source": [
-    "Delete test instance:"
+    "Delete test instance through testuser1:"
    ]
   },
   {

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/docs/hub-cloud/05-init-hosted-instance.ipynb

@@ -94,7 +94,9 @@
     "assert ln_setup.settings.instance.owner == ln_setup.settings.user.handle\n",
     "assert ln_setup.settings.instance.name == \"my-hosted\"\n",
     "assert ln_setup.settings.storage.root.as_posix().startswith(HOSTED_BUCKETS)\n",
-    "assert ln_setup.settings.storage.id is not None"
+    "assert ln_setup.settings.storage.id is not None\n",
+    "\n",
+    "assert ln_setup.settings.storage._mark_storage_root.exists()"
    ]
   },
   {
@@ -123,7 +125,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.9.17"
   }
  },
  "nbformat": 4,

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/docs/hub-cloud/06-connect-hosted-instance.ipynb

@@ -145,6 +145,8 @@
     "target_dir.rmdir()\n",
     "assert not target_dir.exists()\n",
     "\n",
+    "assert ln_setup.settings.storage._mark_storage_root.exists()\n",
+    "\n",
     "ln_setup.delete(\"my-hosted\", force=True)\n",
     "delete_instance(f\"testuser1/{instance_name}\")"
    ]

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.71.0"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.71.1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/lamindb_setup/_connect_instance.py

@@ -4,6 +4,7 @@ import os
 from typing import TYPE_CHECKING
 from uuid import UUID
 
+from django.db import ProgrammingError
 from lamin_utils import logger
 
 from ._check_setup import _check_instance_setup
@@ -158,6 +159,8 @@ def connect(
                     root=storage_result["root"],
                     region=storage_result["region"],
                     uid=storage_result["lnid"],
+                    uuid=UUID(storage_result["id"]),
+                    instance_id=UUID(instance_result["id"]),
                 )
                 isettings = InstanceSettings(
                     id=UUID(instance_result["id"]),
@@ -220,6 +223,17 @@ def connect(
 
         if storage is not None and isettings.dialect == "sqlite":
             update_root_field_in_default_storage(isettings)
+        # below is for backfilling the instance_uid value
+        # we'll enable it once more people migrated to 0.71.0
+        # ssettings_record = isettings.storage.record
+        # if ssettings_record.instance_uid is None:
+        #     ssettings_record.instance_uid = isettings.uid
+        #     # try saving if not read-only access
+        #     try:
+        #         ssettings_record.save()
+        #     # raised by django when the access is denied
+        #     except ProgrammingError:
+        #         pass
         load_from_isettings(isettings)
     except Exception as e:
         if isettings is not None:

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/lamindb_setup/_delete.py

@@ -6,7 +6,11 @@ from uuid import UUID
 
 from lamin_utils import logger
 
-from ._connect_instance import INSTANCE_NOT_FOUND_MESSAGE
+from ._connect_instance import (
+    INSTANCE_NOT_FOUND_MESSAGE,
+    InstanceNotFoundError,
+    get_owner_name_from_identifier,
+)
 from .core._hub_core import connect_instance as load_instance_from_hub
 from .core._hub_core import delete_instance as delete_instance_on_hub
 from .core._hub_core import get_storage_records_for_instance
@@ -54,55 +58,48 @@ def delete_by_isettings(isettings: InstanceSettings) -> None:
         settings._instance_settings = None
 
 
-def delete(
-    instance_name: str, force: bool = False, require_empty: bool = True
-) -> int | None:
+def delete(slug: str, force: bool = False, require_empty: bool = True) -> int | None:
     """Delete a LaminDB instance.
 
     Args:
-        instance_name (str): The name of the instance to delete.
-        force (bool): Whether to skip the confirmation prompt.
-        require_empty (bool): Whether to check if the instance is empty before deleting.
+        slug: The instance slug `account_handle/instance_name` or URL.
+            If the instance is owned by you, it suffices to pass the instance name.
+        force: Whether to skip the confirmation prompt.
+        require_empty: Whether to check if the instance is empty before deleting.
     """
-    if "/" in instance_name:
-        logger.warning(
-            "Deleting the instance of another user is currently not supported with the"
-            " CLI. Please provide only the instance name when deleting an instance ('/'"
-            " delimiter not allowed)."
-        )
-        raise ValueError("Invalid instance name: '/' delimiter not allowed.")
-    instance_slug = f"{settings.user.handle}/{instance_name}"
+    instance_owner, instance_name = get_owner_name_from_identifier(slug)
     if settings._instance_exists and settings.instance.name == instance_name:
         isettings = settings.instance
     else:
-        settings_file = instance_settings_file(instance_name, settings.user.handle)
+        settings_file = instance_settings_file(instance_name, instance_owner)
         if not settings_file.exists():
             hub_result = load_instance_from_hub(
-                owner=settings.user.handle, name=instance_name
+                owner=instance_owner, name=instance_name
             )
             if isinstance(hub_result, str):
                 message = INSTANCE_NOT_FOUND_MESSAGE.format(
-                    owner=settings.user.handle,
+                    owner=instance_owner,
                     name=instance_name,
                     hub_result=hub_result,
                 )
-                logger.warning(message)
-                return None
+                raise InstanceNotFoundError(message)
             instance_result, storage_result = hub_result
             ssettings = StorageSettings(
                 root=storage_result["root"],
                 region=storage_result["region"],
                 uid=storage_result["lnid"],
+                uuid=UUID(storage_result["id"]),
             )
             isettings = InstanceSettings(
                 id=UUID(instance_result["id"]),
-                owner=settings.user.handle,
+                owner=instance_owner,
                 name=instance_name,
                 storage=ssettings,
                 keep_artifacts_local=bool(instance_result["keep_artifacts_local"]),
                 db=instance_result["db"] if "db" in instance_result else None,
                 schema=instance_result["schema_str"],
                 git_repo=instance_result["git_repo"],
+                is_on_hub=True,
             )
         else:
             isettings = load_instance_settings(settings_file)
@@ -117,7 +114,7 @@ def delete(
     if not force:
         valid_responses = ["y", "yes"]
         user_input = (
-            input(f"Are you sure you want to delete instance {instance_slug}? (y/n) ")
+            input(f"Are you sure you want to delete instance {isettings.slug}? (y/n) ")
             .strip()
             .lower()
         )
@@ -161,10 +158,12 @@ def delete(
                 ssettings._mark_storage_root.unlink(
                     missing_ok=True  # this is totally weird, but needed on Py3.11
                 )
-    logger.info(f"deleting instance {instance_slug}")
+    logger.info(f"deleting instance {isettings.slug}")
     # below we can skip check_storage_is_empty() because we already called
     # it above
-    if settings.user.handle != "anonymous":
+    if settings.user.handle != "anonymous" and isettings.is_on_hub:
+        # start with deleting things on the hub
+        # this will error if the user doesn't have permission
         delete_instance_on_hub(isettings._id, require_empty=False)
     delete_by_isettings(isettings)
     # if .lndb file was delete, then we might count -1

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/lamindb_setup/_init_instance.py

@@ -277,10 +277,14 @@ def init(
 
         if isettings is not None:
             delete_by_isettings(isettings)
-            if settings.user.handle != "anonymous":
+            if settings.user.handle != "anonymous" and isettings.is_on_hub:
                 delete_instance_record(isettings._id)
             isettings._get_settings_file().unlink(missing_ok=True)  # type: ignore
-        if ssettings is not None and settings.user.handle != "anonymous":
+        if (
+            ssettings is not None
+            and settings.user.handle != "anonymous"
+            and ssettings.is_on_hub
+        ):
             delete_storage_record(ssettings._uuid)  # type: ignore
         raise e
     return None

lamindb_setup-0.71.0/lamindb_setup/_add_remote_storage.py → lamindb_setup-0.71.1/lamindb_setup/_set_managed_storage.py

@@ -12,10 +12,8 @@ if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
 
 
-def add_managed_storage(root: UPathStr, **fs_kwargs):
-    """Add a remote default storage location to a local instance.
-
-    This can be used to selectively share data.
+def set_managed_storage(root: UPathStr, **fs_kwargs):
+    """Add or switch to another managed storage location.
 
     Args:
         root: `UPathStr` - The new storage root, e.g., an S3 bucket.

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/lamindb_setup/core/_hub_core.py

@@ -53,6 +53,11 @@ def _delete_storage_record(storage_uuid: UUID, client: Client) -> None:
     response = client.table("storage").delete().eq("id", storage_uuid.hex).execute()
     if response.data:
         logger.important(f"deleted storage record on hub {storage_uuid.hex}")
+    else:
+        raise PermissionError(
+            f"Deleting of storage with {storage_uuid.hex} was not successful. Probably, you"
+            " don't have sufficient permissions."
+        )
 
 
 def update_instance_record(instance_uuid: UUID, fields: dict) -> None:

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/lamindb_setup/core/_hub_crud.py

@@ -100,7 +100,7 @@ def update_instance(instance_id: str, instance_fields: dict, client: Client):
         client.table("instance").update(instance_fields).eq("id", instance_id).execute()
     )
     if len(response.data) == 0:
-        raise RuntimeError(
+        raise PermissionError(
             f"Update of instance with {instance_id} was not successful. Probably, you"
             " don't have sufficient permissions."
         )
@@ -187,3 +187,8 @@ def _delete_instance_record(instance_id: UUID, client: Client) -> None:
     response = client.table("instance").delete().eq("id", instance_id.hex).execute()
     if response.data:
         logger.important(f"deleted instance record on hub {instance_id.hex}")
+    else:
+        raise PermissionError(
+            f"Deleting of instance with {instance_id.hex} was not successful. Probably, you"
+            " don't have sufficient permissions."
+        )

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/lamindb_setup/core/_settings_load.py

@@ -89,6 +89,7 @@ def setup_instance_from_store(store: InstanceSettingsStore) -> InstanceSettings:
         db=store.db if store.db != "null" else None,  # type: ignore
         schema=store.schema_str if store.schema_str != "null" else None,
         git_repo=store.git_repo if store.git_repo != "null" else None,
+        keep_artifacts_local=store.keep_artifacts_local,  # type: ignore
     )
 
 

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/lamindb_setup/core/_settings_save.py

@@ -41,6 +41,8 @@ def save_settings(
         for store_key, type in type_hints.items():
             if type == Optional[str]:
                 type = str
+            if type == Optional[bool]:
+                type = bool
             if "__" not in store_key:
                 if store_key == "storage_root":
                     value = settings.storage.root_as_str

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/lamindb_setup/core/_settings_storage.py

@@ -177,7 +177,6 @@ class StorageSettings:
         # local storage
         self._has_local = False
         self._local = None
-        self._is_on_hub: bool | None = None
 
     @property
     def id(self) -> int:
@@ -320,16 +319,10 @@ class StorageSettings:
 
         Only works if user has access to the instance.
         """
-        if self._is_on_hub is None:
-            from ._hub_client import call_with_fallback_auth
-            from ._hub_crud import select_storage
-
-            response = call_with_fallback_auth(select_storage, id=self._uuid.hex)  # type: ignore
-            if response is None:
-                self._is_on_hub = False
-            else:
-                self._is_on_hub = True
-        return self._is_on_hub
+        if self._uuid is None:
+            return False
+        else:
+            return True
 
     def key_to_filepath(self, filekey: Path | UPath | str) -> UPath:
         """Cloud or local filepath from filekey."""

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/lamindb_setup/core/_settings_store.py

@@ -57,6 +57,7 @@ class InstanceSettingsStore(BaseSettings):
     schema_str: Optional[str]
     id: str
     git_repo: Optional[str]
+    keep_artifacts_local: Optional[bool]
 
     class Config:
         env_prefix = "lamindb_instance_"

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/lamindb_setup/core/upath.py

@@ -56,7 +56,10 @@ VALID_SUFFIXES = {
     ".zarr",
     ".json",
 }
-
+VALID_COMPOSITE_SUFFIXES = {
+    ".anndata.zarr",
+    ".spatialdata.zarr",
+}
 
 TRAILING_SEP = (os.sep, os.altsep) if os.altsep is not None else os.sep
 
@@ -74,6 +77,12 @@ def extract_suffix_from_path(path: Path, arg_name: str | None = None) -> str:
     total_suffix = "".join(path.suffixes)
     if total_suffix in VALID_SUFFIXES:
         return total_suffix
+    elif total_suffix.endswith(tuple(VALID_COMPOSITE_SUFFIXES)):
+        # below seems slow but OK for now
+        for suffix in VALID_COMPOSITE_SUFFIXES:
+            if total_suffix.endswith(suffix):
+                break
+        return suffix
     else:
         print_hint = True
         arg_name = "file" if arg_name is None else arg_name  # for the warning

lamindb_setup-0.71.1/tests/hub-cloud/test_delete_instance.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+import lamindb_setup as ln_setup
+import pytest
+from lamindb_setup._connect_instance import InstanceNotFoundError
+
+
+def test_delete_invalid_name():
+    with pytest.raises(InstanceNotFoundError):
+        ln_setup.delete("invalid/name")

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/tests/hub-cloud/test_init_instance.py

@@ -5,6 +5,7 @@ from uuid import UUID
 
 import lamindb_setup as ln_setup
 import pytest
+from lamindb_setup._connect_instance import InstanceNotFoundError
 from lamindb_setup.core._hub_client import connect_hub_with_auth
 from lamindb_setup.core._hub_core import _connect_instance
 from lamindb_setup.core._hub_crud import (
@@ -27,7 +28,10 @@ def get_hub_client():
 def test_init_instance_postgres_default_name(get_hub_client):
     hub = get_hub_client
     instance_name = "pgtest"
-    ln_setup.delete(instance_name, force=True)
+    try:
+        ln_setup.delete(instance_name, force=True)
+    except InstanceNotFoundError:
+        pass
     # now, run init
     ln_setup.init(storage="./mydatapg", db=pgurl, _test=True)
     assert ln_setup.settings.instance.slug == "testuser2/pgtest"

lamindb_setup-0.71.1/tests/hub-cloud/test_set_storage.py

@@ -0,0 +1,15 @@
+import lamindb_setup as ln_setup
+import pytest
+from lamindb_setup._connect_instance import InstanceNotFoundError
+from lamindb_setup._set_managed_storage import set_managed_storage
+
+
+def test_set_storage_sqlite():
+    try:
+        ln_setup.delete("mydata", force=True)
+    except InstanceNotFoundError:
+        pass
+    ln_setup.init(storage="./mydata", _test=True)
+    with pytest.raises(ValueError):
+        set_managed_storage("mydata2")
+    ln_setup.delete("mydata", force=True)

{lamindb_setup-0.71.0 → lamindb_setup-0.71.1}/tests/storage/test_storage_basis.py

@@ -19,6 +19,7 @@ def test_extract_suffix_from_path():
         ("logs/date.log.123", ""),  # digits are no valid suffixes
         ("salmon.merged.gene_counts.tsv", ".tsv"),
         ("salmon.merged.gene_counts.tsv.gz", ".tsv.gz"),
+        ("filename.v1.1.0.spatialdata.zarr", ".spatialdata.zarr"),
     ]
     for path, suffix in collection:
         filepath = Path(path)

lamindb_setup-0.71.0/tests/hub-cloud/test_delete_instance.py

@@ -1,13 +0,0 @@
-from __future__ import annotations
-
-import lamindb_setup as ln_setup
-import pytest
-
-
-def test_delete_invalid_name():
-    with pytest.raises(ValueError) as error:
-        ln_setup.delete("invalid/name")
-    assert (
-        error.exconly()
-        == "ValueError: Invalid instance name: '/' delimiter not allowed."
-    )

lamindb_setup-0.71.0/tests/hub-cloud/test_set_storage.py

@@ -1,11 +0,0 @@
-import lamindb_setup as ln_setup
-import pytest
-from lamindb_setup._add_remote_storage import add_managed_storage
-
-
-def test_set_storage_sqlite():
-    ln_setup.delete("mydata", force=True)
-    ln_setup.init(storage="./mydata", _test=True)
-    with pytest.raises(ValueError):
-        add_managed_storage("mydata2")
-    ln_setup.delete("mydata", force=True)