lamindb_setup 0.76.4__tar.gz → 0.76.6__tar.gz

{lamindb_setup-0.76.4 → lamindb_setup-0.76.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.76.4
+Version: 0.76.6
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown

{lamindb_setup-0.76.4 → lamindb_setup-0.76.6}/docs/hub-cloud/03-add-managed-storage.ipynb

@@ -433,7 +433,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.9.17"
   },
   "vscode": {
    "interpreter": {

{lamindb_setup-0.76.4 → lamindb_setup-0.76.6}/docs/notebooks.md

@@ -5,12 +5,12 @@
 
 hub-cloud/init-local-instance
 hub-cloud/connect-local-instance
-hub-cloud/set-storage
+hub-cloud/add-managed-storage
 hub-cloud/test-bionty
 hub-cloud/init-hosted-instance
 hub-cloud/connect-hosted-instance
-hub-cloud/test-multi-session
 hub-cloud/keep-artifacts-local
+hub-cloud/test-multi-session
 
 hub-prod/test-cache-management
 hub-prod/test-empty-init

{lamindb_setup-0.76.4 → lamindb_setup-0.76.6}/lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.76.4"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.76.6"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

{lamindb_setup-0.76.4 → lamindb_setup-0.76.6}/lamindb_setup/_init_instance.py

@@ -250,7 +250,7 @@ def init(
         # see a way of making this more elegant; should become possible if we
         # remove the instance.storage_id FK on the hub
         prevent_register_hub = is_local_db_url(db) if db is not None else False
-        ssettings = init_storage(
+        ssettings, _ = init_storage(
             storage,
             instance_id=instance_id,
             init_instance=True,

{lamindb_setup-0.76.4 → lamindb_setup-0.76.6}/lamindb_setup/_set_managed_storage.py

@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING
 from lamin_utils import logger
 
 from ._init_instance import register_storage_in_instance
+from .core._hub_core import delete_storage_record
 from .core._settings import settings
 from .core._settings_storage import init_storage
 
@@ -28,14 +29,27 @@ def set_managed_storage(root: UPathStr, **fs_kwargs):
         raise ValueError(
             "Can't add additional managed storage locations for instances that aren't managed through the hub."
         )
-    ssettings = init_storage(
+    # here the storage is registered in the hub
+    # hub_record_status="hub-record-created" if a new record is created
+    # "hub-record-retireved" if the storage is in the hub already
+    ssettings, hub_record_status = init_storage(
         root=root, instance_id=settings.instance._id, register_hub=True
     )
     if ssettings._instance_id is None:
         raise ValueError(
             f"Cannot manage storage without write access: {ssettings.root}"
         )
+
+    # here the storage is saved in the instance
+    # if any error happens the record in the hub is deleted
+    # if it was created earlier and not retrieved
+    try:
+        register_storage_in_instance(ssettings)
+    except Exception as e:
+        if hub_record_status == "hub-record-created" and ssettings._uuid is not None:
+            delete_storage_record(ssettings._uuid)  # type: ignore
+        raise e
+
     settings.instance._storage = ssettings
     settings.instance._persist()  # this also updates the settings object
-    register_storage_in_instance(ssettings)
     settings.storage._set_fs_kwargs(**fs_kwargs)

{lamindb_setup-0.76.4 → lamindb_setup-0.76.6}/lamindb_setup/core/_hub_core.py

@@ -4,7 +4,7 @@ import json
 import os
 import uuid
 from importlib import metadata
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
 from uuid import UUID
 
 from lamin_utils import logger
@@ -119,7 +119,7 @@ def _select_storage(
 def init_storage(
     ssettings: StorageSettings,
     auto_populate_instance: bool = True,
-) -> None:
+) -> Literal["hub-record-retireved", "hub-record-created"]:
     if settings.user.handle != "anonymous":
         return call_with_fallback_auth(
             _init_storage,
@@ -131,21 +131,21 @@ def init_storage(
             _select_storage, ssettings=ssettings, update_uid=True
         )
         if storage_exists:
-            return None
+            return "hub-record-retireved"
         else:
             raise ValueError("Log in to create a storage location on the hub.")
 
 
 def _init_storage(
     ssettings: StorageSettings, auto_populate_instance: bool, client: Client
-) -> None:
+) -> Literal["hub-record-retireved", "hub-record-created"]:
     from lamindb_setup import settings
 
     # storage roots are always stored without the trailing slash in the SQL
     # database
     root = ssettings.root_as_str
     if _select_storage(ssettings, update_uid=True, client=client):
-        return None
+        return "hub-record-retireved"
     if ssettings.type_is_cloud:
         id = uuid.uuid5(uuid.NAMESPACE_URL, root)
     else:
@@ -181,7 +181,7 @@ def _init_storage(
     # on root & description
     client.table("storage").upsert(fields).execute()
     ssettings._uuid_ = id
-    return None
+    return "hub-record-created"
 
 
 def delete_instance(identifier: UUID | str, require_empty: bool = True) -> str | None:

{lamindb_setup-0.76.4 → lamindb_setup-0.76.6}/lamindb_setup/core/_settings_instance.py

@@ -230,7 +230,7 @@ class InstanceSettings:
                 return None
         local_root = UPath(local_root)
         assert isinstance(local_root, LocalPathClasses)
-        self._storage_local = init_storage(local_root, self._id, register_hub=True)  # type: ignore
+        self._storage_local, _ = init_storage(local_root, self._id, register_hub=True)  # type: ignore
         register_storage_in_instance(self._storage_local)  # type: ignore
         logger.important(f"defaulting to local storage: {self._storage_local.root}")
 

{lamindb_setup-0.76.4 → lamindb_setup-0.76.6}/lamindb_setup/core/_settings_storage.py

@@ -83,10 +83,17 @@ def init_storage(
     register_hub: bool | None = None,
     prevent_register_hub: bool = False,
     init_instance: bool = False,
-) -> StorageSettings:
+) -> tuple[
+    StorageSettings,
+    Literal["hub-record-not-created", "hub-record-retireved", "hub-record-created"],
+]:
     if root is None:
         raise ValueError("`storage` argument can't be `None`")
     root_str = str(root)  # ensure we have a string
+    if ".lamindb" in root_str:
+        raise ValueError(
+            'Please pass a folder name that does not end or contain ".lamindb"'
+        )
     uid = base62(12)
     region = None
     lamin_env = os.getenv("LAMIN_ENV")
@@ -117,13 +124,19 @@ def init_storage(
         region=region,
         instance_id=instance_id,
     )
+    # this stores the result of init_storage_hub
+    hub_record_status: Literal[
+        "hub-record-not-created", "hub-record-retireved", "hub-record-created"
+    ] = "hub-record-not-created"
     # the below might update the uid with one that's already taken on the hub
     if not prevent_register_hub:
         if ssettings.type_is_cloud or register_hub:
             from ._hub_core import delete_storage_record
             from ._hub_core import init_storage as init_storage_hub
 
-            init_storage_hub(ssettings, auto_populate_instance=not init_instance)
+            hub_record_status = init_storage_hub(
+                ssettings, auto_populate_instance=not init_instance
+            )
     # below comes last only if everything else was successful
     try:
         # (federated) credentials for AWS access are provisioned under-the-hood
@@ -133,10 +146,14 @@ def init_storage(
         logger.important(
             f"due to lack of write access, LaminDB won't manage storage location: {ssettings.root}"
         )
-        if ssettings._uuid is not None:
+        # we have to check hub_record_status here because
+        # _select_storage inside init_storage_hub also populates ssettings._uuid
+        # and we don't want to delete an existing storage record here
+        # only newly created
+        if hub_record_status == "hub-record-created" and ssettings._uuid is not None:
             delete_storage_record(ssettings._uuid)  # type: ignore
         ssettings._instance_id = None
-    return ssettings
+    return ssettings, hub_record_status
 
 
 def _process_cache_path(cache_path: str | Path | UPath | None):
@@ -171,7 +188,7 @@ class StorageSettings:
                 (self._root_init / ".lamindb").mkdir(parents=True, exist_ok=True)
                 self._root_init = self._root_init.resolve()
             except Exception:
-                logger.warning("unable to create .lamindb folder")
+                logger.warning(f"unable to create .lamindb folder in {self._root_init}")
                 pass
         self._root = None
         self._instance_id = instance_id

lamindb_setup-0.76.6/tests/hub-cloud/scripts/script-to-fail-managed-storage.py

@@ -0,0 +1,26 @@
+from pathlib import Path
+
+import lamindb_setup as ln_setup
+import pytest
+from django.db.utils import ProgrammingError
+from lamindb_setup._set_managed_storage import set_managed_storage
+from lamindb_setup.core._hub_client import connect_hub_with_auth
+
+# a user should have read-only access to laminlabs/lamin-site-assets
+ln_setup.login("testuser1")
+ln_setup.connect("laminlabs/lamin-site-assets")
+
+test_root = Path("./test_script_ci_storage").resolve().as_posix()
+
+with pytest.raises(ProgrammingError) as error:
+    set_managed_storage(test_root)
+assert error.exconly().endswith(
+    "ProgrammingError: permission denied for table lnschema_core_storage"
+)
+
+hub_client = connect_hub_with_auth()
+
+records = hub_client.table("storage").select("*").eq("root", test_root).execute().data
+assert len(records) == 0
+
+hub_client.auth.sign_out()

lamindb_setup-0.76.6/tests/hub-cloud/test_fail_managed_storage.py

@@ -0,0 +1,12 @@
+import os
+import subprocess
+
+
+def test_fail_managed_storage():
+    if os.environ["LAMIN_ENV"] == "prod":
+        result = subprocess.run(
+            "python ./tests/hub-cloud/scripts/script-to-fail-managed-storage.py",
+            shell=True,
+            capture_output=True,
+        )
+        assert result.returncode == 0

{lamindb_setup-0.76.4 → lamindb_setup-0.76.6}/tests/hub-local/test_all.py

@@ -121,7 +121,7 @@ def create_myinstance(create_testadmin1_session):  # -> Dict
         # cannot yet pass instance_id here as it does not yet exist
         storage=init_storage_base(
             "s3://lamindb-ci/myinstance",
-        ),
+        )[0],
         db=db_str,
     )
     init_instance(isettings)
@@ -329,6 +329,6 @@ def test_init_storage_with_non_existing_bucket(create_testadmin1_session):
         init_storage(
             ssettings=init_storage_base(
                 "s3://non_existing_storage_root", instance_id=uuid4()
-            )
+            )[0]
         )
     assert error.exconly().endswith("Not Found")