lamindb_setup 0.73.1__tar.gz → 0.73.3__tar.gz

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/.gitignore

@@ -108,3 +108,4 @@ lamin_sphinx
 docs/conf.py
 _docs_tmp*
 *.lndb
+_is_initialized

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.73.1
+Version: 0.73.3
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <laminlabs@gmail.com>
 Description-Content-Type: text/markdown

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/docs/changelog.md

@@ -3,6 +3,12 @@
 <!-- prettier-ignore -->
 Name | PR | Developer | Date | Version
 --- | --- | --- | --- | ---
+💚 Fix test | [784](https://github.com/laminlabs/lamindb-setup/pull/784) | [falexwolf](https://github.com/falexwolf) | 2024-06-26 | 0.73.3
+♻️ Save schema to hub upon init | [782](https://github.com/laminlabs/lamindb-setup/pull/782) | [falexwolf](https://github.com/falexwolf) | 2024-06-24 |
+💚 Fix schema serialization test | [781](https://github.com/laminlabs/lamindb-setup/pull/781) | [falexwolf](https://github.com/falexwolf) | 2024-06-24 |
+Better error message | [780](https://github.com/laminlabs/lamindb-setup/pull/780) | [falexwolf](https://github.com/falexwolf) | 2024-06-20 |
+🐛 Fix permission error when screening for local storage root | [778](https://github.com/laminlabs/lamindb-setup/pull/778) | [falexwolf](https://github.com/falexwolf) | 2024-06-19 | 0.73.2
+🐛 Fix same root check in get_locker | [777](https://github.com/laminlabs/lamindb-setup/pull/777) | [Koncopd](https://github.com/Koncopd) | 2024-06-13 |
 ✨ Add .ome.zarr as composite suffix | [775](https://github.com/laminlabs/lamindb-setup/pull/775) | [sunnyosun](https://github.com/sunnyosun) | 2024-06-05 |
 🏗️ Update instance schema in the hub | [774](https://github.com/laminlabs/lamindb-setup/pull/774) | [fredericenard](https://github.com/fredericenard) | 2024-06-04 |
 ⬆️ Migrate to upath 0.2.2 | [723](https://github.com/laminlabs/lamindb-setup/pull/723) | [Koncopd](https://github.com/Koncopd) | 2024-06-02 |

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/docs/hub-prod/test-sqlite-lock.ipynb

@@ -75,6 +75,9 @@
    "source": [
     "ln_setup.init(storage=\"s3://lamindb-ci/test-load-lock\", name=\"test-load-lock\")\n",
     "instance_id = ln_setup.settings.instance._id\n",
+    "\n",
+    "assert ln_setup.settings.instance._cloud_sqlite_locker is ln_setup.settings.instance._cloud_sqlite_locker\n",
+    "\n",
     "ln_setup.close()"
    ]
   },

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.73.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.73.3"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/lamindb_setup/_init_instance.py

@@ -33,8 +33,7 @@ def get_schema_module_name(schema_name) -> str:
         if module_spec is not None:
             return name
     raise ImportError(
-        f"Python package for '{schema_name}' is not installed, tried two package names:"
-        f" {name_attempts}\nHave you installed the schema package using `pip install`?"
+        f"Python package for '{schema_name}' is not installed.\nIf your package is on PyPI, run `pip install {schema_name}`"
     )
 
 
@@ -256,7 +255,10 @@ def init(
             schema=schema,
             uid=ssettings.uid,
         )
-        if isettings.is_remote and instance_state != "instance-corrupted-or-deleted":
+        register_on_hub = (
+            isettings.is_remote and instance_state != "instance-corrupted-or-deleted"
+        )
+        if register_on_hub:
             init_instance_hub(isettings)
         validate_sqlite_state(isettings)
         isettings._persist()
@@ -270,8 +272,10 @@ def init(
                 "locked instance (to unlock and push changes to the cloud SQLite file,"
                 " call: lamin close)"
             )
-        # we can debate whether this is the right setting, but this is how
-        # things have been and we'd like to not easily break backward compat
+        if register_on_hub and isettings.dialect != "sqlite":
+            from ._schema_metadata import update_schema_in_hub
+
+            update_schema_in_hub()
         settings.auto_connect = True
     except Exception as e:
         from ._delete import delete_by_isettings

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/lamindb_setup/_migrate.py

@@ -75,21 +75,15 @@ class migrate:
         from lamindb_setup.core._hub_client import call_with_fallback_auth
         from lamindb_setup.core._hub_crud import (
             select_collaborator,
-            select_instance_by_id,
             update_instance,
         )
 
-        instance_id_str = settings.instance._id.hex
-        instance = call_with_fallback_auth(
-            select_instance_by_id, instance_id=instance_id_str
-        )
-        instance_is_on_hub = instance is not None
-        if instance_is_on_hub:
+        if settings.instance.is_on_hub:
             # double check that user is an admin, otherwise will fail below
-            # without idempotence
+            # due to insufficient SQL permissions with cryptic error
             collaborator = call_with_fallback_auth(
                 select_collaborator,
-                instance_id=instance_id_str,
+                instance_id=settings.instance._id,
                 account_id=settings.user._uuid,
             )
             if collaborator is None or collaborator["role"] != "admin":
@@ -104,7 +98,7 @@ class migrate:
         # this sets up django and deploys the migrations
         setup_django(settings.instance, deploy_migrations=True)
         # this populates the hub
-        if instance_is_on_hub:
+        if settings.instance.is_on_hub:
             logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
             # TODO: integrate update of instance table within update_schema_in_hub & below
             if settings.instance.dialect != "sqlite":

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/lamindb_setup/_register_instance.py

@@ -27,3 +27,7 @@ def register(_test: bool = False):
     init_instance_hub(isettings)
     isettings._is_on_hub = True
     isettings._persist()
+    if isettings.dialect != "sqlite" and not _test:
+        from ._schema_metadata import update_schema_in_hub
+
+        update_schema_in_hub()

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/lamindb_setup/core/_settings.py

@@ -51,7 +51,11 @@ class SetupSettings:
     @property
     def user(self) -> UserSettings:
         """:class:`~lamindb.setup.core.UserSettings`."""
-        if self._user_settings is None or self._user_settings_env != get_env_name():
+        env_changed = (
+            self._user_settings_env is not None
+            and self._user_settings_env != get_env_name()
+        )
+        if self._user_settings is None or env_changed:
             self._user_settings = load_or_create_user_settings()
             self._user_settings_env = get_env_name()
             if self._user_settings and self._user_settings.uid is None:
@@ -61,10 +65,11 @@ class SetupSettings:
     @property
     def instance(self) -> InstanceSettings:
         """:class:`~lamindb.setup.core.InstanceSettings`."""
-        if (
-            self._instance_settings is None
-            or self._instance_settings_env != get_env_name()
-        ):
+        env_changed = (
+            self._instance_settings_env is not None
+            and self._instance_settings_env != get_env_name()
+        )
+        if self._instance_settings is None or env_changed:
             self._instance_settings = load_instance_settings()
             self._instance_settings_env = get_env_name()
         return self._instance_settings  # type: ignore

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/lamindb_setup/core/_settings_instance.py

@@ -120,13 +120,25 @@ class InstanceSettings:
             if root_path.exists():
                 marker_path = root_path / ".lamindb/_is_initialized"
                 if marker_path.exists():
-                    uid = marker_path.read_text()
+                    try:
+                        uid = marker_path.read_text()
+                    except PermissionError:
+                        logger.warning(
+                            f"ignoring the following location because no permission to read it: {marker_path}"
+                        )
+                        continue
                     if uid == record.uid:
                         found = True
                         break
                     elif uid == "":
-                        # legacy instance that was not yet marked properly
-                        mark_storage_root(record.root, record.uid)
+                        try:
+                            # legacy instance that was not yet marked properly
+                            mark_storage_root(record.root, record.uid)
+                        except PermissionError:
+                            logger.warning(
+                                f"ignoring the following location because no permission to write to it: {marker_path}"
+                            )
+                            continue
                     else:
                         continue
                 else:

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/lamindb_setup/core/_settings_storage.py

@@ -121,6 +121,8 @@ def init_storage(
         init_storage_hub(ssettings)
     # below comes last only if everything else was successful
     try:
+        # (federated) credentials for AWS access are provisioned under-the-hood
+        # discussion: https://laminlabs.slack.com/archives/C04FPE8V01W/p1719260587167489
         mark_storage_root(ssettings.root, ssettings.uid)  # type: ignore
     except Exception:
         logger.important(

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/lamindb_setup/core/cloud_sqlite_locker.py

@@ -111,6 +111,7 @@ class Locker:
         if self._counter <= MAX_MSG_COUNTER:
             self._counter += 1
 
+    # Lamport's bakery algorithm
     def _lock_unsafe(self):
         if self._has_lock:
             return None
@@ -186,7 +187,7 @@ def get_locker(isettings) -> Locker:
     if (
         _locker is None
         or _locker.user != user_uid
-        or _locker.root is not storage_root
+        or _locker.root != storage_root
         or _locker.instance_id != isettings._id
     ):
         _locker = Locker(user_uid, storage_root, isettings._id)

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/tests/hub-cloud/test_connect_instance.py

@@ -28,7 +28,7 @@ def test_connect_after_revoked_access():
             # if a previous test run failed, this will
             # error with a violation of a unique constraint
             collaborator_handler.add_by_slug(
-                "laminlabs/static-test-instance-private-sqlite",
+                "testuser1/static-test-instance-private-sqlite",
                 "testuser2",
                 "write",
                 "default",
@@ -38,19 +38,19 @@ def test_connect_after_revoked_access():
             pass
         ln_setup.login("testuser2@lamin.ai")
         ln_setup.connect(
-            "https://lamin.ai/laminlabs/static-test-instance-private-sqlite", _test=True
+            "https://lamin.ai/testuser1/static-test-instance-private-sqlite", _test=True
         )
         assert (
             ln_setup.settings.instance.storage.root_as_str
             == "s3://lamindb-setup-private-bucket"
         )
         collaborator_handler.delete_by_slug(
-            "laminlabs/static-test-instance-private-sqlite", "testuser2"
+            "testuser1/static-test-instance-private-sqlite", "testuser2"
         )
         # make the instance private
         with pytest.raises(InstanceNotFoundError):
             ln_setup.connect(
-                "https://lamin.ai/laminlabs/static-test-instance-private-sqlite",
+                "https://lamin.ai/testuser1/static-test-instance-private-sqlite",
                 _test=True,
             )
 
@@ -61,7 +61,7 @@ def test_connect_after_private_public_switch():
         # this assumes that testuser1 is an admin of static-test-instance-private-sqlite
         ln_setup.login("testuser1@lamin.ai")
         ln_setup.connect(
-            "https://lamin.ai/laminlabs/static-test-instance-private-sqlite", _test=True
+            "https://lamin.ai/testuser1/static-test-instance-private-sqlite", _test=True
         )
         admin_hub = connect_hub_with_auth()
         # make the instance private
@@ -74,7 +74,7 @@ def test_connect_after_private_public_switch():
         ln_setup.login("testuser2")
         with pytest.raises(InstanceNotFoundError):
             ln_setup.connect(
-                "https://lamin.ai/laminlabs/static-test-instance-private-sqlite",
+                "https://lamin.ai/testuser1/static-test-instance-private-sqlite",
                 _test=True,
             )
         # make the instance public
@@ -85,7 +85,7 @@ def test_connect_after_private_public_switch():
         )
         # load instance with non-collaborator user, should work now
         ln_setup.connect(
-            "https://lamin.ai/laminlabs/static-test-instance-private-sqlite", _test=True
+            "https://lamin.ai/testuser1/static-test-instance-private-sqlite", _test=True
         )
         # make the instance private again
         update_instance(

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/tests/hub-local/conftest.py

@@ -14,10 +14,8 @@ local_setup_state = setup_local_hub()
 
 
 def pytest_configure():
-    if os.environ["LAMIN_ENV"] == "local":
-        local_setup_state.__enter__()
-    else:
-        logger.warning("you're running non-local tests")
+    assert os.environ["LAMIN_ENV"] == "local"
+    local_setup_state.__enter__()
 
 
 def pytest_unconfigure():

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/tests/hub-local/test_update_schema_in_hub.py

@@ -9,17 +9,20 @@ def setup_instance():
         storage="./test_storage",
         schema="bionty,wetlab",
         db="postgresql://postgres:postgres@127.0.0.1:54322/postgres",
-        name="testdb",
+        name="test-update-schema",
     )
     ln_setup.register()
     yield
-    ln_setup.delete("testdb", force=True)
+    ln_setup.delete("test-update-schema", force=True)
 
 
 def test_update_schema_in_hub(setup_instance):
     is_new, schema_uuid, schema = update_schema_in_hub()
 
-    assert is_new is True
+    # TODO: construct a test in which is_new is true
+    # currently it's false because the schema is already registered
+    # via `lamin register` earlier in the test
+    assert not is_new
     assert _dict_to_uuid(schema["json"]) == schema_uuid
 
     assert len(schema["module_set_info"]) == 3
@@ -38,6 +41,7 @@ def test_update_schema_in_hub(setup_instance):
     assert "bionty" in schema["json"]
     assert "wetlab" in schema["json"]
 
+    print(schema["json"]["core"]["artifact"]["fields"]["id"])
     assert schema["json"]["core"]["artifact"]["fields"]["id"] == {
         "type": "AutoField",
         "column": "id",
@@ -53,6 +57,7 @@ def test_update_schema_in_hub(setup_instance):
         "through": None,
     }
 
+    print(schema["json"]["core"]["artifact"]["fields"]["created_by"])
     assert schema["json"]["core"]["artifact"]["fields"]["created_by"] == {
         "type": "ForeignKey",
         "column": "created_by_id",
@@ -68,6 +73,7 @@ def test_update_schema_in_hub(setup_instance):
         "through": None,
     }
 
+    print(schema["json"]["bionty"]["gene"]["fields"]["pathways"])
     assert schema["json"]["bionty"]["gene"]["fields"]["pathways"] == {
         "type": "ManyToManyField",
         "column": None,
@@ -87,23 +93,24 @@ def test_update_schema_in_hub(setup_instance):
         },
     }
 
+    print(schema["json"]["wetlab"]["well"]["fields"]["artifacts"])
     assert schema["json"]["wetlab"]["well"]["fields"]["artifacts"] == {
         "type": "ManyToManyField",
         "column": None,
+        "through": {
+            "wetlab_well": "well_id",
+            "link_table_name": "wetlab_well_artifacts",
+            "lnschema_core_artifact": "artifact_id",
+        },
         "field_name": "artifacts",
         "model_name": "well",
         "schema_name": "wetlab",
-        "select_term": 'ARRAY(SELECT JSONB_BUILD_OBJECT((\'version\')::text, U0."version", (\'created_at\')::text, U0."created_at", (\'updated_at\')::text, U0."updated_at", (\'id\')::text, U0."id", (\'uid\')::text, U0."uid", (\'description\')::text, U0."description", (\'key\')::text, U0."key", (\'suffix\')::text, U0."suffix", (\'accessor\')::text, U0."accessor", (\'size\')::text, U0."size", (\'hash\')::text, U0."hash", (\'hash_type\')::text, U0."hash_type", (\'n_objects\')::text, U0."n_objects", (\'n_observations\')::text, U0."n_observations", (\'visibility\')::text, U0."visibility", (\'key_is_virtual\')::text, U0."key_is_virtual") AS "data" FROM "lnschema_core_artifact" U0 INNER JOIN "wetlab_well_artifacts" U1 ON (U0."id" = U1."artifact_id") WHERE U1."well_id" = ("wetlab_well"."id") LIMIT 5) AS "artifacts"',
+        "select_term": 'ARRAY(SELECT JSONB_BUILD_OBJECT((\'version\')::text, U0."version", (\'created_at\')::text, U0."created_at", (\'updated_at\')::text, U0."updated_at", (\'id\')::text, U0."id", (\'uid\')::text, U0."uid", (\'description\')::text, U0."description", (\'key\')::text, U0."key", (\'suffix\')::text, U0."suffix", (\'type\')::text, U0."type", (\'accessor\')::text, U0."accessor", (\'size\')::text, U0."size", (\'hash\')::text, U0."hash", (\'hash_type\')::text, U0."hash_type", (\'n_objects\')::text, U0."n_objects", (\'n_observations\')::text, U0."n_observations", (\'visibility\')::text, U0."visibility", (\'key_is_virtual\')::text, U0."key_is_virtual") AS "data" FROM "lnschema_core_artifact" U0 INNER JOIN "wetlab_well_artifacts" U1 ON (U0."id" = U1."artifact_id") WHERE U1."well_id" = ("wetlab_well"."id") LIMIT 5) AS "artifacts"',
         "is_link_table": False,
         "relation_type": "many-to-many",
         "related_field_name": "wells",
         "related_model_name": "artifact",
         "related_schema_name": "core",
-        "through": {
-            "wetlab_well": "well_id",
-            "link_table_name": "wetlab_well_artifacts",
-            "lnschema_core_artifact": "artifact_id",
-        },
     }
 
 

{lamindb_setup-0.73.1 → lamindb_setup-0.73.3}/tests/storage/test_storage_access.py

@@ -17,7 +17,7 @@ def test_connect_instance_with_private_storage_and_no_storage_access():
     # this should fail because of insufficient AWS permission for the storage bucket
     # that has the SQLite file on it, but because _test=True, we don't actually try to
     # pull the SQLite file -> see the line below to proxy this
-    ln_setup.connect("laminlabs/static-test-instance-private-sqlite", _test=True)
+    ln_setup.connect("testuser1/static-test-instance-private-sqlite", _test=True)
     with pytest.raises(PermissionError):
         (ln_setup.settings.storage.root / "test_file").exists()
     # connecting to a postgres instance should work because it doesn't touch