lamindb_setup 1.5.2__tar.gz → 1.6.0__tar.gz

{lamindb_setup-1.5.2 → lamindb_setup-1.6.0}/.github/workflows/build.yml

@@ -231,14 +231,6 @@ jobs:
     needs: [hub-local, hub-cloud]
     runs-on: ubuntu-latest
     steps:
-      - uses: voxmedia/github-action-slack-notify-build@v1
-        if: ${{ needs.hub-local.result == 'success' && needs.hub-cloud.result == 'success' && github.event_name == 'repository_dispatch' }}
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.SLACK_GITHUB_ACTION }}
-        with:
-          channel_id: C05S2C02JHM
-          status: SUCCESS
-          color: good
       - uses: voxmedia/github-action-slack-notify-build@v1
         if: ${{ ( needs.hub-local.result == 'failure' || needs.hub-cloud.result == 'failure' ) && github.event_name == 'repository_dispatch' }}
         env:

{lamindb_setup-1.5.2 → lamindb_setup-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.5.2
+Version: 1.6.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10

{lamindb_setup-1.5.2 → lamindb_setup-1.6.0}/docs/hub-cloud/08-test-multi-session.ipynb

@@ -1,7 +1,6 @@
 {
  "cells": [
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -28,6 +27,15 @@
     "import pytest"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln_setup.core.django.reset_django()  # django is not configured, so no effect"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -100,7 +108,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -127,6 +134,40 @@
     "assert ln_setup.settings.instance.slug == \"testuser1/testsetup\""
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Reset `django` and connect to another instance:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln_setup.core.django.reset_django()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln_setup.connect(\"testuser1/testsetup-prepare\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert ln_setup.settings.instance.slug == \"testuser1/testsetup-prepare\""
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -141,7 +182,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "py39",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -155,7 +196,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.10.16"
   }
  },
  "nbformat": 4,

{lamindb_setup-1.5.2 → lamindb_setup-1.6.0}/lamindb_setup/__init__.py

@@ -33,7 +33,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.5.2"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.6.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 

{lamindb_setup-1.5.2 → lamindb_setup-1.6.0}/lamindb_setup/_connect_instance.py

@@ -67,10 +67,12 @@ def update_db_using_local(
     db_updated = None
     # check if postgres
     if hub_instance_result["db_scheme"] == "postgresql":
-        db_dsn_hub = LaminDsnModel(db=hub_instance_result["db"])
         if db is not None:
-            db_dsn_local = LaminDsnModel(db=db)
+            # use only the provided db if it is set
+            db_dsn_hub = LaminDsnModel(db=db)
+            db_dsn_local = db_dsn_hub
         else:
+            db_dsn_hub = LaminDsnModel(db=hub_instance_result["db"])
             # read directly from the environment
             if os.getenv("LAMINDB_INSTANCE_DB") is not None:
                 logger.important("loading db URL from env variable LAMINDB_INSTANCE_DB")
@@ -78,16 +80,15 @@ def update_db_using_local(
             # read from a cached settings file in case the hub result is only
             # read level or inexistent
             elif settings_file.exists() and (
-                db_dsn_hub.db.user is None
-                or (db_dsn_hub.db.user is not None and "read" in db_dsn_hub.db.user)
+                db_dsn_hub.db.user in {None, "none"} or "read" in db_dsn_hub.db.user  # type:ignore
             ):
                 isettings = load_instance_settings(settings_file)
                 db_dsn_local = LaminDsnModel(db=isettings.db)
             else:
                 # just take the default hub result and ensure there is actually a user
                 if (
-                    db_dsn_hub.db.user == "none"
-                    and db_dsn_hub.db.password == "none"
+                    db_dsn_hub.db.user in {None, "none"}
+                    and db_dsn_hub.db.password in {None, "none"}
                     and raise_permission_error
                 ):
                     raise PermissionError(
@@ -95,13 +96,13 @@ def update_db_using_local(
                         " a DB URL and pass it via --db <db_url>"
                     )
                 db_dsn_local = db_dsn_hub
-        if not check_db_dsn_equal_up_to_credentials(db_dsn_hub.db, db_dsn_local.db):
-            raise ValueError(
-                "The local differs from the hub database information:\n 1. did you"
-                " pass a wrong db URL with --db?\n 2. did your database get updated by"
-                " an admin?\nConsider deleting your cached database environment:\nrm"
-                f" {settings_file.as_posix()}"
-            )
+            if not check_db_dsn_equal_up_to_credentials(db_dsn_hub.db, db_dsn_local.db):
+                raise ValueError(
+                    "The local differs from the hub database information:\n"
+                    "did your database get updated by an admin?\n"
+                    "Consider deleting your cached database environment:\nrm"
+                    f" {settings_file.as_posix()}"
+                )
         db_updated = LaminDsn.build(
             scheme=db_dsn_hub.db.scheme,
             user=db_dsn_local.db.user,
@@ -345,11 +346,15 @@ def migrate_lnschema_core(
     if db_type == "sqlite":
         import sqlite3
 
+        # maybe also use LAMINDB_DJANGO_DATABASE_URL here?
         conn = sqlite3.connect(parsed_uri.path)
     elif db_type in ["postgresql", "postgres"]:
         import psycopg2
 
-        conn = psycopg2.connect(isettings.db)
+        # do not ignore LAMINDB_DJANGO_DATABASE_URL if it is set
+        conn = psycopg2.connect(
+            os.environ.get("LAMINDB_DJANGO_DATABASE_URL", isettings.db)
+        )
     else:
         raise ValueError("Unsupported database type. Use 'sqlite' or 'postgresql' URI.")
 

{lamindb_setup-1.5.2 → lamindb_setup-1.6.0}/lamindb_setup/_init_instance.py

@@ -98,13 +98,31 @@ def register_user(usettings):
 def register_initial_records(isettings: InstanceSettings, usettings):
     """Register space, user & storage in DB."""
     from django.db.utils import OperationalError
-    from lamindb.models import Space
+    from lamindb.models import Branch, Space
 
     try:
         Space.objects.get_or_create(
+            uid="A",
             name="All",
             description="Every team & user with access to the instance has access.",
         )
+        Branch.objects.get_or_create(
+            id=-1,
+            uid="T",
+            name="Trash",
+            description="The trash.",
+        )
+        Branch.objects.get_or_create(
+            id=0,
+            uid="A",
+            name="Archive",
+            description="The archive.",
+        )
+        Branch.objects.get_or_create(
+            uid="M",
+            name="Main",
+            description="The main & default branch of the instance.",
+        )
         register_user(usettings)
         register_storage_in_instance(isettings.storage)
     except OperationalError as error:
@@ -206,7 +224,7 @@ class CannotSwitchDefaultInstance(SystemExit):
 MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE = """
 You cannot write to different instances in the same Python session.
 
-Do you want to read from another instance via `Record.using()`? For example:
+Do you want to read from another instance via `SQLRecord.using()`? For example:
 
 ln.Artifact.using("laminlabs/cellxgene").filter()
 

{lamindb_setup-1.5.2 → lamindb_setup-1.6.0}/lamindb_setup/_schema_metadata.py

@@ -159,7 +159,7 @@ class FieldMetadata(BaseModel):
 
 class _ModelHandler:
     def __init__(self, model, module_name: str, included_modules: list[str]) -> None:
-        from lamindb.models import LinkORM
+        from lamindb.models import IsLink
 
         self.model = model
         self.class_name = model.__name__
@@ -168,7 +168,7 @@ class _ModelHandler:
         self.table_name = model._meta.db_table
         self.included_modules = included_modules
         self.fields = self._get_fields_metadata(self.model)
-        self.is_link_table = issubclass(model, LinkORM)
+        self.is_link_table = issubclass(model, IsLink)
         self.name_field = model._name_field if hasattr(model, "_name_field") else None
         self.ontology_id_field = (
             model._ontology_id_field if hasattr(model, "_ontology_id_field") else None
@@ -245,7 +245,7 @@ class _ModelHandler:
         return related_fields
 
     def _get_field_metadata(self, model, field: Field):
-        from lamindb.models import LinkORM
+        from lamindb.models import IsLink
 
         internal_type = field.get_internal_type()
         model_name = field.model._meta.model_name
@@ -296,7 +296,7 @@ class _ModelHandler:
             model_name=model_name,
             field_name=field_name,
             type=internal_type,
-            is_link_table=issubclass(field.model, LinkORM),
+            is_link_table=issubclass(field.model, IsLink),
             is_primary_key=is_primary_key,
             is_editable=is_editable,
             max_length=max_length,
@@ -400,7 +400,7 @@ class _SchemaHandler:
         return self.to_dict(include_django_objects=False)
 
     def _get_modules_metadata(self):
-        from lamindb.models import Record, Registry
+        from lamindb.models import Registry, SQLRecord
 
         all_models = {
             module_name: {
@@ -411,7 +411,7 @@ class _SchemaHandler:
                     module_name
                 ).models.__dict__.values()
                 if model.__class__ is Registry
-                and model is not Record
+                and model is not SQLRecord
                 and not model._meta.abstract
                 and model.__get_module_name__() == module_name
             }

{lamindb_setup-1.5.2 → lamindb_setup-1.6.0}/lamindb_setup/core/_hub_core.py

@@ -36,6 +36,7 @@ from ._hub_utils import (
 from ._settings import settings
 from ._settings_instance import InstanceSettings
 from ._settings_storage import StorageSettings, base62
+from .hashing import hash_and_encode_as_b62
 
 if TYPE_CHECKING:
     from supabase import Client  # type: ignore
@@ -116,6 +117,28 @@ def _select_storage(
         return True
 
 
+def _select_storage_or_parent(path: str, client: Client) -> dict | None:
+    result = client.rpc("existing_root_or_child", {"_path": path}).execute().data
+    if result["root"] is None:
+        return None
+    result["uid"] = result.pop("lnid")
+    result["instance_uid"] = hash_and_encode_as_b62(
+        UUID(result.pop("instance_id")).hex
+    )[:12]
+    return result
+
+
+def select_storage_or_parent(path: str, access_token: str | None = None) -> dict | None:
+    if settings.user.handle != "anonymous" or access_token is not None:
+        return call_with_fallback_auth(
+            _select_storage_or_parent,
+            path=path,
+            access_token=access_token,
+        )
+    else:
+        return call_with_fallback(_select_storage_or_parent, path=path)
+
+
 def init_storage_hub(
     ssettings: StorageSettings,
     auto_populate_instance: bool = True,

{lamindb_setup-1.5.2 → lamindb_setup-1.6.0}/lamindb_setup/core/_settings.py

@@ -5,6 +5,7 @@ import sys
 from typing import TYPE_CHECKING
 
 from appdirs import AppDirs
+from lamin_utils import logger
 
 from ._settings_load import (
     load_instance_settings,
@@ -162,7 +163,16 @@ class SetupSettings:
             self._cache_dir = cache_dir
         else:
             cache_dir = self._cache_dir
-        cache_dir.mkdir(parents=True, exist_ok=True)
+        try:
+            cache_dir.mkdir(parents=True, exist_ok=True)
+        # we don not want this to error
+        # beause no actual writing happens on just getting the cache dir
+        # in cloud_to_local_no_update for example
+        # so it should not fail on read-only systems
+        except Exception as e:
+            logger.warning(
+                f"Failed to create lamin cache directory at {cache_dir}: {e}"
+            )
         return cache_dir
 
     @property

{lamindb_setup-1.5.2 → lamindb_setup-1.6.0}/lamindb_setup/core/_settings_store.py

@@ -2,6 +2,7 @@ import os
 from pathlib import Path
 from typing import Optional
 
+from lamin_utils import logger
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 if "LAMIN_SETTINGS_DIR" in os.environ:
@@ -12,7 +13,11 @@ else:
     # hence, let's take home/.lamin
     settings_dir = Path.home() / ".lamin"
 
-settings_dir.mkdir(parents=True, exist_ok=True)
+
+try:
+    settings_dir.mkdir(parents=True, exist_ok=True)
+except Exception as e:
+    logger.warning(f"Failed to create lamin settings directory at {settings_dir}: {e}")
 
 
 def get_settings_file_name_prefix():

{lamindb_setup-1.5.2 → lamindb_setup-1.6.0}/lamindb_setup/core/django.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 # flake8: noqa
 import builtins
 import os
+import sys
+import importlib as il
 import jwt
 import time
 from pathlib import Path
@@ -170,8 +172,7 @@ def setup_django(
 
         module_names = ["core"] + list(isettings.modules)
         raise_import_error = True if init else False
-        installed_apps = ["django.contrib.contenttypes"]
-        installed_apps += [
+        installed_apps = [
             package_name
             for name in module_names
             if (
@@ -242,3 +243,44 @@ def setup_django(
 
     if isettings.keep_artifacts_local:
         isettings._search_local_root()
+
+
+# THIS IS NOT SAFE
+# especially if lamindb is imported already
+# django.setup fails if called for the second time
+# reset_django() allows to call setup again,
+# needed to connect to a different instance in the same process if connected already
+# there could be problems if models are already imported from lamindb or other modules
+# these 'old' models can have any number of problems
+def reset_django():
+    from django.conf import settings
+    from django.apps import apps
+    from django.db import connections
+
+    if not settings.configured:
+        return
+
+    connections.close_all()
+
+    if getattr(settings, "_wrapped", None) is not None:
+        settings._wrapped = None
+
+    app_names = {"django"} | {app.name for app in apps.get_app_configs()}
+
+    apps.app_configs.clear()
+    apps.apps_ready = apps.models_ready = apps.ready = apps.loading = False
+    apps.clear_cache()
+
+    # i suspect it is enough to just drop django and all the apps from sys.modules
+    # the code above is just a precaution
+    for module_name in list(sys.modules):
+        if module_name.partition(".")[0] in app_names:
+            del sys.modules[module_name]
+
+    il.invalidate_caches()
+
+    global db_token_manager
+    db_token_manager = DBTokenManager()
+
+    global IS_SETUP
+    IS_SETUP = False

{lamindb_setup-1.5.2 → lamindb_setup-1.6.0}/tests/hub-cloud/test_connect_instance.py

@@ -117,31 +117,22 @@ def test_connect_with_db_parameter():
         ln_setup.login("testuser1")
         # test load from hub
         ln_setup.connect("laminlabs/lamindata", _test=True)
-        assert "root" in ln_setup.settings.instance.db
+        # this test assumes fine-grained access
+        assert ln_setup.settings.instance._db_permissions == "jwt"
+        assert "jwt" in ln_setup.settings.instance.db
         # test load from provided db argument
         db = "postgresql://testdbuser:testpwd@database2.cmyfs24wugc3.us-east-1.rds.amazonaws.com:5432/db1"
         ln_setup.connect("laminlabs/lamindata", _db=db, _test=True)
         assert "testdbuser" in ln_setup.settings.instance.db
-        # test ignore loading from cache because hub result has >read access
+        # test ignore loading from cache because hub result has jwt access
         ln_setup.connect("laminlabs/lamindata", _test=True)
-        assert "root" in ln_setup.settings.instance.db
+        assert "jwt" in ln_setup.settings.instance.db
 
         # now take a user that has no collaborator status
         ln_setup.login("testuser2")
-        # the cached high priviledge connection string remains active
+        # receives public connection
         ln_setup.connect("laminlabs/lamindata", _test=True)
-        assert "root" in ln_setup.settings.instance.db
+        assert "public" in ln_setup.settings.instance.db
         # now pass the connection string
         ln_setup.connect("laminlabs/lamindata", _db=db, _test=True)
         assert "testdbuser" in ln_setup.settings.instance.db
-        # now the cache is used
-        ln_setup.connect("laminlabs/lamindata", _test=True)
-        assert "testdbuser" in ln_setup.settings.instance.db
-
-        # test corrupted input
-        db_corrupted = "postgresql://testuser:testpwd@wrongserver:5432/db1"
-        with pytest.raises(ValueError) as error:
-            ln_setup.connect("laminlabs/lamindata", _db=db_corrupted, _test=True)
-        assert error.exconly().startswith(
-            "ValueError: The local differs from the hub database information"
-        )

{lamindb_setup-1.5.2 → lamindb_setup-1.6.0}/tests/hub-local/test_all.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import os
 import subprocess
+from unittest.mock import patch
 from uuid import UUID, uuid4
 
 import lamindb_setup as ln_setup
@@ -17,6 +18,7 @@ from lamindb_setup.core._hub_core import (
     connect_instance_hub,
     init_instance_hub,
     init_storage_hub,
+    select_storage_or_parent,
     sign_in_hub,
     sign_up_local_hub,
 )
@@ -39,7 +41,6 @@ from lamindb_setup.core._settings_storage import init_storage as init_storage_ba
 from lamindb_setup.core._settings_store import instance_settings_file
 from lamindb_setup.core._settings_user import UserSettings
 from laminhub_rest.core.legacy._instance_collaborator import InstanceCollaboratorHandler
-from laminhub_rest.core.organization import OrganizationHandler
 from laminhub_rest.test.instance.utils import (
     create_hosted_test_instance,
     delete_hosted_test_instance,
@@ -168,11 +169,6 @@ def create_myinstance(create_testadmin1_session):  # -> Dict
 
 @pytest.fixture(scope="session")
 def create_instance_fine_grained_access(create_testadmin1_session):
-    client, testadmin1 = create_testadmin1_session
-
-    org_handler = OrganizationHandler(client)
-    org_handler.create(testadmin1._uuid)
-
     instance = create_hosted_test_instance("instance_access_v2", access_v2=True)
     yield instance
     delete_hosted_test_instance(instance)
@@ -390,6 +386,20 @@ def test_init_storage_incorrect_protocol():
     assert "Protocol incorrect-protocol is not supported" in error.exconly()
 
 
+def test_select_storage_or_parent(create_myinstance):
+    # check not exisitng
+    assert select_storage_or_parent("s3://does-not-exist") is None
+
+    root = "s3://lamindb-ci/myinstance"
+
+    result = select_storage_or_parent(root)
+    assert result["root"] == root
+    # check with a child path and anonymous user
+    with patch.object(ln_setup.settings.user, "handle", new="anonymous"):
+        result = select_storage_or_parent(root + "/subfolder")
+    assert result["root"] == root
+
+
 def test_fine_grained_access(
     create_testadmin1_session, create_instance_fine_grained_access
 ):