lamindb_setup 1.10.1__tar.gz → 1.11.0__tar.gz

{lamindb_setup-1.10.1 → lamindb_setup-1.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.10.1
+Version: 1.11.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10

{lamindb_setup-1.10.1 → lamindb_setup-1.11.0}/docs/hub-cloud/03-add-managed-storage.ipynb

@@ -52,6 +52,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import os\n",
     "import laminci\n",
     "import pytest\n",
     "from pathlib import Path\n",
@@ -305,6 +306,16 @@
     "Cloud storage:"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "47affaad",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cloud_storage = f\"s3://lamindb-ci/storage3_{os.getenv('LAMIN_ENV', 'prod')}\""
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -312,7 +323,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "set_managed_storage(\"s3://lamindb-ci/storage3\")"
+    "set_managed_storage(cloud_storage)"
    ]
   },
   {
@@ -327,7 +338,7 @@
    "outputs": [],
    "source": [
     "assert ln_setup.settings.storage.type_is_cloud\n",
-    "assert ln_setup.settings.storage.root_as_str == \"s3://lamindb-ci/storage3\"\n",
+    "assert ln_setup.settings.storage.root_as_str == cloud_storage\n",
     "assert ln_setup.settings.storage.region == \"us-west-1\"\n",
     "assert (\n",
     "    ln_setup.settings.storage.root / \".lamindb/storage_uid.txt\"\n",
@@ -354,7 +365,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "set_managed_storage(\"s3://lamindb-ci/storage3\", cache_regions=False)"
+    "set_managed_storage(cloud_storage, cache_regions=False)"
    ]
   },
   {
@@ -395,71 +406,61 @@
    "id": "83a2ee6b",
    "metadata": {},
    "source": [
-    "Add testuser2 as a collaborator to the instance:"
+    "Add testuser2 as a collaborator to the instance, sign them in and let them add another storage location:"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "9a4029b0",
+   "id": "67e455af",
    "metadata": {},
    "outputs": [],
    "source": [
-    "from laminhub_rest.core.legacy._instance_collaborator import InstanceCollaboratorHandler\n",
-    "from laminhub_rest.core.account import AccountHandler\n",
     "from lamindb_setup.core._hub_client import connect_hub_with_auth\n",
-    "from lamindb.models import User\n",
+    "from laminhub_rest.core._central_client import SupabaseClientWrapper\n",
+    "from laminhub_rest.core.instance_collaborator import InstanceCollaboratorHandler\n",
+    "from laminhub_rest.core.organization import OrganizationMemberHandler\n",
+    "\n",
+    "admin_hub = SupabaseClientWrapper(connect_hub_with_auth())\n",
+    "organization_member_handler = OrganizationMemberHandler(admin_hub)\n",
+    "\n",
+    "assert ln_setup.settings.user.handle == \"testuser1\"\n",
+    "organization_id = ln_setup.settings.user._uuid  # testuser1\n",
     "\n",
-    "admin_hub = connect_hub_with_auth()\n",
-    "testuser2 = AccountHandler(admin_hub).get_by_handle(\"testuser2\")\n",
-    "InstanceCollaboratorHandler(admin_hub).add(\n",
-    "    instance_id=instance_id,\n",
-    "    account_id=testuser2.id,\n",
-    "    role=\"write\",\n",
-    "    schema_id=None,\n",
-    "    skip_insert_user_table=True,\n",
-    ")\n",
-    "User.objects.create(uid=testuser2.lnid, handle=testuser2.handle, name=testuser2.name)\n",
-    "admin_hub.auth.close()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "e9d64cad",
-   "metadata": {},
-   "source": [
-    "Sign them in and let them add another storage location:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d6915638-db49-4d3d-bd91-819bb4719ef7",
-   "metadata": {},
-   "outputs": [],
-   "source": [
     "ln_setup.login(\"testuser2\")\n",
-    "set_managed_storage(\"./storage4\", host=\"testuser2-laptop\")\n",
-    "assert ln_setup.settings.storage.root_as_str == f\"{Path.cwd()}/storage4\""
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "60ac7cb2",
-   "metadata": {},
-   "source": [
-    "Attempt to delete instance with testuser2:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6de90900",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "with pytest.raises(PermissionError) as error:\n",
-    "    ln_setup.delete(\"testuser1/test-add-managed-storage\", force=True)"
+    "assert ln_setup.settings.user.handle == \"testuser2\"\n",
+    "account_id = ln_setup.settings.user._uuid\n",
+    "\n",
+    "try:\n",
+    "    organization_member_handler.add(\n",
+    "        organization_id=organization_id,\n",
+    "        account_id=account_id,\n",
+    "        role=\"member\",\n",
+    "    )\n",
+    "    try:\n",
+    "        InstanceCollaboratorHandler(admin_hub).add(\n",
+    "            account_id=account_id,\n",
+    "            instance_id=instance_id,\n",
+    "            role=\"write\",\n",
+    "        )\n",
+    "    except TypeError:\n",
+    "        # the above failes in the mid due to no resource_db_server associated with the instance\n",
+    "        # but still adds the collaborator\n",
+    "        pass\n",
+    "\n",
+    "    # although technically a user doesn't have to be a collaborator to insert into storage table\n",
+    "    set_managed_storage(\"./storage4\", host=\"testuser2-laptop\")\n",
+    "    assert ln_setup.settings.storage.root_as_str == f\"{Path.cwd()}/storage4\"\n",
+    "    # attempt to delete the instance with testuser2\n",
+    "    with pytest.raises(PermissionError) as error:\n",
+    "        ln_setup.delete(\"testuser1/test-add-managed-storage\", force=True)\n",
+    "\n",
+    "finally:\n",
+    "    organization_member_handler.remove(\n",
+    "        organization_id=organization_id,\n",
+    "        account_id=account_id,\n",
+    "    )\n",
+    "    admin_hub.auth.sign_out(options={\"scope\": \"local\"})"
    ]
   },
   {
@@ -510,7 +511,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "py312",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -524,7 +525,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.8"
+   "version": "3.10.16"
   }
  },
  "nbformat": 4,

{lamindb_setup-1.10.1 → lamindb_setup-1.11.0}/lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.10.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.11.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 

{lamindb_setup-1.10.1 → lamindb_setup-1.11.0}/lamindb_setup/_connect_instance.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import importlib
 import os
 import sys
+import types
 from typing import TYPE_CHECKING, Any
 from uuid import UUID
 
@@ -184,24 +185,26 @@ def _connect_instance(
 def reset_django_module_variables():
     # This function updates all module-level references to Django classes
     # But it will fail to update function level references
+    # This is not a problem unless for the function that calls ln.connect() itself
     # So, if a user has
+    #
     # def my_function():
     #     import lamindb as ln
-    #     ...
+    #     ln.connect(...)
     #
-    # Then it will **not** work and the `ln` variable will become stale and hold a reference
+    # Then it will **not** work and the `ln` variable becomes stale and hold a reference
     # to the old classes
-    # There doesn't seem to be an easy way to fix this problem
-
-    logger.important_hint("resetting django module variables")
-
-    import types
+    # Other functions that dynamically import are no problem because the variables
+    # are automatically refreshed when the function runs the next time after ln.connect() was called
+    logger.debug("resetting django module variables")
 
+    # django.apps needs to be a local import to refresh variables
     from django.apps import apps
 
     app_names = {app.name for app in apps.get_app_configs()}
-
-    for name, module in sys.modules.items():
+    # always copy before iterations over sys.modules
+    # see https://docs.python.org/3/library/sys.html#sys.modules
+    for name, module in sys.modules.copy().items():
         if (
             module is not None
             and (not name.startswith("__") or name == "__main__")

{lamindb_setup-1.10.1 → lamindb_setup-1.11.0}/lamindb_setup/_init_instance.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import importlib
 import os
 import uuid
+from pathlib import Path
 from typing import TYPE_CHECKING, Literal
 from uuid import UUID
 

{lamindb_setup-1.10.1 → lamindb_setup-1.11.0}/lamindb_setup/_migrate.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import requests  # type: ignore
 from django.db import connection
 from django.db.migrations.loader import MigrationLoader
 from lamin_utils import logger
@@ -142,6 +143,13 @@ class migrate:
             logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
             if settings.instance.dialect != "sqlite":
                 update_schema_in_hub()
+                logger.warning(
+                    "clearing instance cache in hub; if this fails, re-run with latest lamindb version"
+                )
+                requests.delete(
+                    f"{settings.instance.api_url}/cache/instances/{settings.instance._id.hex}",
+                    headers={"Authorization": f"Bearer {settings.user.access_token}"},
+                )
             call_with_fallback_auth(
                 update_instance,
                 instance_id=settings.instance._id.hex,

{lamindb_setup-1.10.1 → lamindb_setup-1.11.0}/lamindb_setup/_schema_metadata.py

@@ -404,23 +404,27 @@ class _SchemaHandler:
         return self.to_dict(include_django_objects=False)
 
     def _get_modules_metadata(self):
+        from django.apps import apps
         from lamindb.models import Registry, SQLRecord
 
-        all_models = {
-            module_name: {
-                model._meta.model_name: _ModelHandler(
-                    model, module_name, self.included_modules
-                )
-                for model in self._get_schema_module(
-                    module_name
-                ).models.__dict__.values()
-                if model.__class__ is Registry
+        all_models = {module_name: {} for module_name in self.included_modules}
+
+        # Iterate through all registered Django models
+        for model in apps.get_models():
+            # Check if model meets the criteria
+            if (
+                model.__class__ is Registry
                 and model is not SQLRecord
                 and not model._meta.abstract
-                and model.__get_module_name__() == module_name
-            }
-            for module_name in self.included_modules
-        }
+            ):
+                module_name = model.__get_module_name__()
+                # Only include if module is in our included list
+                if module_name in self.included_modules:
+                    model_name = model._meta.model_name
+                    all_models[module_name][model_name] = _ModelHandler(
+                        model, module_name, self.included_modules
+                    )
+
         assert all_models
         return all_models
 

{lamindb_setup-1.10.1 → lamindb_setup-1.11.0}/lamindb_setup/core/_hub_core.py

@@ -661,7 +661,12 @@ def _sign_in_hub(email: str, password: str, handle: str | None, client: Client):
             "password": password,
         }
     )
-    data = client.table("account").select("*").eq("id", auth.user.id).execute().data
+    # normally public.account.id is equal to auth.user.id
+    # but it might be not the case in the future
+    # this is why we check public.account.user_id that references auth.user.id
+    data = (
+        client.table("account").select("*").eq("user_id", auth.user.id).execute().data
+    )
     if data:  # sync data from hub to local cache in case it was updated on the hub
         user = data[0]
         user_uuid = UUID(user["id"])
@@ -710,8 +715,9 @@ def _sign_in_hub_api_key(api_key: str, client: Client):
     # probably need more info here to avoid additional queries
     # like handle, uid etc
     account_id = jwt.decode(access_token, options={"verify_signature": False})["sub"]
+
     client.postgrest.auth(access_token)
-    # normally public.account.id is equal to auth.user.id
+
     data = client.table("account").select("*").eq("id", account_id).execute().data
     if data:
         user = data[0]

{lamindb_setup-1.10.1 → lamindb_setup-1.11.0}/lamindb_setup/core/_settings.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import os
 import sys
 import warnings
+from pathlib import Path
 from typing import TYPE_CHECKING
 
 from lamin_utils import logger
@@ -23,8 +24,6 @@ from ._settings_store import (
 from .upath import LocalPathClasses, UPath
 
 if TYPE_CHECKING:
-    from pathlib import Path
-
     from lamindb.models import Branch, Space
 
     from lamindb_setup.core import InstanceSettings, StorageSettings, UserSettings
@@ -60,6 +59,7 @@ class SetupSettings:
 
     _auto_connect_path: Path = settings_dir / "auto_connect"
     _private_django_api_path: Path = settings_dir / "private_django_api"
+    _work_dir: Path = settings_dir / "work_dir.txt"
 
     _cache_dir: Path | None = None
 
@@ -70,6 +70,25 @@ class SetupSettings:
     def _instance_settings_path(self) -> Path:
         return current_instance_settings_file()
 
+    @property
+    def work_dir(self) -> Path | None:
+        """Get or set the current working directory.
+
+        If setting it to `None`, the working directory is unset
+        """
+        if not self._work_dir.exists():
+            return None
+        return Path(self._work_dir.read_text())
+
+    @work_dir.setter
+    def work_dir(self, value: str | Path | None) -> None:
+        if value is None:
+            if self._work_dir.exists():
+                self._work_dir.unlink()
+        else:
+            value_str = Path(value).expanduser().resolve().as_posix()
+            self._work_dir.write_text(value_str)
+
     @property
     def settings_dir(self) -> Path:
         """The directory that holds locally persisted settings."""
@@ -317,6 +336,7 @@ class SetupSettings:
         repr += "\nConfig:\n"
         repr += f" - private Django API: {self.private_django_api}\n"
         repr += "Local directories:\n"
+        repr += f" - working directory: {self.work_dir}\n"
         repr += f" - cache: {self.cache_dir.as_posix()}\n"
         repr += f" - user settings: {settings_dir.as_posix()}\n"
         repr += f" - system settings: {system_settings_dir.as_posix()}\n"

{lamindb_setup-1.10.1 → lamindb_setup-1.11.0}/lamindb_setup/core/django.py

@@ -168,7 +168,10 @@ def setup_django(
                 ssl_require = False
             else:
                 ssl_require = not is_local_db_url(instance_db)
-            options = {"connect_timeout": os.getenv("PGCONNECT_TIMEOUT", 20)}
+            options = {
+                "connect_timeout": os.getenv("PGCONNECT_TIMEOUT", 20),
+                "gssencmode": "disable",
+            }
         else:
             ssl_require = False
             options = {}

{lamindb_setup-1.10.1 → lamindb_setup-1.11.0}/lamindb_setup/core/upath.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import math
 import os
 import warnings
 from collections import defaultdict
@@ -353,6 +354,15 @@ def upload_from(
             destination = fsspec.utils.other_paths(
                 files, self.as_posix(), exists=False, flatten=False
             )
+    elif self.protocol == "s3" and "chunksize" not in kwargs:
+        size = local_path.stat().st_size
+        MiB = 1024**2
+        DEFAULT_CHUNKSIZE = 50 * MiB  # so in s3fs
+        if size / DEFAULT_CHUNKSIZE > 10000:  # should be no more than 10k parts for s3
+            raw = math.ceil(size / 10000)
+            step = 5 * MiB
+            rounded = math.ceil(raw / step) * step
+            kwargs["chunksize"] = rounded
 
     # the below lines are to avoid s3fs triggering create_bucket in upload if
     # dirs are present, it allows to avoid the permission error

{lamindb_setup-1.10.1 → lamindb_setup-1.11.0}/tests/hub-cloud/scripts/script-init-pass-user-no-writes.py

@@ -16,7 +16,7 @@ current_instance_exists = current_instance_file.exists()
 testuser1 = load_user_settings(user_settings_file_handle("testuser1"))
 assert testuser1.handle == "testuser1"
 
-storage = f"s3://lamindb-ci/{os.environ['LAMIN_ENV']}_test/test-init-no-writes"
+storage = f"s3://lamindb-ci/{os.getenv('LAMIN_ENV', 'prod')}_test/test-init-no-writes"
 
 ln_setup.init(
     storage=storage,

{lamindb_setup-1.10.1 → lamindb_setup-1.11.0}/tests/hub-cloud/scripts/script-to-fail-managed-storage.py

@@ -14,9 +14,10 @@ test_root = Path("./test_script_ci_storage").resolve().as_posix()
 
 with pytest.raises(ProgrammingError) as error:
     set_managed_storage(test_root, host="test-host-1234")
-assert error.exconly().endswith(
+error_str = error.exconly()
+assert error_str.endswith(
     "ProgrammingError: permission denied for table lamindb_storage"
-)
+), error_str
 
 hub_client = connect_hub_with_auth()
 

{lamindb_setup-1.10.1 → lamindb_setup-1.11.0}/tests/hub-cloud/test_connect_instance.py

@@ -39,7 +39,6 @@ def test_connect_after_revoked_access():
                 "testuser1/static-test-instance-private-sqlite",
                 "testuser2",
                 "write",
-                "default",
                 skip_insert_user_table=True,
             )
         except APIError: