lamindb_setup 1.7.3__tar.gz → 1.8.0__tar.gz

{lamindb_setup-1.7.3 → lamindb_setup-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.7.3
+Version: 1.8.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10

{lamindb_setup-1.7.3 → lamindb_setup-1.8.0}/docs/hub-cloud/03-add-managed-storage.ipynb

@@ -83,6 +83,16 @@
     "ln_setup.init(storage=\"./storage1\", name=\"test-add-managed-storage\", db=pgurl)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "95a61826",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln_setup.settings.instance.is_on_hub"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "b62e84cb",
@@ -148,6 +158,16 @@
     "ln_setup.register()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "825c34b7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln_setup.settings.instance.is_on_hub"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "f7e1bfaa",
@@ -163,7 +183,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "set_managed_storage(\"./storage2\")"
+    "set_managed_storage(\"./storage2\", host=\"testuser1-laptop\")"
    ]
   },
   {
@@ -200,7 +220,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "set_managed_storage(\"./storage1\")"
+    "set_managed_storage(\"./storage1\", host=\"testuser1-laptop\")"
    ]
   },
   {
@@ -233,7 +253,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "set_managed_storage(\"./storage1\")"
+    "set_managed_storage(\"./storage1\", host=\"testuser1-laptop\")"
    ]
   },
   {
@@ -258,7 +278,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "set_managed_storage(\"./storage2\")"
+    "set_managed_storage(\"./storage2\", host=\"testuser1-laptop\")"
    ]
   },
   {
@@ -419,7 +439,7 @@
    "outputs": [],
    "source": [
     "ln_setup.login(\"testuser2\")\n",
-    "set_managed_storage(\"./storage4\")\n",
+    "set_managed_storage(\"./storage4\", host=\"testuser2-laptop\")\n",
     "assert ln_setup.settings.storage.root_as_str == f\"{Path.cwd()}/storage4\""
    ]
   },

{lamindb_setup-1.7.3 → lamindb_setup-1.8.0}/docs/hub-cloud/07-keep-artifacts-local.ipynb

@@ -22,6 +22,8 @@
     "from upath import UPath\n",
     "from pathlib import Path\n",
     "\n",
+    "# os.environ[\"LAMIN_ENV\"] = \"prod\"\n",
+    "\n",
     "name = f\"keep-artifacts-local-setup-{os.environ['LAMIN_ENV']}\"\n",
     "storage = UPath(f\"s3://lamindb-ci/{name}\").as_posix()\n",
     "\n",
@@ -43,7 +45,7 @@
    "outputs": [],
    "source": [
     "with pytest.raises(ValueError) as error:\n",
-    "    ln_setup.settings.instance.storage_local\n",
+    "    ln_setup.settings.instance.local_storage\n",
     "assert (\n",
     "    error.exconly()\n",
     "    == \"ValueError: `keep_artifacts_local` is not enabled for this instance.\"\n",
@@ -58,7 +60,11 @@
    "source": [
     "ln_setup.settings.instance._keep_artifacts_local = True\n",
     "with pytest.raises(ValueError) as error:\n",
-    "    ln_setup.settings.instance.storage_local"
+    "    ln_setup.settings.instance.local_storage\n",
+    "assert (\n",
+    "    error.exconly()\n",
+    "    == \"ValueError: No storage location found in current environment: create one via, e.g., ln.Storage(root='/dir/our_shared_dir', host='our-server-123).save()\"\n",
+    ")"
    ]
   },
   {
@@ -68,16 +74,17 @@
    "outputs": [],
    "source": [
     "# now set local storage location\n",
-    "ln_setup.settings.instance.storage_local = \"./my_storage_local\"\n",
+    "ln_setup.settings.instance.local_storage = \"./my_local_storage\", \"testuser1-laptop\"\n",
     "\n",
     "assert (\n",
-    "    ln_setup.settings.instance.storage_local.root.as_posix()\n",
-    "    == UPath(\"./my_storage_local\").resolve().as_posix()\n",
+    "    ln_setup.settings.instance.local_storage.root.as_posix()\n",
+    "    == UPath(\"./my_local_storage\").resolve().as_posix()\n",
     ")\n",
     "assert (\n",
-    "    ln_setup.settings.instance.storage_local.root / \".lamindb/storage_uid.txt\"\n",
-    ").read_text().splitlines()[0] == ln_setup.settings.instance.storage_local.uid\n",
-    "assert ln_setup.settings.instance.storage_local is not None\n",
+    "    ln_setup.settings.instance.local_storage.root / \".lamindb/storage_uid.txt\"\n",
+    ").read_text().splitlines()[0] == ln_setup.settings.instance.local_storage.uid\n",
+    "assert ln_setup.settings.instance.local_storage is not None\n",
+    "assert ln_setup.settings.instance.local_storage.region == \"testuser1-laptop\"\n",
     "# the remote storage location is still in the regular slot\n",
     "assert ln_setup.settings.instance.storage.root.as_posix() == storage"
    ]
@@ -95,7 +102,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln_setup.settings.instance.storage_local = \"./my_storage_local2\""
+    "ln_setup.settings.instance.local_storage = \"./my_local_storage2\", \"testuser1-laptop\""
    ]
   },
   {
@@ -105,12 +112,12 @@
    "outputs": [],
    "source": [
     "assert (\n",
-    "    ln_setup.settings.instance.storage_local.root.as_posix()\n",
-    "    == UPath(\"./my_storage_local2\").resolve().as_posix()\n",
+    "    ln_setup.settings.instance.local_storage.root.as_posix()\n",
+    "    == UPath(\"./my_local_storage2\").resolve().as_posix()\n",
     ")\n",
     "assert (\n",
-    "    ln_setup.settings.instance.storage_local.root / \".lamindb/storage_uid.txt\"\n",
-    ").read_text().splitlines()[0] == ln_setup.settings.instance.storage_local.uid"
+    "    ln_setup.settings.instance.local_storage.root / \".lamindb/storage_uid.txt\"\n",
+    ").read_text().splitlines()[0] == ln_setup.settings.instance.local_storage.uid"
    ]
   },
   {
@@ -126,7 +133,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln_setup.settings.instance.storage_local = \"./my_storage_local2\""
+    "ln_setup.settings.instance.local_storage = \"./my_local_storage2\", \"testuser1-laptop\""
    ]
   },
   {
@@ -142,7 +149,18 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln_setup.settings.instance.storage_local = \"./my_storage_local\""
+    "ln_setup.settings.instance.local_storage = \"./my_local_storage\", \"testuser1-laptop\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import lamindb as ln\n",
+    "\n",
+    "ln.Storage.df()"
    ]
   },
   {
@@ -158,7 +176,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "test_file = ln_setup.settings.instance.storage_local.root / \".lamindb/test_file.txt\"\n",
+    "test_file = ln_setup.settings.instance.local_storage.root / \".lamindb/test_file.txt\"\n",
     "test_file.write_text(\"test\")"
    ]
   },
@@ -168,7 +186,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln_setup.settings.instance.storage_local.root.view_tree()"
+    "ln_setup.settings.instance.local_storage.root.view_tree()"
    ]
   },
   {
@@ -227,7 +245,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "py312",
    "language": "python",
    "name": "python3"
   },
@@ -241,7 +259,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.16"
+   "version": "3.12.8"
   }
  },
  "nbformat": 4,

{lamindb_setup-1.7.3 → lamindb_setup-1.8.0}/lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.7.3"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.8.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 

{lamindb_setup-1.7.3 → lamindb_setup-1.8.0}/lamindb_setup/_set_managed_storage.py

@@ -7,17 +7,19 @@ from lamin_utils import logger
 from ._init_instance import register_storage_in_instance
 from .core._hub_core import delete_storage_record
 from .core._settings import settings
-from .core._settings_storage import init_storage
+from .core._settings_storage import StorageSettings, init_storage
 
 if TYPE_CHECKING:
     from lamindb_setup.types import UPathStr
 
 
-def set_managed_storage(root: UPathStr, **fs_kwargs):
+def set_managed_storage(root: UPathStr, host: str | None = None, **fs_kwargs):
     """Add or switch to another managed storage location.
 
     Args:
         root: `UPathStr` - The new storage root, e.g., an S3 bucket.
+        host: `str | None = None` For a shared local storage location, pass a globally unique host identifier, e.g. `"my-institute-cluster-1"`, `"my-server-abcd"`, ...
+            Discuss the naming convention with an admin.
         **fs_kwargs: Additional fsspec arguments for cloud root, e.g., profile.
 
     """
@@ -29,6 +31,16 @@ def set_managed_storage(root: UPathStr, **fs_kwargs):
         raise ValueError(
             "Can't add additional managed storage locations for instances that aren't managed through the hub."
         )
+
+    # we do not just query the instance storage table because
+    # we might need some information from the hub
+    if not StorageSettings(root).type_is_cloud and host is None:
+        host = "unspecified-host"
+        logger.warning(
+            "setting local storage locations with a single path is deprecated, "
+            "use a tuple of (local_root, host) instead"
+        )
+
     # here the storage is registered in the hub
     # hub_record_status="hub-record-created" if a new record is created
     # "hub-record-retrieved" if the storage is in the hub already
@@ -36,13 +48,14 @@ def set_managed_storage(root: UPathStr, **fs_kwargs):
         root=root,
         instance_id=settings.instance._id,
         instance_slug=settings.instance.slug,
-        register_hub=True,
+        register_hub=settings.instance.is_on_hub,
+        prevent_register_hub=not settings.instance.is_on_hub,
+        region=host,
     )
     if ssettings._instance_id is None:
         raise ValueError(
             f"Cannot manage storage without write access: {ssettings.root}"
         )
-
     # here the storage is saved in the instance
     # if any error happens the record in the hub is deleted
     # if it was created earlier and not retrieved

{lamindb_setup-1.7.3 → lamindb_setup-1.8.0}/lamindb_setup/core/_settings_instance.py

@@ -32,6 +32,8 @@ if TYPE_CHECKING:
 
     from ._settings_user import UserSettings
 
+LOCAL_STORAGE_MESSAGE = "No storage location found in current environment: create one via, e.g., ln.Storage(root='/dir/our_shared_dir', host='our-server-123).save()"
+
 
 def sanitize_git_repo_url(repo_url: str) -> str:
     assert repo_url.startswith("https://")
@@ -82,7 +84,7 @@ class InstanceSettings:
         self._git_repo = None if git_repo is None else sanitize_git_repo_url(git_repo)
         # local storage
         self._keep_artifacts_local = keep_artifacts_local
-        self._storage_local: StorageSettings | None = None
+        self._local_storage: StorageSettings | None = None
         self._is_on_hub = is_on_hub
         # private, needed for api requests
         self._api_url = api_url
@@ -151,7 +153,7 @@ class InstanceSettings:
         except ProgrammingError:
             logger.error("not able to load Storage registry: please migrate")
             return None
-        found = False
+        found = []
         for record in all_local_records:
             root_path = Path(record.root)
             if root_path.exists():
@@ -175,18 +177,16 @@ class InstanceSettings:
                     )
                     continue
                 if uid == record.uid:
-                    found = True
-                    break
+                    found.append(record)
         if found:
+            if len(found) > 1:
+                found_display = "\n - ".join([f"{record.root}" for record in found])
+                logger.important(f"found locations:\n - {found_display}")
+            logger.important(f"defaulting to local storage: {record.root}")
             return StorageSettings(record.root)
         elif not mute_warning:
-            logger.warning(
-                "none of the registered local storage locations were found:\n    "
-                + "\n    ".join(r.root for r in all_local_records)
-            )
-            logger.important(
-                "please register a new local storage location via `ln.settings.storage_local = local_root_path` and re-load/connect the instance"
-            )
+            start = LOCAL_STORAGE_MESSAGE[0].lower()
+            logger.warning(f"{start}{LOCAL_STORAGE_MESSAGE[1:]}")
         return None
 
     @property
@@ -209,8 +209,8 @@ class InstanceSettings:
         return self._storage
 
     @property
-    def storage_local(self) -> StorageSettings:
-        """An additional local default storage.
+    def local_storage(self) -> StorageSettings:
+        """An additional local storage location.
 
         Is only available if :attr:`keep_artifacts_local` is enabled.
 
@@ -218,46 +218,73 @@ class InstanceSettings:
         """
         if not self._keep_artifacts_local:
             raise ValueError("`keep_artifacts_local` is not enabled for this instance.")
-        if self._storage_local is None:
-            self._storage_local = self._search_local_root()
-        if self._storage_local is None:
-            # raise an error, there was a warning just before in search_local_root
-            raise ValueError()
-        return self._storage_local
-
-    @storage_local.setter
-    def storage_local(self, local_root: Path | str):
+        if self._local_storage is None:
+            self._local_storage = self._search_local_root()
+        if self._local_storage is None:
+            raise ValueError(LOCAL_STORAGE_MESSAGE)
+        return self._local_storage
+
+    @local_storage.setter
+    def local_storage(self, local_root_host: tuple[Path | str, str]):
         from lamindb_setup._init_instance import register_storage_in_instance
 
+        if not isinstance(local_root_host, tuple):
+            local_root = local_root_host
+            host = "unspecified-host"
+        else:
+            local_root, host = local_root_host
+
         local_root = Path(local_root)
         if not self._keep_artifacts_local:
             raise ValueError("`keep_artifacts_local` is not enabled for this instance.")
-        storage_local = self._search_local_root(
+        local_storage = self._search_local_root(
             local_root=StorageSettings(local_root).root_as_str, mute_warning=True
         )
-        if storage_local is not None:
+        if local_storage is not None:
             # great, we're merely switching storage location
-            self._storage_local = storage_local
-            logger.important(f"defaulting to local storage: {storage_local.root}")
+            self._local_storage = local_storage
             return None
-        storage_local = self._search_local_root(mute_warning=True)
-        if storage_local is not None:
+        local_storage = self._search_local_root(mute_warning=True)
+        if local_storage is not None:
             if os.getenv("LAMIN_TESTING") == "true":
                 response = "y"
             else:
                 response = input(
                     "You already configured a local storage root for this instance in this"
-                    f" environment: {self.storage_local.root}\nDo you want to register another one? (y/n)"
+                    f" environment: {self.local_storage.root}\nDo you want to register another one? (y/n)"
                 )
             if response != "y":
                 return None
+        if host == "unspecified-host":
+            logger.warning(
+                "setting local_storage with a single path is deprecated for creating storage locations"
+            )
+            logger.warning(
+                "use this instead: ln.Storage(root='/dir/our_shared_dir', host='our-server-123').save()"
+            )
         local_root = UPath(local_root)
         assert isinstance(local_root, LocalPathClasses)
-        self._storage_local, _ = init_storage(
-            local_root, instance_id=self._id, instance_slug=self.slug, register_hub=True
+        self._local_storage, _ = init_storage(
+            local_root,
+            instance_id=self._id,
+            instance_slug=self.slug,
+            register_hub=True,
+            region=host,
         )  # type: ignore
-        register_storage_in_instance(self._storage_local)  # type: ignore
-        logger.important(f"defaulting to local storage: {self._storage_local.root}")
+        register_storage_in_instance(self._local_storage)  # type: ignore
+        logger.important(
+            f"defaulting to local storage: {self._local_storage.root} on host {host}"
+        )
+
+    @property
+    @deprecated("local_storage")
+    def storage_local(self) -> StorageSettings:
+        return self.local_storage
+
+    @storage_local.setter
+    @deprecated("local_storage")
+    def storage_local(self, local_root_host: tuple[Path | str, str]):
+        self.local_storage = local_root_host  # type: ignore
 
     @property
     def slug(self) -> str:

{lamindb_setup-1.7.3 → lamindb_setup-1.8.0}/lamindb_setup/core/_settings_storage.py

@@ -87,6 +87,15 @@ def get_storage_region(path: UPathStr) -> str | None:
     return region
 
 
+def get_storage_type(root_as_str: str) -> StorageType:
+    import fsspec
+
+    convert = {"file": "local"}
+    # init_storage checks that the root protocol belongs to VALID_PROTOCOLS
+    protocol = fsspec.utils.get_protocol(root_as_str)
+    return convert.get(protocol, protocol)  # type: ignore
+
+
 def mark_storage_root(
     root: UPathStr, uid: str, instance_id: UUID, instance_slug: str
 ) -> Literal["__marked__"] | str:
@@ -125,6 +134,7 @@ def init_storage(
     init_instance: bool = False,
     created_by: UUID | None = None,
     access_token: str | None = None,
+    region: str | None = None,
 ) -> tuple[
     StorageSettings,
     Literal["hub-record-not-created", "hub-record-retrieved", "hub-record-created"],
@@ -145,7 +155,6 @@ def init_storage(
         # this means we constructed a hosted location of shape s3://bucket-name/uid
         # within LaminHub
         assert root_str.endswith(uid)
-    region = None
     lamin_env = os.getenv("LAMIN_ENV")
     if root_str.startswith("create-s3"):
         if root_str != "create-s3":
@@ -177,6 +186,10 @@ def init_storage(
     register_hub = (
         register_hub or ssettings.type_is_cloud
     )  # default to registering cloud storage
+    if register_hub and not ssettings.type_is_cloud and ssettings.host is None:
+        raise ValueError(
+            "`host` must be set for local storage locations that are registered on the hub"
+        )
     hub_record_status = init_storage_hub(
         ssettings,
         auto_populate_instance=not init_instance,
@@ -226,7 +239,10 @@ def init_storage(
 
 
 class StorageSettings:
-    """Settings for a storage location (local or cloud)."""
+    """Settings for a storage location (local or cloud).
+
+    Do not instantiate this class yourself, use `ln.Storage` instead.
+    """
 
     def __init__(
         self,
@@ -385,6 +401,18 @@ class StorageSettings:
         """`True` if `storage_root` is in cloud, `False` otherwise."""
         return self.type != "local"
 
+    @property
+    def host(self) -> str | None:
+        """Host identifier for local storage locations.
+
+        Is `None` for locations with `type != "local"`.
+
+        A globally unique user-defined host identifier (cluster, server, laptop, etc.).
+        """
+        if self.type != "local":
+            return None
+        return self.region
+
     @property
     def region(self) -> str | None:
         """Storage region."""
@@ -398,12 +426,7 @@ class StorageSettings:
 
         Returns the protocol as a stringe, e.g., "local", "s3", "gs", "http", "https".
         """
-        import fsspec
-
-        convert = {"file": "local"}
-        # init_storage checks that the root protocol belongs to VALID_PROTOCOLS
-        protocol = fsspec.utils.get_protocol(self.root_as_str)
-        return convert.get(protocol, protocol)  # type: ignore
+        return get_storage_type(self.root_as_str)
 
     @property
     def is_on_hub(self) -> bool:

{lamindb_setup-1.7.3 → lamindb_setup-1.8.0}/lamindb_setup/core/upath.py

@@ -928,9 +928,13 @@ def check_storage_is_empty(
     # if the storage_uid.txt was somehow deleted, we restore a dummy version of it
     # because we need it to count files in an empty directory on S3 (otherwise permission error)
     if not (root_upath / STORAGE_UID_FILE_KEY).exists():
-        (root_upath / STORAGE_UID_FILE_KEY).write_text(
-            "was deleted, restored during delete"
-        )
+        try:
+            (root_upath / STORAGE_UID_FILE_KEY).write_text(
+                "was deleted, restored during delete"
+            )
+        except FileNotFoundError:
+            # this can happen if the root is a local non-existing path
+            pass
     if account_for_sqlite_file:
         n_offset_objects += 1  # the SQLite file is in the ".lamindb" directory
     if root_string.startswith(HOSTED_BUCKETS):

{lamindb_setup-1.7.3 → lamindb_setup-1.8.0}/tests/hub-cloud/scripts/script-to-fail-managed-storage.py

@@ -13,7 +13,7 @@ ln_setup.connect("laminlabs/lamin-site-assets")
 test_root = Path("./test_script_ci_storage").resolve().as_posix()
 
 with pytest.raises(ProgrammingError) as error:
-    set_managed_storage(test_root)
+    set_managed_storage(test_root, host="test-host-1234")
 assert error.exconly().endswith(
     "ProgrammingError: permission denied for table lamindb_storage"
 )

{lamindb_setup-1.7.3 → lamindb_setup-1.8.0}/tests/hub-local/conftest.py

@@ -11,15 +11,12 @@ from laminhub_rest.dev import (
 supabase_resources = SupabaseResources()
 
 
-pytest_plugins = [
-    "laminhub_rest.test.account.fixtures",
-    "laminhub_rest.test.common_fixtures",
-]
-
-
 def pytest_configure():
     os.environ["LAMIN_ENV"] = "local"
     os.environ["LAMIN_CLOUD_VERSION"] = "0.1"
+    os.environ["LAMIN_TEST_INSTANCE_ACCESS_V2"] = "true"
+    os.environ["LAMIN_TEST_INSTANCE_SCHEMA_STR"] = ""
+    os.environ["LAMIN_TEST_INSTANCE_PUBLIC"] = "false"
     remove_lamin_local_settings()
     supabase_resources.start_local()
     supabase_resources.reset_local()

{lamindb_setup-1.7.3 → lamindb_setup-1.8.0}/tests/hub-local/scripts/script-connect-fine-grained-access.py

@@ -9,7 +9,7 @@ from lamindb_setup.core.django import db_token_manager
 
 assert os.environ["LAMIN_ENV"] == "local"
 
-ln_setup.connect("instance_access_v2")
+ln_setup.connect("instance_test__access_v2")
 
 isettings = ln_setup.settings.instance
 

{lamindb_setup-1.7.3 → lamindb_setup-1.8.0}/tests/hub-local/test_all.py

@@ -42,10 +42,7 @@ from lamindb_setup.core._settings_storage import init_storage as init_storage_ba
 from lamindb_setup.core._settings_store import instance_settings_file
 from lamindb_setup.core._settings_user import UserSettings
 from laminhub_rest.core.legacy._instance_collaborator import InstanceCollaboratorHandler
-from laminhub_rest.test.instance.utils import (
-    create_hosted_test_instance,
-    delete_hosted_test_instance,
-)
+from laminhub_rest.test.instance import create_instance
 from postgrest.exceptions import APIError
 from supafunc.errors import FunctionsHttpError
 
@@ -183,10 +180,13 @@ def create_myinstance(create_testadmin1_session):  # -> Dict
 
 @pytest.fixture(scope="session")
 def create_instance_fine_grained_access(create_testadmin1_session):
-    instance = create_hosted_test_instance("instance_access_v2", access_v2=True)
+    client, _ = create_testadmin1_session
+
+    instance = create_instance("instance_test", client=client)
 
     yield instance
-    delete_hosted_test_instance(instance)
+
+    delete(instance.name, force=True)
 
 
 def test_connection_string_decomp(create_myinstance, create_testadmin1_session):