lamindb_setup 1.7.3__py3-none-any.whl → 1.8.0__py3-none-any.whl

lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.7.3"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.8.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 

lamindb_setup/_set_managed_storage.py

@@ -7,17 +7,19 @@ from lamin_utils import logger
 from ._init_instance import register_storage_in_instance
 from .core._hub_core import delete_storage_record
 from .core._settings import settings
-from .core._settings_storage import init_storage
+from .core._settings_storage import StorageSettings, init_storage
 
 if TYPE_CHECKING:
     from lamindb_setup.types import UPathStr
 
 
-def set_managed_storage(root: UPathStr, **fs_kwargs):
+def set_managed_storage(root: UPathStr, host: str | None = None, **fs_kwargs):
     """Add or switch to another managed storage location.
 
     Args:
         root: `UPathStr` - The new storage root, e.g., an S3 bucket.
+        host: `str | None = None` For a shared local storage location, pass a globally unique host identifier, e.g. `"my-institute-cluster-1"`, `"my-server-abcd"`, ...
+            Discuss the naming convention with an admin.
         **fs_kwargs: Additional fsspec arguments for cloud root, e.g., profile.
 
     """
@@ -29,6 +31,16 @@ def set_managed_storage(root: UPathStr, **fs_kwargs):
         raise ValueError(
             "Can't add additional managed storage locations for instances that aren't managed through the hub."
         )
+
+    # we do not just query the instance storage table because
+    # we might need some information from the hub
+    if not StorageSettings(root).type_is_cloud and host is None:
+        host = "unspecified-host"
+        logger.warning(
+            "setting local storage locations with a single path is deprecated, "
+            "use a tuple of (local_root, host) instead"
+        )
+
     # here the storage is registered in the hub
     # hub_record_status="hub-record-created" if a new record is created
     # "hub-record-retrieved" if the storage is in the hub already
@@ -36,13 +48,14 @@ def set_managed_storage(root: UPathStr, **fs_kwargs):
         root=root,
         instance_id=settings.instance._id,
         instance_slug=settings.instance.slug,
-        register_hub=True,
+        register_hub=settings.instance.is_on_hub,
+        prevent_register_hub=not settings.instance.is_on_hub,
+        region=host,
     )
     if ssettings._instance_id is None:
         raise ValueError(
             f"Cannot manage storage without write access: {ssettings.root}"
         )
-
     # here the storage is saved in the instance
     # if any error happens the record in the hub is deleted
     # if it was created earlier and not retrieved

lamindb_setup/core/_settings_instance.py

@@ -32,6 +32,8 @@ if TYPE_CHECKING:
 
     from ._settings_user import UserSettings
 
+LOCAL_STORAGE_MESSAGE = "No storage location found in current environment: create one via, e.g., ln.Storage(root='/dir/our_shared_dir', host='our-server-123).save()"
+
 
 def sanitize_git_repo_url(repo_url: str) -> str:
     assert repo_url.startswith("https://")
@@ -82,7 +84,7 @@ class InstanceSettings:
         self._git_repo = None if git_repo is None else sanitize_git_repo_url(git_repo)
         # local storage
         self._keep_artifacts_local = keep_artifacts_local
-        self._storage_local: StorageSettings | None = None
+        self._local_storage: StorageSettings | None = None
         self._is_on_hub = is_on_hub
         # private, needed for api requests
         self._api_url = api_url
@@ -151,7 +153,7 @@ class InstanceSettings:
         except ProgrammingError:
             logger.error("not able to load Storage registry: please migrate")
             return None
-        found = False
+        found = []
         for record in all_local_records:
             root_path = Path(record.root)
             if root_path.exists():
@@ -175,18 +177,16 @@ class InstanceSettings:
                     )
                     continue
                 if uid == record.uid:
-                    found = True
-                    break
+                    found.append(record)
         if found:
+            if len(found) > 1:
+                found_display = "\n - ".join([f"{record.root}" for record in found])
+                logger.important(f"found locations:\n - {found_display}")
+            logger.important(f"defaulting to local storage: {record.root}")
             return StorageSettings(record.root)
         elif not mute_warning:
-            logger.warning(
-                "none of the registered local storage locations were found:\n    "
-                + "\n    ".join(r.root for r in all_local_records)
-            )
-            logger.important(
-                "please register a new local storage location via `ln.settings.storage_local = local_root_path` and re-load/connect the instance"
-            )
+            start = LOCAL_STORAGE_MESSAGE[0].lower()
+            logger.warning(f"{start}{LOCAL_STORAGE_MESSAGE[1:]}")
         return None
 
     @property
@@ -209,8 +209,8 @@ class InstanceSettings:
         return self._storage
 
     @property
-    def storage_local(self) -> StorageSettings:
-        """An additional local default storage.
+    def local_storage(self) -> StorageSettings:
+        """An additional local storage location.
 
         Is only available if :attr:`keep_artifacts_local` is enabled.
 
@@ -218,46 +218,73 @@ class InstanceSettings:
         """
         if not self._keep_artifacts_local:
             raise ValueError("`keep_artifacts_local` is not enabled for this instance.")
-        if self._storage_local is None:
-            self._storage_local = self._search_local_root()
-        if self._storage_local is None:
-            # raise an error, there was a warning just before in search_local_root
-            raise ValueError()
-        return self._storage_local
-
-    @storage_local.setter
-    def storage_local(self, local_root: Path | str):
+        if self._local_storage is None:
+            self._local_storage = self._search_local_root()
+        if self._local_storage is None:
+            raise ValueError(LOCAL_STORAGE_MESSAGE)
+        return self._local_storage
+
+    @local_storage.setter
+    def local_storage(self, local_root_host: tuple[Path | str, str]):
         from lamindb_setup._init_instance import register_storage_in_instance
 
+        if not isinstance(local_root_host, tuple):
+            local_root = local_root_host
+            host = "unspecified-host"
+        else:
+            local_root, host = local_root_host
+
         local_root = Path(local_root)
         if not self._keep_artifacts_local:
             raise ValueError("`keep_artifacts_local` is not enabled for this instance.")
-        storage_local = self._search_local_root(
+        local_storage = self._search_local_root(
             local_root=StorageSettings(local_root).root_as_str, mute_warning=True
         )
-        if storage_local is not None:
+        if local_storage is not None:
             # great, we're merely switching storage location
-            self._storage_local = storage_local
-            logger.important(f"defaulting to local storage: {storage_local.root}")
+            self._local_storage = local_storage
             return None
-        storage_local = self._search_local_root(mute_warning=True)
-        if storage_local is not None:
+        local_storage = self._search_local_root(mute_warning=True)
+        if local_storage is not None:
             if os.getenv("LAMIN_TESTING") == "true":
                 response = "y"
             else:
                 response = input(
                     "You already configured a local storage root for this instance in this"
-                    f" environment: {self.storage_local.root}\nDo you want to register another one? (y/n)"
+                    f" environment: {self.local_storage.root}\nDo you want to register another one? (y/n)"
                 )
             if response != "y":
                 return None
+        if host == "unspecified-host":
+            logger.warning(
+                "setting local_storage with a single path is deprecated for creating storage locations"
+            )
+            logger.warning(
+                "use this instead: ln.Storage(root='/dir/our_shared_dir', host='our-server-123').save()"
+            )
         local_root = UPath(local_root)
         assert isinstance(local_root, LocalPathClasses)
-        self._storage_local, _ = init_storage(
-            local_root, instance_id=self._id, instance_slug=self.slug, register_hub=True
+        self._local_storage, _ = init_storage(
+            local_root,
+            instance_id=self._id,
+            instance_slug=self.slug,
+            register_hub=True,
+            region=host,
         )  # type: ignore
-        register_storage_in_instance(self._storage_local)  # type: ignore
-        logger.important(f"defaulting to local storage: {self._storage_local.root}")
+        register_storage_in_instance(self._local_storage)  # type: ignore
+        logger.important(
+            f"defaulting to local storage: {self._local_storage.root} on host {host}"
+        )
+
+    @property
+    @deprecated("local_storage")
+    def storage_local(self) -> StorageSettings:
+        return self.local_storage
+
+    @storage_local.setter
+    @deprecated("local_storage")
+    def storage_local(self, local_root_host: tuple[Path | str, str]):
+        self.local_storage = local_root_host  # type: ignore
 
     @property
     def slug(self) -> str:

lamindb_setup/core/_settings_storage.py

@@ -87,6 +87,15 @@ def get_storage_region(path: UPathStr) -> str | None:
     return region
 
 
+def get_storage_type(root_as_str: str) -> StorageType:
+    import fsspec
+
+    convert = {"file": "local"}
+    # init_storage checks that the root protocol belongs to VALID_PROTOCOLS
+    protocol = fsspec.utils.get_protocol(root_as_str)
+    return convert.get(protocol, protocol)  # type: ignore
+
+
 def mark_storage_root(
     root: UPathStr, uid: str, instance_id: UUID, instance_slug: str
 ) -> Literal["__marked__"] | str:
@@ -125,6 +134,7 @@ def init_storage(
     init_instance: bool = False,
     created_by: UUID | None = None,
     access_token: str | None = None,
+    region: str | None = None,
 ) -> tuple[
     StorageSettings,
     Literal["hub-record-not-created", "hub-record-retrieved", "hub-record-created"],
@@ -145,7 +155,6 @@ def init_storage(
         # this means we constructed a hosted location of shape s3://bucket-name/uid
         # within LaminHub
         assert root_str.endswith(uid)
-    region = None
     lamin_env = os.getenv("LAMIN_ENV")
     if root_str.startswith("create-s3"):
         if root_str != "create-s3":
@@ -177,6 +186,10 @@ def init_storage(
     register_hub = (
         register_hub or ssettings.type_is_cloud
     )  # default to registering cloud storage
+    if register_hub and not ssettings.type_is_cloud and ssettings.host is None:
+        raise ValueError(
+            "`host` must be set for local storage locations that are registered on the hub"
+        )
     hub_record_status = init_storage_hub(
         ssettings,
         auto_populate_instance=not init_instance,
@@ -226,7 +239,10 @@ def init_storage(
 
 
 class StorageSettings:
-    """Settings for a storage location (local or cloud)."""
+    """Settings for a storage location (local or cloud).
+
+    Do not instantiate this class yourself, use `ln.Storage` instead.
+    """
 
     def __init__(
         self,
@@ -385,6 +401,18 @@ class StorageSettings:
         """`True` if `storage_root` is in cloud, `False` otherwise."""
         return self.type != "local"
 
+    @property
+    def host(self) -> str | None:
+        """Host identifier for local storage locations.
+
+        Is `None` for locations with `type != "local"`.
+
+        A globally unique user-defined host identifier (cluster, server, laptop, etc.).
+        """
+        if self.type != "local":
+            return None
+        return self.region
+
     @property
     def region(self) -> str | None:
         """Storage region."""
@@ -398,12 +426,7 @@ class StorageSettings:
 
         Returns the protocol as a stringe, e.g., "local", "s3", "gs", "http", "https".
         """
-        import fsspec
-
-        convert = {"file": "local"}
-        # init_storage checks that the root protocol belongs to VALID_PROTOCOLS
-        protocol = fsspec.utils.get_protocol(self.root_as_str)
-        return convert.get(protocol, protocol)  # type: ignore
+        return get_storage_type(self.root_as_str)
 
     @property
     def is_on_hub(self) -> bool:

lamindb_setup/core/upath.py

@@ -928,9 +928,13 @@ def check_storage_is_empty(
     # if the storage_uid.txt was somehow deleted, we restore a dummy version of it
     # because we need it to count files in an empty directory on S3 (otherwise permission error)
     if not (root_upath / STORAGE_UID_FILE_KEY).exists():
-        (root_upath / STORAGE_UID_FILE_KEY).write_text(
-            "was deleted, restored during delete"
-        )
+        try:
+            (root_upath / STORAGE_UID_FILE_KEY).write_text(
+                "was deleted, restored during delete"
+            )
+        except FileNotFoundError:
+            # this can happen if the root is a local non-existing path
+            pass
     if account_for_sqlite_file:
         n_offset_objects += 1  # the SQLite file is in the ".lamindb" directory
     if root_string.startswith(HOSTED_BUCKETS):

{lamindb_setup-1.7.3.dist-info → lamindb_setup-1.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.7.3
+Version: 1.8.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10

{lamindb_setup-1.7.3.dist-info → lamindb_setup-1.8.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-lamindb_setup/__init__.py,sha256=YCc_Qq5IYq91E9yQdWzQwsOoWSjH5qBpBkZulYdQaJ0,2782
+lamindb_setup/__init__.py,sha256=t86vewd7XZhz-qY1WJCxdsaTAqsjwQXiMF7Y6L2odJQ,2782
 lamindb_setup/_cache.py,sha256=5o749NuW6zi6uP4rmBtwxg7ifWpAHXVngzC0tEgXLgo,2776
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=bXuqx2HEc178RM7gbKZQ65PEVJFu6uSOKiHAs_xz6GI,5575
@@ -14,7 +14,7 @@ lamindb_setup/_migrate.py,sha256=ya-15sc91i4JmEWI4j00T2892x8hdy2fSW-qz4IdxLs,973
 lamindb_setup/_register_instance.py,sha256=zmk7UrOF6qED_zTEaTM8GbZurMSP1SwddkRy7rpYmUY,1215
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
 lamindb_setup/_schema_metadata.py,sha256=yMSuLZc-7iWUV7tETNuKmcDeMW0wtUr5ypKxsKC-m7k,14898
-lamindb_setup/_set_managed_storage.py,sha256=9-ECe4dJNgiO5LFrDFhSkpdIK3Zzmh_Hk_WartfTaRw,2005
+lamindb_setup/_set_managed_storage.py,sha256=2muJxbjhZTSgMcyLuLS0PyXlCl5tSYI3BVaSYjrFYkM,2761
 lamindb_setup/_setup_user.py,sha256=8BSGsW5jfmB4FlkhMt5osYXBbVCdOQeAVATb-oAYxa0,4495
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
 lamindb_setup/errors.py,sha256=H1UM-bii0U2vPyjprOBgZK4ijZJgzgCViyGWPd8v5yU,1493
@@ -31,10 +31,10 @@ lamindb_setup/core/_hub_crud.py,sha256=Jz0d8wFKM1Pv9B9byyUJPlCIMkIzk56Jd-c3Awpm9
 lamindb_setup/core/_hub_utils.py,sha256=6dyDGyzYFgVfR_lE3VN3CP1jGp98gxPtr-T91PAP05U,2687
 lamindb_setup/core/_private_django_api.py,sha256=By63l3vIEtK1pq246FhHq3tslxsaTJGKm5VakYluWp4,2656
 lamindb_setup/core/_settings.py,sha256=0la2eYAHcdMYophheGsgQaD2nJGQrPuX4jMV5GKDiC0,12946
-lamindb_setup/core/_settings_instance.py,sha256=nXSOc6qPwamcoD-4W8sIOFD_L-W8VO8e0tT-U0INuYo,19518
+lamindb_setup/core/_settings_instance.py,sha256=iG4Me-9e9Z4yj3EG3YB3OtkiZ-j-u8R1WPXIeQhMNmw,20484
 lamindb_setup/core/_settings_load.py,sha256=JWd0_hBy04xjKo-tH4y8C9RkaywjrmoT0PsKzVme0n4,5176
 lamindb_setup/core/_settings_save.py,sha256=XZx-vow7BT6y3JpRBB2UOJp2vwc7jOGea4wSgOPqjPU,3262
-lamindb_setup/core/_settings_storage.py,sha256=AY3OpJd2uOR8nTrJUzESq_sS1P1Uo_y9oqi5_NdcSPQ,16353
+lamindb_setup/core/_settings_storage.py,sha256=wVwDDD51UsJz5gS6juERbWHCdyucP0R2DLaUGhnLUpQ,17079
 lamindb_setup/core/_settings_store.py,sha256=QmeWIGdIyq7UmjfHiEB_0xRD8hY-8-ZR2WntIKfwTKI,2714
 lamindb_setup/core/_settings_user.py,sha256=K2a6nQ0fhEiSb9mCY_p6ItNrHZ3J_j7EfO7CjZap9aA,1462
 lamindb_setup/core/_setup_bionty_sources.py,sha256=ox3X-SHiHa2lNPSWjwZhINypbLacX6kGwH6hVVrSFZc,1505
@@ -43,8 +43,8 @@ lamindb_setup/core/django.py,sha256=dOt1OkUnZeYOo-LTjatQWQFh_MnjRf9IwwvRZhCwdZQ,
 lamindb_setup/core/exceptions.py,sha256=qjMzqy_uzPA7mCOdnoWnS_fdA6OWbdZGftz-YYplrY0,84
 lamindb_setup/core/hashing.py,sha256=Y8Uc5uSGTfU6L2R_gb5w8DdHhGRog7RnkK-e9FEMjPY,3680
 lamindb_setup/core/types.py,sha256=T7NwspfRHgIIpYsXDcApks8jkOlGeGRW-YbVLB7jNIo,67
-lamindb_setup/core/upath.py,sha256=tvu178E08VSC7YUNrcDYxri4ODuVtX3BGNhJO8UZf00,34302
-lamindb_setup-1.7.3.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-1.7.3.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-lamindb_setup-1.7.3.dist-info/METADATA,sha256=LkK3UmxVpUDNES_P4zFdkeMx02PzrEWjAtqPrpX0-PQ,1797
-lamindb_setup-1.7.3.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=UDPcZaNp3WtZSYrX8hGQ1lq214dBn079bz0FPWFjIbA,34449
+lamindb_setup-1.8.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-1.8.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
+lamindb_setup-1.8.0.dist-info/METADATA,sha256=oxlCgEfbGqZ8zUi2aKMyQdCZoVTkngUl1SKM-Dc1mdU,1797
+lamindb_setup-1.8.0.dist-info/RECORD,,