lamindb_setup 0.76.3__py2.py3-none-any.whl → 0.76.5__py2.py3-none-any.whl

lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.76.3"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.76.5"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

lamindb_setup/_init_instance.py

@@ -250,7 +250,7 @@ def init(
         # see a way of making this more elegant; should become possible if we
         # remove the instance.storage_id FK on the hub
         prevent_register_hub = is_local_db_url(db) if db is not None else False
-        ssettings = init_storage(
+        ssettings, _ = init_storage(
             storage,
             instance_id=instance_id,
             init_instance=True,

lamindb_setup/_set_managed_storage.py

@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING
 from lamin_utils import logger
 
 from ._init_instance import register_storage_in_instance
+from .core._hub_core import delete_storage_record
 from .core._settings import settings
 from .core._settings_storage import init_storage
 
@@ -28,14 +29,27 @@ def set_managed_storage(root: UPathStr, **fs_kwargs):
         raise ValueError(
             "Can't add additional managed storage locations for instances that aren't managed through the hub."
         )
-    ssettings = init_storage(
+    # here the storage is registered in the hub
+    # hub_record_status="hub_record_created" if a new record is created
+    # "hub_record_retrieved" if the storage is in the hub already
+    ssettings, hub_record_status = init_storage(
         root=root, instance_id=settings.instance._id, register_hub=True
     )
     if ssettings._instance_id is None:
         raise ValueError(
             f"Cannot manage storage without write access: {ssettings.root}"
         )
+
+    # here the storage is saved in the instance
+    # if any error happens the record in the hub is deleted
+    # if it was created earlier and not retrieved
+    try:
+        register_storage_in_instance(ssettings)
+    except Exception as e:
+        if hub_record_status == "hub_record_created" and ssettings._uuid is not None:
+            delete_storage_record(ssettings._uuid)  # type: ignore
+        raise e
+
     settings.instance._storage = ssettings
     settings.instance._persist()  # this also updates the settings object
-    register_storage_in_instance(ssettings)
     settings.storage._set_fs_kwargs(**fs_kwargs)

lamindb_setup/core/_hub_core.py

@@ -4,7 +4,7 @@ import json
 import os
 import uuid
 from importlib import metadata
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
 from uuid import UUID
 
 from lamin_utils import logger
@@ -19,10 +19,10 @@ from ._hub_crud import (
     _delete_instance_record,
     select_account_by_handle,
     select_db_user_by_instance,
+    select_default_storage_by_instance_id,
     select_instance_by_id_with_storage,
     select_instance_by_name,
     select_instance_by_owner_name,
-    select_storage,
 )
 from ._hub_crud import update_instance as _update_instance_record
 from ._hub_utils import (
@@ -119,7 +119,7 @@ def _select_storage(
 def init_storage(
     ssettings: StorageSettings,
     auto_populate_instance: bool = True,
-) -> None:
+) -> Literal["hub_record_retrieved", "hub_record_created"]:
     if settings.user.handle != "anonymous":
         return call_with_fallback_auth(
             _init_storage,
@@ -131,21 +131,21 @@ def init_storage(
             _select_storage, ssettings=ssettings, update_uid=True
         )
         if storage_exists:
-            return None
+            return "hub_record_retrieved"
         else:
             raise ValueError("Log in to create a storage location on the hub.")
 
 
 def _init_storage(
     ssettings: StorageSettings, auto_populate_instance: bool, client: Client
-) -> None:
+) -> Literal["hub_record_retrieved", "hub_record_created"]:
     from lamindb_setup import settings
 
     # storage roots are always stored without the trailing slash in the SQL
     # database
     root = ssettings.root_as_str
     if _select_storage(ssettings, update_uid=True, client=client):
-        return None
+        return "hub_record_retrieved"
     if ssettings.type_is_cloud:
         id = uuid.uuid5(uuid.NAMESPACE_URL, root)
     else:
@@ -181,7 +181,7 @@ def _init_storage(
     # on root & description
     client.table("storage").upsert(fields).execute()
     ssettings._uuid_ = id
-    return None
+    return "hub_record_created"
 
 
 def delete_instance(identifier: UUID | str, require_empty: bool = True) -> str | None:
@@ -338,7 +338,7 @@ def _connect_instance(
         if instance is None:
             return "instance-not-found"
         # get default storage
-        storage = select_storage(instance["storage_id"], client)
+        storage = select_default_storage_by_instance_id(instance["id"], client)
         if storage is None:
             return "storage-does-not-exist-on-hub"
     else:
@@ -364,7 +364,7 @@ def _connect_instance(
             database=instance["db_database"],
         )
         instance["db"] = db_dsn
-    return instance, storage
+    return instance, storage  # type: ignore
 
 
 def access_aws(storage_root: str, access_token: str | None = None) -> dict[str, dict]:

lamindb_setup/core/_hub_crud.py

@@ -17,10 +17,11 @@ def select_instance_by_owner_name(
             client.table("instance")
             .select(
                 "*, account!inner!instance_account_id_28936e8f_fk_account_id(*),"
-                " storage!instance_storage_id_87963cc8_fk_storage_id(*)"
+                " storage!inner!storage_instance_id_359fca71_fk_instance_id(*)"
             )
-            .eq("account.handle", owner)
             .eq("name", name)
+            .eq("account.handle", owner)
+            .eq("storage.is_default", True)
             .execute()
             .data
         )
@@ -28,7 +29,11 @@ def select_instance_by_owner_name(
         return None
     if len(data) == 0:
         return None
-    return data[0]
+    result = data[0]
+    # this is now a list
+    # assume only one default storage
+    result["storage"] = result["storage"][0]
+    return result
 
 
 # --------------- ACCOUNT ----------------------
@@ -131,8 +136,20 @@ def select_collaborator(
 # --------------- STORAGE ----------------------
 
 
-def select_storage(id: str, client: Client):
-    data = client.table("storage").select("*").eq("id", id).execute().data
+def select_default_storage_by_instance_id(
+    instance_id: str, client: Client
+) -> dict | None:
+    try:
+        data = (
+            client.table("storage")
+            .select("*")
+            .eq("instance_id", instance_id)
+            .eq("is_default", True)
+            .execute()
+            .data
+        )
+    except Exception:
+        return None
     if len(data) == 0:
         return None
     return data[0]

lamindb_setup/core/_settings_instance.py

@@ -230,7 +230,7 @@ class InstanceSettings:
                 return None
         local_root = UPath(local_root)
         assert isinstance(local_root, LocalPathClasses)
-        self._storage_local = init_storage(local_root, self._id, register_hub=True)  # type: ignore
+        self._storage_local, _ = init_storage(local_root, self._id, register_hub=True)  # type: ignore
         register_storage_in_instance(self._storage_local)  # type: ignore
         logger.important(f"defaulting to local storage: {self._storage_local.root}")
 

lamindb_setup/core/_settings_load.py

@@ -36,14 +36,9 @@ def load_instance_settings(instance_settings_file: Path | None = None):
             content = f.read()
         raise SettingsEnvFileOutdated(
             f"\n\n{error}\n\nYour instance settings file with\n\n{content}\nis invalid"
-            f" (likely outdated), please delete {instance_settings_file} &"
-            " re-initialize (local) or re-connect to the instance (remote)"
+            f" (likely outdated), see validation error. Please delete {instance_settings_file} &"
+            " reload (remote) or re-initialize (local) the instance with the same name & storage location."
         ) from error
-    if settings_store.id == "null":
-        raise ValueError(
-            "Your instance._id is undefined, please either load your instance from the"
-            f" hub or update {instance_settings_file} with a new id: {uuid4().hex}"
-        )
     isettings = setup_instance_from_store(settings_store)
     return isettings
 

lamindb_setup/core/_settings_storage.py

@@ -83,7 +83,10 @@ def init_storage(
     register_hub: bool | None = None,
     prevent_register_hub: bool = False,
     init_instance: bool = False,
-) -> StorageSettings:
+) -> tuple[
+    StorageSettings,
+    Literal["hub_record_not_created", "hub_record_retrieved", "hub_record_created"],
+]:
     if root is None:
         raise ValueError("`storage` argument can't be `None`")
     root_str = str(root)  # ensure we have a string
@@ -117,13 +120,19 @@ def init_storage(
         region=region,
         instance_id=instance_id,
     )
+    # this stores the result of init_storage_hub
+    hub_record_status: Literal[
+        "hub_record_not_created", "hub_record_retrieved", "hub_record_created"
+    ] = "hub_record_not_created"
     # the below might update the uid with one that's already taken on the hub
     if not prevent_register_hub:
         if ssettings.type_is_cloud or register_hub:
             from ._hub_core import delete_storage_record
             from ._hub_core import init_storage as init_storage_hub
 
-            init_storage_hub(ssettings, auto_populate_instance=not init_instance)
+            hub_record_status = init_storage_hub(
+                ssettings, auto_populate_instance=not init_instance
+            )
     # below comes last only if everything else was successful
     try:
         # (federated) credentials for AWS access are provisioned under-the-hood
@@ -133,10 +142,14 @@ def init_storage(
         logger.important(
             f"due to lack of write access, LaminDB won't manage storage location: {ssettings.root}"
         )
-        if ssettings._uuid is not None:
+        # we have to check hub_record_status here because
+        # _select_storage inside init_storage_hub also populates ssettings._uuid
+        # and we don't want to delete an existing storage record here
+        # only newly created
+        if hub_record_status == "hub_record_created" and ssettings._uuid is not None:
             delete_storage_record(ssettings._uuid)  # type: ignore
         ssettings._instance_id = None
-    return ssettings
+    return ssettings, hub_record_status
 
 
 def _process_cache_path(cache_path: str | Path | UPath | None):

lamindb_setup/core/upath.py

@@ -18,7 +18,7 @@ from upath.implementations.cloud import CloudPath, S3Path  # keep CloudPath!
 from upath.implementations.local import LocalPath, PosixUPath, WindowsUPath
 
 from ._aws_credentials import HOSTED_BUCKETS, get_aws_credentials_manager
-from .hashing import b16_to_b64, hash_md5s_from_dir
+from .hashing import HASH_LENGTH, b16_to_b64, hash_md5s_from_dir
 
 if TYPE_CHECKING:
     from .types import UPathStr
@@ -707,9 +707,9 @@ def get_stat_file_cloud(stat: dict) -> tuple[int, str, str]:
     else:
         stripped_etag, suffix = etag.split("-")
         suffix = suffix.strip('"')
-        hash = f"{b16_to_b64(stripped_etag)}-{suffix}"
-        hash_type = "md5-n"  # this is the S3 chunk-hashing strategy
-    return size, hash, hash_type
+        hash = b16_to_b64(stripped_etag)
+        hash_type = f"md5-{suffix}"  # this is the S3 chunk-hashing strategy
+    return size, hash[:HASH_LENGTH], hash_type
 
 
 def get_stat_dir_cloud(path: UPath) -> tuple[int, str, str, int]:

{lamindb_setup-0.76.3.dist-info → lamindb_setup-0.76.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.76.3
+Version: 0.76.5
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown

{lamindb_setup-0.76.3.dist-info → lamindb_setup-0.76.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-lamindb_setup/__init__.py,sha256=hXWSzOTJWwpoCg2nw2RH87bEJu7oXk_jyxAbJdcrsYU,1542
+lamindb_setup/__init__.py,sha256=NzRrDVvtOt5b48-AV5mbNzjP35fl6FOZy8R6GWGSOy4,1542
 lamindb_setup/_cache.py,sha256=wA7mbysANwe8hPNbjDo9bOmXJ0xIyaS5iyxIpxSWji4,846
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=cNEL9Q4yPpmEkGKHH8JgullWl1VUZwALJ4RHn9wZypY,2613
@@ -8,12 +8,12 @@ lamindb_setup/_delete.py,sha256=Y8KSFYgY0UHAvjd7cCL6hZ_XiLeJwx50BguVATcj_Xo,5524
 lamindb_setup/_django.py,sha256=EoyWvFzH0i9wxjy4JZhcoXCTckztP_Mrl6FbYQnMmLE,1534
 lamindb_setup/_exportdb.py,sha256=43g77-tH-vAlTn8ig1mMD9-KXLKvxUeDLaq0gVu3l-c,2114
 lamindb_setup/_importdb.py,sha256=yYYShzUajTsR-cTW4CZ-UNDWZY2uE5PAgNbp-wn8Ogc,1874
-lamindb_setup/_init_instance.py,sha256=pR_pK38okij7CfnQDecJLKR5lhhTeE4oZ14Lb9bMMQg,12388
+lamindb_setup/_init_instance.py,sha256=VxHgD2i0hrFm2f_WCX76YmS_Lsx2iufrMtfab82r8X0,12391
 lamindb_setup/_migrate.py,sha256=P4n3x0SYzO9szjF2-JMa7z4mQadtWjHv5ow4HbCDZLI,8864
 lamindb_setup/_register_instance.py,sha256=alQuYp2f8Ct8xvRC1gt8p_HZ0tqCd3gZD3kiPBLPpsI,1269
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
 lamindb_setup/_schema_metadata.py,sha256=nF29OuMNswJUCyvSyOHEUQLQ7PcfB7uwvczloBYSWuU,13187
-lamindb_setup/_set_managed_storage.py,sha256=mNZrANn-9rwZ0oGWxxg0wS0T0VOQCWyo4nSSyNAE15Q,1419
+lamindb_setup/_set_managed_storage.py,sha256=aFVxk8sl6Jzh-LU_1sxICm_rcDPMr5Q9qmwPtUtrUbA,2043
 lamindb_setup/_setup_user.py,sha256=6Oc7Rke-yRQSZbuntdUAz8QbJ6UuPzYHI9FnYlf_q-A,3670
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
 lamindb_setup/core/__init__.py,sha256=BxIVMX5HQq8oZ1OuY_saUEJz5Tdd7gaCPngxVu5iou4,417
@@ -22,15 +22,15 @@ lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-Oq
 lamindb_setup/core/_deprecated.py,sha256=3qxUI1dnDlSeR0BYrv7ucjqRBEojbqotPgpShXs4KF8,2520
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
 lamindb_setup/core/_hub_client.py,sha256=Gpt3TmxWQMVenKugCu1agUb-xGN9YFsQOKmrHuFiiDs,5605
-lamindb_setup/core/_hub_core.py,sha256=XqXZatJOIDXukfdwL3Buq3WXZK-VVGg5OFRm2PrzO1o,16836
-lamindb_setup/core/_hub_crud.py,sha256=b1XF7AJpM9Q-ttm9nPG-r3OTRWHQaGzAGIyvmb83NTo,4859
+lamindb_setup/core/_hub_core.py,sha256=YQ8-RiNbhTzXC_bGob3WfL47Svm0g8U8e1kxxn1nlzY,17049
+lamindb_setup/core/_hub_crud.py,sha256=eZErpq9t1Cp2ULBSi457ekrcqfesw4Y6IJgaqyrINMY,5276
 lamindb_setup/core/_hub_utils.py,sha256=w5IRtrxZcvxmGSJslzuZF89ewkzXV4cCUmZUVrqmAfo,3026
 lamindb_setup/core/_private_django_api.py,sha256=KIn43HOhiRjkbTbddyJqv-WNTTa1bAizbM1tWXoXPBg,2869
 lamindb_setup/core/_settings.py,sha256=46axQ5HPvI0X9YuotgdpuSOfSo7qYU1DudIx3vxpFk0,4471
-lamindb_setup/core/_settings_instance.py,sha256=rPTL_Iwv-ZVMRzu_EV_E7APFkEXLoD0TgUrz5293-vg,17586
-lamindb_setup/core/_settings_load.py,sha256=63PTIVLk3oH1XUmznswygcOk_8VADrJLMB6O9AOppYg,3907
+lamindb_setup/core/_settings_instance.py,sha256=azgqSWEnTfZM9ghgdF6atvIxoUP6fx49Y9tZTqMQTuc,17589
+lamindb_setup/core/_settings_load.py,sha256=bRkGgaEnLZmnOlU5ptDoSM-YXzC2l0Cb7RBoBY6VH9k,3717
 lamindb_setup/core/_settings_save.py,sha256=n_tYfb9EBSxwm4LHyPRHJptE5uB8lmHhcRkz1JkAmhg,2781
-lamindb_setup/core/_settings_storage.py,sha256=bvEOm2aH4M6xBTfTARcWkx6b-jtKYl_AIUbH5yQC9GI,13435
+lamindb_setup/core/_settings_storage.py,sha256=aP5cdEy9xmvNFAtU8AmpBifYWxeSFwp4Vt1UwpaKJ7o,14080
 lamindb_setup/core/_settings_store.py,sha256=VEE8Ff2A7y4j8ksOaa7g48jNaOqe1PBnBjb1PKKGUKU,2115
 lamindb_setup/core/_settings_user.py,sha256=P2lC4WDRAFfT-Xq3MlXJ-wMKIHCoGNhMTQfRGIAyUNQ,1344
 lamindb_setup/core/_setup_bionty_sources.py,sha256=ANBIiWHoYxLBQqsiKs-rCq3k-p6D34T4NM28gkY6YLA,3389
@@ -39,8 +39,8 @@ lamindb_setup/core/django.py,sha256=QUQm3zt5QIiD8uv6o9vbSm_bshqiSWzKSkgD3z2eJCg,
 lamindb_setup/core/exceptions.py,sha256=eoI7AXgATgDVzgArtN7CUvpaMUC067vsBg5LHCsWzDM,305
 lamindb_setup/core/hashing.py,sha256=Y2cvEaqtm3KwpHqj5ZG2f_sLaXhsQT4oDrmJdHbOQeo,3116
 lamindb_setup/core/types.py,sha256=bcYnZ0uM_2NXKJCl94Mmc-uYrQlRUUVKG3sK2N-F-N4,532
-lamindb_setup/core/upath.py,sha256=Bhxr1B1sFuIp13YTBOEYJSmSOwmU6owk8giBGLCKdg4,27194
-lamindb_setup-0.76.3.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-0.76.3.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
-lamindb_setup-0.76.3.dist-info/METADATA,sha256=29omTu3ShcauGqhxT0g_ZAAuOBZveomsA-hFBH3OGzs,1638
-lamindb_setup-0.76.3.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=EPLLm62q-Y3hZzd-286cynFMttXKDNXNOKL3_QGkeug,27215
+lamindb_setup-0.76.5.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-0.76.5.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
+lamindb_setup-0.76.5.dist-info/METADATA,sha256=RLVkEp_g_smCBEWDglvm2IcBKUbCV3SHFfNOZjOngNk,1638
+lamindb_setup-0.76.5.dist-info/RECORD,,