lamindb_setup 0.76.2__tar.gz → 0.76.4__tar.gz

{lamindb_setup-0.76.2 → lamindb_setup-0.76.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.76.2
+Version: 0.76.4
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown

{lamindb_setup-0.76.2 → lamindb_setup-0.76.4}/lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.76.2"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.76.4"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

{lamindb_setup-0.76.2 → lamindb_setup-0.76.4}/lamindb_setup/_schema_metadata.py

@@ -356,7 +356,9 @@ class _SchemaHandler:
         return self.to_dict(include_django_objects=False)
 
     def _get_modules_metadata(self):
-        return {
+        from lnschema_core.models import Record, Registry
+
+        all_models = {
             module_name: {
                 model._meta.model_name: _ModelHandler(
                     model, module_name, self.included_modules
@@ -364,13 +366,15 @@ class _SchemaHandler:
                 for model in self._get_schema_module(
                     module_name
                 ).models.__dict__.values()
-                if model.__class__.__name__ == "RecordMeta"
-                and model.__name__ not in ["Record", "ORM"]
+                if model.__class__ is Registry
+                and model is not Record
                 and not model._meta.abstract
                 and model.__get_schema_name__() == module_name
             }
             for module_name in self.included_modules
         }
+        assert all_models
+        return all_models
 
     def _get_module_set_info(self):
         # TODO: rely on schemamodule table for this

{lamindb_setup-0.76.2 → lamindb_setup-0.76.4}/lamindb_setup/core/_hub_core.py

@@ -19,10 +19,10 @@ from ._hub_crud import (
     _delete_instance_record,
     select_account_by_handle,
     select_db_user_by_instance,
+    select_default_storage_by_instance_id,
     select_instance_by_id_with_storage,
     select_instance_by_name,
     select_instance_by_owner_name,
-    select_storage,
 )
 from ._hub_crud import update_instance as _update_instance_record
 from ._hub_utils import (
@@ -338,7 +338,7 @@ def _connect_instance(
         if instance is None:
             return "instance-not-found"
         # get default storage
-        storage = select_storage(instance["storage_id"], client)
+        storage = select_default_storage_by_instance_id(instance["id"], client)
         if storage is None:
             return "storage-does-not-exist-on-hub"
     else:
@@ -364,7 +364,7 @@ def _connect_instance(
             database=instance["db_database"],
         )
         instance["db"] = db_dsn
-    return instance, storage
+    return instance, storage  # type: ignore
 
 
 def access_aws(storage_root: str, access_token: str | None = None) -> dict[str, dict]:

{lamindb_setup-0.76.2 → lamindb_setup-0.76.4}/lamindb_setup/core/_hub_crud.py

@@ -17,10 +17,11 @@ def select_instance_by_owner_name(
             client.table("instance")
             .select(
                 "*, account!inner!instance_account_id_28936e8f_fk_account_id(*),"
-                " storage!instance_storage_id_87963cc8_fk_storage_id(*)"
+                " storage!inner!storage_instance_id_359fca71_fk_instance_id(*)"
             )
-            .eq("account.handle", owner)
             .eq("name", name)
+            .eq("account.handle", owner)
+            .eq("storage.is_default", True)
             .execute()
             .data
         )
@@ -28,7 +29,11 @@ def select_instance_by_owner_name(
         return None
     if len(data) == 0:
         return None
-    return data[0]
+    result = data[0]
+    # this is now a list
+    # assume only one default storage
+    result["storage"] = result["storage"][0]
+    return result
 
 
 # --------------- ACCOUNT ----------------------
@@ -131,8 +136,20 @@ def select_collaborator(
 # --------------- STORAGE ----------------------
 
 
-def select_storage(id: str, client: Client):
-    data = client.table("storage").select("*").eq("id", id).execute().data
+def select_default_storage_by_instance_id(
+    instance_id: str, client: Client
+) -> dict | None:
+    try:
+        data = (
+            client.table("storage")
+            .select("*")
+            .eq("instance_id", instance_id)
+            .eq("is_default", True)
+            .execute()
+            .data
+        )
+    except Exception:
+        return None
     if len(data) == 0:
         return None
     return data[0]

{lamindb_setup-0.76.2 → lamindb_setup-0.76.4}/lamindb_setup/core/_settings_load.py

@@ -36,14 +36,9 @@ def load_instance_settings(instance_settings_file: Path | None = None):
             content = f.read()
         raise SettingsEnvFileOutdated(
             f"\n\n{error}\n\nYour instance settings file with\n\n{content}\nis invalid"
-            f" (likely outdated), please delete {instance_settings_file} &"
-            " re-initialize (local) or re-connect to the instance (remote)"
+            f" (likely outdated), see validation error. Please delete {instance_settings_file} &"
+            " reload (remote) or re-initialize (local) the instance with the same name & storage location."
         ) from error
-    if settings_store.id == "null":
-        raise ValueError(
-            "Your instance._id is undefined, please either load your instance from the"
-            f" hub or update {instance_settings_file} with a new id: {uuid4().hex}"
-        )
     isettings = setup_instance_from_store(settings_store)
     return isettings
 

{lamindb_setup-0.76.2 → lamindb_setup-0.76.4}/lamindb_setup/core/upath.py

@@ -18,7 +18,7 @@ from upath.implementations.cloud import CloudPath, S3Path  # keep CloudPath!
 from upath.implementations.local import LocalPath, PosixUPath, WindowsUPath
 
 from ._aws_credentials import HOSTED_BUCKETS, get_aws_credentials_manager
-from .hashing import b16_to_b64, hash_md5s_from_dir
+from .hashing import HASH_LENGTH, b16_to_b64, hash_md5s_from_dir
 
 if TYPE_CHECKING:
     from .types import UPathStr
@@ -28,7 +28,7 @@ LocalPathClasses = (PosixUPath, WindowsUPath, LocalPath)
 # also see https://gist.github.com/securifera/e7eed730cbe1ce43d0c29d7cd2d582f4
 #    ".gz" is not listed here as it typically occurs with another suffix
 # the complete list is at lamindb.core.storage._suffixes
-VALID_SUFFIXES = {
+VALID_SIMPLE_SUFFIXES = {
     #
     # without readers
     #
@@ -56,11 +56,8 @@ VALID_SUFFIXES = {
     ".zarr",
     ".json",
 }
-VALID_COMPOSITE_SUFFIXES = {
-    ".anndata.zarr",
-    ".spatialdata.zarr",
-    ".ome.zarr",
-}
+# below gets updated within lamindb because it's frequently changing
+VALID_COMPOSITE_SUFFIXES = {".anndata.zarr"}
 
 TRAILING_SEP = (os.sep, os.altsep) if os.altsep is not None else os.sep
 
@@ -76,7 +73,7 @@ def extract_suffix_from_path(path: Path, arg_name: str | None = None) -> str:
         return process_digits(path.suffix)
 
     total_suffix = "".join(path.suffixes)
-    if total_suffix in VALID_SUFFIXES:
+    if total_suffix in VALID_SIMPLE_SUFFIXES:
         return total_suffix
     elif total_suffix.endswith(tuple(VALID_COMPOSITE_SUFFIXES)):
         # below seems slow but OK for now
@@ -94,7 +91,7 @@ def extract_suffix_from_path(path: Path, arg_name: str | None = None) -> str:
         # in COMPRESSION_SUFFIXES to detect something like .random.gz and then
         # add ".random.gz" but concluded it's too dangerous it's safer to just
         # use ".gz" in such a case
-        if path.suffixes[-2] in VALID_SUFFIXES:
+        if path.suffixes[-2] in VALID_SIMPLE_SUFFIXES:
             suffix = "".join(path.suffixes[-2:])
             msg += f"inferring: '{suffix}'"
             # do not print a warning for things like .tar.gz, .fastq.gz
@@ -105,7 +102,7 @@ def extract_suffix_from_path(path: Path, arg_name: str | None = None) -> str:
             msg += (
                 f"using only last suffix: '{suffix}' - if you want your composite"
                 " suffix to be recognized add it to"
-                " lamindb.core.storage.VALID_SUFFIXES.add()"
+                " lamindb.core.storage.VALID_SIMPLE_SUFFIXES.add()"
             )
         if print_hint:
             logger.hint(msg)
@@ -710,9 +707,9 @@ def get_stat_file_cloud(stat: dict) -> tuple[int, str, str]:
     else:
         stripped_etag, suffix = etag.split("-")
         suffix = suffix.strip('"')
-        hash = f"{b16_to_b64(stripped_etag)}-{suffix}"
-        hash_type = "md5-n"  # this is the S3 chunk-hashing strategy
-    return size, hash, hash_type
+        hash = b16_to_b64(stripped_etag)
+        hash_type = f"md5-{suffix}"  # this is the S3 chunk-hashing strategy
+    return size, hash[:HASH_LENGTH], hash_type
 
 
 def get_stat_dir_cloud(path: UPath) -> tuple[int, str, str, int]:

{lamindb_setup-0.76.2 → lamindb_setup-0.76.4}/tests/hub-local/conftest.py

@@ -2,9 +2,11 @@ from __future__ import annotations
 
 import os
 
-from laminhub_rest.dev._seed import seed_local_test
-from laminhub_rest.dev._supabase import SupabaseResources
-from laminhub_rest.dev._utils import remove_lamin_local_settings
+from laminhub_rest.dev import (
+    SupabaseResources,
+    remove_lamin_local_settings,
+    seed_local_test,
+)
 
 supabase_resources = SupabaseResources()
 

{lamindb_setup-0.76.2 → lamindb_setup-0.76.4}/tests/storage/test_storage_basis.py

@@ -19,7 +19,7 @@ def test_extract_suffix_from_path():
         ("logs/date.log.123", ""),  # digits are no valid suffixes
         ("salmon.merged.gene_counts.tsv", ".tsv"),
         ("salmon.merged.gene_counts.tsv.gz", ".tsv.gz"),
-        ("filename.v1.1.0.spatialdata.zarr", ".spatialdata.zarr"),
+        ("filename.v1.1.0.anndata.zarr", ".anndata.zarr"),
     ]
     for path, suffix in collection:
         filepath = Path(path)

{lamindb_setup-0.76.2 → lamindb_setup-0.76.4}/tests/storage/test_storage_stats.py

@@ -2,7 +2,21 @@ from __future__ import annotations
 
 from lamindb_setup.core._aws_credentials import HOSTED_REGIONS
 from lamindb_setup.core._settings_storage import get_storage_region
-from lamindb_setup.core.upath import UPath, compute_file_tree, get_stat_dir_cloud
+from lamindb_setup.core.upath import (
+    UPath,
+    compute_file_tree,
+    get_stat_dir_cloud,
+    get_stat_file_cloud,
+)
+
+
+def test_get_stat_file_cloud_aws():
+    string_path = "s3://bionty-assets/df_all__ncbitaxon__2023-06-20__Organism.parquet"
+    path = UPath(string_path, anon=True)
+    size, hash, hash_type = get_stat_file_cloud(path.stat().as_info())
+    assert hash == "zQxieeCkNGNJWhPl3OfM8A"
+    assert hash_type == "md5-5"
+    assert size == 78148228
 
 
 def test_get_stat_dir_cloud_aws():