lamindb_setup 0.81.2__py3-none-any.whl → 0.81.4__py3-none-any.whl

lamindb_setup/__init__.py

@@ -33,7 +33,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.81.2"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.81.4"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 import sys as _sys

lamindb_setup/core/_hub_core.py

@@ -239,6 +239,10 @@ def _delete_instance(
             )
             root_string = storage_record["root"]
             # gate storage and instance deletion on empty storage location for
+            # normally auth.get_session() doesn't have access_token
+            # so this block is useless i think (Sergei)
+            # the token is received from user settings inside create_path
+            # might be needed in the hub though
             if client.auth.get_session() is not None:
                 access_token = client.auth.get_session().access_token
             else:
@@ -251,7 +255,6 @@ def _delete_instance(
             check_storage_is_empty(
                 root_path, account_for_sqlite_file=account_for_sqlite_file
             )
-    _update_instance_record(instance_with_storage["id"], {"storage_id": None}, client)
     # first delete the storage records because we will turn instance_id on
     # storage into a FK soon
     for storage_record in storage_records:

lamindb_setup/core/_hub_crud.py

@@ -12,25 +12,23 @@ def select_instance_by_owner_name(
     name: str,
     client: Client,
 ) -> dict | None:
-    try:
-        data = (
-            client.table("instance")
-            .select(
-                "*, account!inner!instance_account_id_28936e8f_fk_account_id(*),"
-                " storage!inner!storage_instance_id_359fca71_fk_instance_id(*)"
-            )
-            .eq("name", name)
-            .eq("account.handle", owner)
-            .eq("storage.is_default", True)
-            .execute()
-            .data
+    # this won't find an instance without the default storage
+    data = (
+        client.table("instance")
+        .select(
+            "*, account!inner!instance_account_id_28936e8f_fk_account_id(*),"
+            " storage!inner!storage_instance_id_359fca71_fk_instance_id(*)"
         )
-    except Exception:
-        return None
+        .eq("name", name)
+        .eq("account.handle", owner)
+        .eq("storage.is_default", True)
+        .execute()
+        .data
+    )
     if len(data) == 0:
         return None
     result = data[0]
-    # this is now a list
+    # this is a list
     # assume only one default storage
     result["storage"] = result["storage"][0]
     return result
@@ -89,15 +87,22 @@ def select_instance_by_id_with_storage(
     instance_id: str,
     client: Client,
 ):
-    response = (
+    # this won't find an instance without the default storage
+    data = (
         client.table("instance")
-        .select("*, storage!instance_storage_id_87963cc8_fk_storage_id(*)")
+        .select("*, storage!inner!storage_instance_id_359fca71_fk_instance_id(*)")
         .eq("id", instance_id)
+        .eq("storage.is_default", True)
         .execute()
+        .data
     )
-    if len(response.data) == 0:
+    if len(data) == 0:
         return None
-    return response.data[0]
+    result = data[0]
+    # this is a list
+    # assume only one default storage
+    result["storage"] = result["storage"][0]
+    return result
 
 
 def update_instance(instance_id: str, instance_fields: dict, client: Client):

lamindb_setup/core/_settings.py

@@ -195,27 +195,35 @@ class SetupPaths:
         filepath: UPathStr, cache_key: str | None = None
     ) -> UPath:
         """Local (or local cache) filepath from filepath without synchronization."""
-        # cache_key is ignored if filepath is a string or a local path
-        # ignores a mere string even if it represents a cloud path
-        if isinstance(filepath, UPath) and not isinstance(filepath, LocalPathClasses):
+        if not isinstance(filepath, UPath):
+            filepath = UPath(filepath)
+        # cache_key is ignored if filepath is a local path
+        if not isinstance(filepath, LocalPathClasses):
             # settings is defined further in this file
-            local_filepath = settings.cache_dir / (
-                filepath.path if cache_key is None else cache_key
-            )
+            if cache_key is None:
+                local_key = filepath.path  # type: ignore
+                protocol = filepath.protocol  # type: ignore
+                if protocol in {"http", "https"}:
+                    local_key = local_key.removeprefix(protocol + "://")
+            else:
+                local_key = cache_key
+            local_filepath = settings.cache_dir / local_key
         else:
             local_filepath = filepath
-        return UPath(local_filepath)
+        return local_filepath
 
     @staticmethod
     def cloud_to_local(
         filepath: UPathStr, cache_key: str | None = None, **kwargs
     ) -> UPath:
         """Local (or local cache) filepath from filepath."""
-        # cache_key is ignored in cloud_to_local_no_update if filepath is local or a string
+        if not isinstance(filepath, UPath):
+            filepath = UPath(filepath)
+        # cache_key is ignored in cloud_to_local_no_update if filepath is local
         local_filepath = SetupPaths.cloud_to_local_no_update(filepath, cache_key)
-        if isinstance(filepath, UPath) and not isinstance(filepath, LocalPathClasses):
+        if not isinstance(filepath, LocalPathClasses):
             local_filepath.parent.mkdir(parents=True, exist_ok=True)
-            filepath.synchronize(local_filepath, **kwargs)
+            filepath.synchronize(local_filepath, **kwargs)  # type: ignore
         return local_filepath
 
 

lamindb_setup/core/_settings_storage.py

@@ -7,6 +7,7 @@ import string
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal
 
+import fsspec
 from lamin_utils import logger
 
 from ._aws_credentials import HOSTED_REGIONS, get_aws_credentials_manager
@@ -24,6 +25,10 @@ if TYPE_CHECKING:
 
 IS_INITIALIZED_KEY = ".lamindb/_is_initialized"
 
+# a list of supported fsspec protocols
+# rename file to local before showing to a user
+VALID_PROTOCOLS = ("file", "gs", "s3", "hf", "http", "https")
+
 
 def base62(n_char: int) -> str:
     """Like nanoid without hyphen and underscore."""
@@ -114,16 +119,11 @@ def init_storage(
             root_str = f"s3://lamin-{region}/{uid}"
         else:
             root_str = f"s3://lamin-hosted-test/{uid}"
-    elif root_str.startswith(("gs://", "s3://", "hf://")):
-        pass
-    else:  # local path
-        try:
-            _ = Path(root_str)
-        except Exception as e:
-            logger.error(
-                "`storage` is not a valid local, GCP storage, AWS S3 path or Hugging Face path"
-            )
-            raise e
+    elif (input_protocol := fsspec.utils.get_protocol(root_str)) not in VALID_PROTOCOLS:
+        valid_protocols = ("local",) + VALID_PROTOCOLS[1:]  # show local instead of file
+        raise ValueError(
+            f"Protocol {input_protocol} is not supported, valid protocols are {', '.join(valid_protocols)}"
+        )
     ssettings = StorageSettings(
         uid=uid,
         root=root_str,
@@ -227,7 +227,7 @@ class StorageSettings:
 
     @property
     def record(self) -> Any:
-        """Storage record in current instance."""
+        """Storage record in the current instance."""
         if self._record is None:
             # dynamic import because of import order
             from lnschema_core.models import Storage
@@ -299,14 +299,15 @@ class StorageSettings:
         return self._region
 
     @property
-    def type(self) -> Literal["local", "s3", "gs"]:
+    def type(self) -> Literal["local", "s3", "gs", "hf", "http", "https"]:
         """AWS S3 vs. Google Cloud vs. local.
 
-        Returns the protocol as a string: "local", "s3", "gs".
+        Returns the protocol as a string: "local", "s3", "gs", "http", "https".
         """
         import fsspec
 
         convert = {"file": "local"}
+        # init_storage checks that the root protocol belongs to VALID_PROTOCOLS
         protocol = fsspec.utils.get_protocol(self.root_as_str)
         return convert.get(protocol, protocol)  # type: ignore
 
@@ -345,5 +346,5 @@ class StorageSettings:
         return self.root / filekey
 
     def local_filepath(self, filekey: UPathStr) -> UPath:
-        """Local (cache) filepath from filekey: `local(filepath(...))`."""
+        """Local (cache) filepath from filekey."""
         return self.cloud_to_local(self.key_to_filepath(filekey))

lamindb_setup/core/upath.py

@@ -190,7 +190,13 @@ class ProgressCallback(fsspec.callbacks.Callback):
         pass
 
     def update_relative_value(self, inc=1):
-        self.value += inc
+        if inc != 0:
+            self.value += inc
+        # this is specific to http filesystem
+        # for some reason the last update is 0 always
+        # here 100% is forced manually in this case
+        elif self.value >= 0.999:
+            self.value = self.size
         self.call()
 
     def branch(self, path_1, path_2, kwargs):
@@ -350,27 +356,19 @@ def synchronize(
         exists = True
         cloud_mts = timestamp
     else:
-        # hf requires special treatment
-        if protocol == "hf":
-            try:
-                stat_hf = self.stat().as_info()
-                is_dir = stat_hf["type"] == "directory"
-                exists = True
-                if not is_dir:
-                    cloud_mts = stat_hf["last_commit"].date.timestamp()
-            except FileNotFoundError:
-                exists = False
-        else:
-            # perform only one network request to check existence, type and timestamp
-            try:
-                cloud_mts = self.modified.timestamp()
-                is_dir = False
-                exists = True
-            except FileNotFoundError:
-                exists = False
-            except IsADirectoryError:
-                is_dir = True
-                exists = True
+        try:
+            cloud_stat = self.stat()
+            cloud_info = cloud_stat.as_info()
+            exists = True
+            is_dir = cloud_info["type"] == "directory"
+            if not is_dir:
+                # hf requires special treatment
+                if protocol == "hf":
+                    cloud_mts = cloud_info["last_commit"].date.timestamp()
+                else:
+                    cloud_mts = cloud_stat.st_mtime
+        except FileNotFoundError:
+            exists = False
 
     if not exists:
         warn_or_error = f"The original path {self} does not exist anymore."
@@ -386,6 +384,7 @@ def synchronize(
         return None
 
     # synchronization logic for directories
+    # to synchronize directories, it should be possible to get modification times
     if is_dir:
         files = self.fs.find(str(self), detail=True)
         if protocol == "s3":
@@ -451,8 +450,16 @@ def synchronize(
         callback, print_progress, objectpath.name, "synchronizing"
     )
     if objectpath.exists():
-        local_mts_obj = objectpath.stat().st_mtime  # type: ignore
-        need_synchronize = cloud_mts > local_mts_obj
+        if cloud_mts != 0:
+            local_mts_obj = objectpath.stat().st_mtime
+            need_synchronize = cloud_mts > local_mts_obj
+        else:
+            # this is true for http for example
+            # where size is present but st_mtime is not
+            # we assume that any change without the change in size is unlikely
+            cloud_size = cloud_stat.st_size
+            local_size_obj = objectpath.stat().st_size
+            need_synchronize = cloud_size != local_size_obj
     else:
         objectpath.parent.mkdir(parents=True, exist_ok=True)
         need_synchronize = True
@@ -464,7 +471,8 @@ def synchronize(
         self.download_to(
             objectpath, recursive=False, print_progress=False, callback=callback
         )
-        os.utime(objectpath, times=(cloud_mts, cloud_mts))
+        if cloud_mts != 0:
+            os.utime(objectpath, times=(cloud_mts, cloud_mts))
     else:
         # nothing happens if parent_update is not defined
         # because of Callback.no_op
@@ -739,7 +747,9 @@ def get_stat_file_cloud(stat: dict) -> tuple[int, str | None, str | None]:
         hash = b16_to_b64(stat["blob_id"])
         hash_type = "sha1"
     # s3
-    elif "ETag" in stat:
+    # StorageClass is checked to be sure that it is indeed s3
+    # because http also has ETag
+    elif "ETag" in stat and "StorageClass" in stat:
         etag = stat["ETag"]
         # small files
         if "-" not in etag:

{lamindb_setup-0.81.2.dist-info → lamindb_setup-0.81.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.81.2
+Version: 0.81.4
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.9
@@ -14,12 +14,11 @@ Requires-Dist: appdirs<2.0.0
 Requires-Dist: requests
 Requires-Dist: universal_pathlib==0.2.5
 Requires-Dist: botocore<2.0.0
-Requires-Dist: gotrue>=2.1.0,<2.9.0
-Requires-Dist: supabase==2.2.1
+Requires-Dist: supabase>=2.8.1,<=2.10.0
 Requires-Dist: psutil
 Requires-Dist: urllib3<2 ; extra == "aws"
 Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
-Requires-Dist: s3fs>=2023.12.2,<=2024.10.0 ; extra == "aws"
+Requires-Dist: s3fs>=2023.12.2,<=2024.10.0,!=2024.10.0 ; extra == "aws"
 Requires-Dist: line_profiler ; extra == "dev"
 Requires-Dist: pyjwt<3.0.0 ; extra == "dev"
 Requires-Dist: psycopg2-binary ; extra == "dev"

{lamindb_setup-0.81.2.dist-info → lamindb_setup-0.81.4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-lamindb_setup/__init__.py,sha256=o71S_vO9_RpZ91eHTH0FZG57w9nwWR54OUufTiq-aN8,1714
+lamindb_setup/__init__.py,sha256=41Rm-lDXIF3ir0Xsj-vpB0EZO5SmkipsRc5pmUzBEvk,1714
 lamindb_setup/_cache.py,sha256=1XnM-V_KprbjpgPY7Bg3FYn53Iz_2_fEgcMOaSdKKbg,1332
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=6cSfpmVOSgU7YiVHfJpBTGTQ7rrnwunt1pJT_jkgNM8,3196
@@ -23,15 +23,15 @@ lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-Oq
 lamindb_setup/core/_deprecated.py,sha256=3qxUI1dnDlSeR0BYrv7ucjqRBEojbqotPgpShXs4KF8,2520
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
 lamindb_setup/core/_hub_client.py,sha256=cN19XbZmvLCxL_GKdOcKbedNRL7kR47vmLmA--NMv-U,6306
-lamindb_setup/core/_hub_core.py,sha256=eUxRz9iJj6RA5-MWgQqqZYAU-di5LQDamRZn6t-VOiM,19838
-lamindb_setup/core/_hub_crud.py,sha256=eZErpq9t1Cp2ULBSi457ekrcqfesw4Y6IJgaqyrINMY,5276
+lamindb_setup/core/_hub_core.py,sha256=cZfEDFq2z0cHjyPqwg7K5K3vw9C2nDHnbIHmmAjVbeM,19997
+lamindb_setup/core/_hub_crud.py,sha256=ro4dvZ0EaHs7_QwJH6BoKl69Wc9HEPDkIc9BFyO_DMg,5460
 lamindb_setup/core/_hub_utils.py,sha256=08NwQsb53-tXa_pr-f0tPTN0FeeVf_i1p3dEbEWD0F4,3016
 lamindb_setup/core/_private_django_api.py,sha256=KIn43HOhiRjkbTbddyJqv-WNTTa1bAizbM1tWXoXPBg,2869
-lamindb_setup/core/_settings.py,sha256=mpGsSb98UsBedLsW2RuowZ17EP2tI2XRGPztqrJtrV4,7952
+lamindb_setup/core/_settings.py,sha256=NYrSGtAw68mlQJgzBnV-DECi-RUBn32eeosyYCBtW3I,8232
 lamindb_setup/core/_settings_instance.py,sha256=ajcq9zRNE598tTqyMkMqaEOubVfFeE998DPtbgyzK3A,18801
 lamindb_setup/core/_settings_load.py,sha256=5OpghcbkrK9KBM_0Iu-61FTI76UbOpPkkJpUittXS-w,4098
 lamindb_setup/core/_settings_save.py,sha256=rxGxgaK5i9exKqSJERQQyY1WZio20meoQJoYXlVW-1w,3138
-lamindb_setup/core/_settings_storage.py,sha256=CYwGZm0fKYN7eLLsU-sOtOKG7HzswQVjTWb0ooHKcNg,11990
+lamindb_setup/core/_settings_storage.py,sha256=FOTyi89BZ48a8elCQ91C4B6oMO4tBnva4J6Tk40N9Ww,12249
 lamindb_setup/core/_settings_store.py,sha256=WcsgOmgnu9gztcrhp-N4OONNZyxICHV8M0HdJllTaEo,2219
 lamindb_setup/core/_settings_user.py,sha256=iz0MqFLKXqm8LYx_CHmr02_oNvYWFLIxKkJLdpS5W08,1476
 lamindb_setup/core/_setup_bionty_sources.py,sha256=jZOPXpipW_5IjMO-bLMk-_wVwk7-5MLd72K2rnqqy7U,4001
@@ -40,8 +40,8 @@ lamindb_setup/core/django.py,sha256=E4U9nUlV2kHd-G5v6iSdFGAAWixlQDxOFwMwOMG9xfw,
 lamindb_setup/core/exceptions.py,sha256=4NpLUNUIfXYVTFX2FvLZF8RW34exk2Vn2X3G4YhnTRg,276
 lamindb_setup/core/hashing.py,sha256=26dtak7XgmrWa_D1zuDyxObRQcriMtnc1yEigkKASmM,3142
 lamindb_setup/core/types.py,sha256=zJii2le38BJUmsNVvzDrbzGYr0yaeb-9Rw9IKmsBr3k,523
-lamindb_setup/core/upath.py,sha256=a0yxP9dmujTn_hkDC0E2UuVsX-Img4i0kVNB-OV5K1s,29038
-lamindb_setup-0.81.2.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-0.81.2.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb_setup-0.81.2.dist-info/METADATA,sha256=qUDiU989EwoOfdbfJ8jXPCiGLXc1TLwgXQQAN5H1Iu0,1745
-lamindb_setup-0.81.2.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=X5Jjm50-br1Qq3eRlO6U4ykidptc2QzoQtq-ilaPt4Q,29602
+lamindb_setup-0.81.4.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-0.81.4.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb_setup-0.81.4.dist-info/METADATA,sha256=xstSHpwOECuKUylOziRLHFHtafUtglGT1CQA0lUGscs,1730
+lamindb_setup-0.81.4.dist-info/RECORD,,