lamindb_setup 0.80.1__py3-none-any.whl → 0.81.0__py3-none-any.whl

lamindb_setup/__init__.py

@@ -33,7 +33,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.80.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.81.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 import sys as _sys

lamindb_setup/core/_settings_storage.py

@@ -114,13 +114,15 @@ def init_storage(
             root_str = f"s3://lamin-{region}/{uid}"
         else:
             root_str = f"s3://lamin-hosted-test/{uid}"
-    elif root_str.startswith(("gs://", "s3://")):
+    elif root_str.startswith(("gs://", "s3://", "hf://")):
         pass
     else:  # local path
         try:
             _ = Path(root_str)
         except Exception as e:
-            logger.error("`storage` is not a valid local, GCP storage or AWS S3 path")
+            logger.error(
+                "`storage` is not a valid local, GCP storage, AWS S3 path or Hugging Face path"
+            )
             raise e
     ssettings = StorageSettings(
         uid=uid,
@@ -161,6 +163,7 @@ def init_storage(
         # only newly created
         if hub_record_status == "hub-record-created" and ssettings._uuid is not None:
             delete_storage_record(ssettings._uuid, access_token=access_token)  # type: ignore
+            hub_record_status = "hub-record-not-created"
         ssettings._instance_id = None
     return ssettings, hub_record_status
 

lamindb_setup/core/hashing.py

@@ -47,14 +47,14 @@ def hash_set(s: set[str]) -> str:
     return to_b64_str(hashlib.md5(bstr).digest())[:HASH_LENGTH]
 
 
-def hash_md5s_from_dir(hashes: Iterable[str]) -> tuple[str, str]:
+def hash_from_hashes_list(hashes: Iterable[str]) -> str:
     # need to sort below because we don't want the order of parsing the dir to
     # affect the hash
     digests = b"".join(
         hashlib.md5(hash.encode("utf-8")).digest() for hash in sorted(hashes)
     )
     digest = hashlib.md5(digests).digest()
-    return to_b64_str(digest)[:HASH_LENGTH], "md5-d"
+    return to_b64_str(digest)[:HASH_LENGTH]
 
 
 def hash_code(file_path: UPathStr):
@@ -110,7 +110,7 @@ def hash_dir(path: Path):
         hashes_sizes = map(hash_size, files)
     hashes, sizes = zip(*hashes_sizes)
 
-    hash, hash_type = hash_md5s_from_dir(hashes)
+    hash, hash_type = hash_from_hashes_list(hashes), "md5-d"
     n_objects = len(hashes)
     size = sum(sizes)
     return size, hash, hash_type, n_objects

lamindb_setup/core/upath.py

@@ -18,7 +18,7 @@ from upath.implementations.cloud import CloudPath, S3Path  # keep CloudPath!
 from upath.implementations.local import LocalPath
 
 from ._aws_credentials import HOSTED_BUCKETS, get_aws_credentials_manager
-from .hashing import HASH_LENGTH, b16_to_b64, hash_md5s_from_dir
+from .hashing import HASH_LENGTH, b16_to_b64, hash_from_hashes_list
 
 if TYPE_CHECKING:
     from .types import UPathStr
@@ -340,22 +340,34 @@ def synchronize(
     timestamp: float | None = None,
 ):
     """Sync to a local destination path."""
+    protocol = self.protocol
     # optimize the number of network requests
     if timestamp is not None:
         is_dir = False
         exists = True
         cloud_mts = timestamp
     else:
-        # perform only one network request to check existence, type and timestamp
-        try:
-            cloud_mts = self.modified.timestamp()
-            is_dir = False
-            exists = True
-        except FileNotFoundError:
-            exists = False
-        except IsADirectoryError:
-            is_dir = True
-            exists = True
+        # hf requires special treatment
+        if protocol == "hf":
+            try:
+                stat_hf = self.stat().as_info()
+                is_dir = stat_hf["type"] == "directory"
+                exists = True
+                if not is_dir:
+                    cloud_mts = stat_hf["last_commit"].date.timestamp()
+            except FileNotFoundError:
+                exists = False
+        else:
+            # perform only one network request to check existence, type and timestamp
+            try:
+                cloud_mts = self.modified.timestamp()
+                is_dir = False
+                exists = True
+            except FileNotFoundError:
+                exists = False
+            except IsADirectoryError:
+                is_dir = True
+                exists = True
 
     if not exists:
         warn_or_error = f"The original path {self} does not exist anymore."
@@ -373,14 +385,18 @@ def synchronize(
     # synchronization logic for directories
     if is_dir:
         files = self.fs.find(str(self), detail=True)
-        protocol_modified = {"s3": "LastModified", "gs": "mtime"}
-        modified_key = protocol_modified.get(self.protocol, None)
-        if modified_key is None:
-            raise ValueError(f"Can't synchronize a directory for {self.protocol}.")
+        if protocol == "s3":
+            get_modified = lambda file_stat: file_stat["LastModified"]
+        elif protocol == "gs":
+            get_modified = lambda file_stat: file_stat["mtime"]
+        elif protocol == "hf":
+            get_modified = lambda file_stat: file_stat["last_commit"].date
+        else:
+            raise ValueError(f"Can't synchronize a directory for {protocol}.")
         if objectpath.exists():
             destination_exists = True
             cloud_mts_max = max(
-                file[modified_key] for file in files.values()
+                get_modified(file) for file in files.values()
             ).timestamp()
             local_mts = [
                 file.stat().st_mtime for file in objectpath.rglob("*") if file.is_file()
@@ -405,9 +421,8 @@ def synchronize(
             for file, stat in callback.wrap(files.items()):
                 file_key = PurePosixPath(file).relative_to(self.path).as_posix()
                 origin_file_keys.append(file_key)
-                timestamp = stat[modified_key].timestamp()
-
-                origin = f"{self.protocol}://{file}"
+                timestamp = get_modified(stat).timestamp()
+                origin = f"{protocol}://{file}"
                 destination = objectpath / file_key
                 child = callback.branched(origin, destination.as_posix())
                 UPath(origin, **self.storage_options).synchronize(
@@ -439,6 +454,10 @@ def synchronize(
         objectpath.parent.mkdir(parents=True, exist_ok=True)
         need_synchronize = True
     if need_synchronize:
+        # hf has sync filesystem
+        # on sync filesystems ChildProgressCallback.branched()
+        # returns the default callback
+        # this is why a difference between s3 and hf in progress bars
         self.download_to(
             objectpath, recursive=False, print_progress=False, callback=callback
         )
@@ -698,38 +717,60 @@ def create_path(path: UPath, access_token: str | None = None) -> UPath:
     return get_aws_credentials_manager().enrich_path(path, access_token)
 
 
-def get_stat_file_cloud(stat: dict) -> tuple[int, str, str]:
+def get_stat_file_cloud(stat: dict) -> tuple[int, str | None, str | None]:
     size = stat["size"]
-    etag = stat["ETag"]
-    # small files
-    if "-" not in etag:
-        # only store hash for non-multipart uploads
-        # we can't rapidly validate multi-part uploaded files client-side
-        # we can add more logic later down-the-road
-        hash = b16_to_b64(etag)
+    hash, hash_type = None, None
+    # gs, use md5Hash instead of etag for now
+    if "md5Hash" in stat:
+        # gs hash is already in base64
+        hash = stat["md5Hash"].strip('"=')
         hash_type = "md5"
-    else:
-        stripped_etag, suffix = etag.split("-")
-        suffix = suffix.strip('"')
-        hash = b16_to_b64(stripped_etag)
-        hash_type = f"md5-{suffix}"  # this is the S3 chunk-hashing strategy
-    return size, hash[:HASH_LENGTH], hash_type
+    # hf
+    elif "blob_id" in stat:
+        hash = b16_to_b64(stat["blob_id"])
+        hash_type = "sha1"
+    # s3
+    elif "ETag" in stat:
+        etag = stat["ETag"]
+        # small files
+        if "-" not in etag:
+            # only store hash for non-multipart uploads
+            # we can't rapidly validate multi-part uploaded files client-side
+            # we can add more logic later down-the-road
+            hash = b16_to_b64(etag)
+            hash_type = "md5"
+        else:
+            stripped_etag, suffix = etag.split("-")
+            suffix = suffix.strip('"')
+            hash = b16_to_b64(stripped_etag)
+            hash_type = f"md5-{suffix}"  # this is the S3 chunk-hashing strategy
+    if hash is not None:
+        hash = hash[:HASH_LENGTH]
+    return size, hash, hash_type
 
 
-def get_stat_dir_cloud(path: UPath) -> tuple[int, str, str, int]:
-    sizes = []
-    md5s = []
+def get_stat_dir_cloud(path: UPath) -> tuple[int, str | None, str | None, int]:
     objects = path.fs.find(path.as_posix(), detail=True)
+    hash, hash_type = None, None
+    compute_list_hash = True
     if path.protocol == "s3":
         accessor = "ETag"
     elif path.protocol == "gs":
         accessor = "md5Hash"
+    elif path.protocol == "hf":
+        accessor = "blob_id"
+    else:
+        compute_list_hash = False
+    sizes = []
+    hashes = []
     for object in objects.values():
         sizes.append(object["size"])
-        md5s.append(object[accessor].strip('"='))
+        if compute_list_hash:
+            hashes.append(object[accessor].strip('"='))
     size = sum(sizes)
-    hash, hash_type = hash_md5s_from_dir(md5s)
-    n_objects = len(md5s)
+    n_objects = len(sizes)
+    if compute_list_hash:
+        hash, hash_type = hash_from_hashes_list(hashes), "md5-d"
     return size, hash, hash_type, n_objects
 
 

{lamindb_setup-0.80.1.dist-info → lamindb_setup-0.81.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.80.1
+Version: 0.81.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.9
@@ -19,7 +19,7 @@ Requires-Dist: supabase==2.2.1
 Requires-Dist: psutil
 Requires-Dist: urllib3<2 ; extra == "aws"
 Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
-Requires-Dist: s3fs>=2023.12.2,<=2024.6.1 ; extra == "aws"
+Requires-Dist: s3fs>=2023.12.2,<=2024.10.0 ; extra == "aws"
 Requires-Dist: line_profiler ; extra == "dev"
 Requires-Dist: pyjwt<3.0.0 ; extra == "dev"
 Requires-Dist: psycopg2-binary ; extra == "dev"
@@ -31,7 +31,7 @@ Requires-Dist: pytest-xdist ; extra == "dev"
 Requires-Dist: nbproject-test>=0.4.3 ; extra == "dev"
 Requires-Dist: pandas ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
-Requires-Dist: gcsfs>=2023.12.2,<=2024.6.1 ; extra == "gcp"
+Requires-Dist: gcsfs>=2023.12.2,<=2024.10.0 ; extra == "gcp"
 Project-URL: Home, https://github.com/laminlabs/lamindb-setup
 Provides-Extra: aws
 Provides-Extra: dev

{lamindb_setup-0.80.1.dist-info → lamindb_setup-0.81.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-lamindb_setup/__init__.py,sha256=kc1LSkxZIe6SeFBsho6Xg6WyYxKcsUYoLNgQlQSRnxo,1714
+lamindb_setup/__init__.py,sha256=Xm__DIYPUbJSjykjcT4bC-mR75b1GFYVGZ0owhVfCyI,1714
 lamindb_setup/_cache.py,sha256=1XnM-V_KprbjpgPY7Bg3FYn53Iz_2_fEgcMOaSdKKbg,1332
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=6cSfpmVOSgU7YiVHfJpBTGTQ7rrnwunt1pJT_jkgNM8,3196
@@ -31,17 +31,17 @@ lamindb_setup/core/_settings.py,sha256=mpGsSb98UsBedLsW2RuowZ17EP2tI2XRGPztqrJtr
 lamindb_setup/core/_settings_instance.py,sha256=ajcq9zRNE598tTqyMkMqaEOubVfFeE998DPtbgyzK3A,18801
 lamindb_setup/core/_settings_load.py,sha256=5OpghcbkrK9KBM_0Iu-61FTI76UbOpPkkJpUittXS-w,4098
 lamindb_setup/core/_settings_save.py,sha256=rxGxgaK5i9exKqSJERQQyY1WZio20meoQJoYXlVW-1w,3138
-lamindb_setup/core/_settings_storage.py,sha256=15B7taJF1zxJ1_qAb67NuXkTFvO2TRTWMt6KTzDf1mw,11875
+lamindb_setup/core/_settings_storage.py,sha256=CYwGZm0fKYN7eLLsU-sOtOKG7HzswQVjTWb0ooHKcNg,11990
 lamindb_setup/core/_settings_store.py,sha256=WcsgOmgnu9gztcrhp-N4OONNZyxICHV8M0HdJllTaEo,2219
 lamindb_setup/core/_settings_user.py,sha256=iz0MqFLKXqm8LYx_CHmr02_oNvYWFLIxKkJLdpS5W08,1476
 lamindb_setup/core/_setup_bionty_sources.py,sha256=o2L5Ww8TKgSqJtL4cGUcpJwLNYxA9BZgddhCMCu_E2g,3428
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=i6TrT7HG0lqliPvZTlsZ_uplPaqhPBbabyfeR32SkA8,7107
 lamindb_setup/core/django.py,sha256=E4U9nUlV2kHd-G5v6iSdFGAAWixlQDxOFwMwOMG9xfw,3864
 lamindb_setup/core/exceptions.py,sha256=4NpLUNUIfXYVTFX2FvLZF8RW34exk2Vn2X3G4YhnTRg,276
-lamindb_setup/core/hashing.py,sha256=bkuvZyAuC7-Y_qZumJd_rybF-upJ5J3KxnKiymRUifw,3148
+lamindb_setup/core/hashing.py,sha256=26dtak7XgmrWa_D1zuDyxObRQcriMtnc1yEigkKASmM,3142
 lamindb_setup/core/types.py,sha256=zJii2le38BJUmsNVvzDrbzGYr0yaeb-9Rw9IKmsBr3k,523
-lamindb_setup/core/upath.py,sha256=q6WvpdXO_-Ajl5qjU4CIf0Q1ZbYxA0Po54LG9iHYn28,27151
-lamindb_setup-0.80.1.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-0.80.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb_setup-0.80.1.dist-info/METADATA,sha256=kMZ0UUKGc3sxehsLkXFPedNRYpo_VNNHKA-iu7I5W4I,1743
-lamindb_setup-0.80.1.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=GD-EW9QSqJH023ox53hPDvjE86hFjXVhb0MSEU02HeY,28702
+lamindb_setup-0.81.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-0.81.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb_setup-0.81.0.dist-info/METADATA,sha256=mEUNPAf6Q4NfqeO2YuUIlADbKkLQLOFFzL-BNkvk3dY,1745
+lamindb_setup-0.81.0.dist-info/RECORD,,