thds.adls 4.4.20251117191451__py3-none-any.whl → 4.5.20260110021526__py3-none-any.whl

thds/adls/__init__.py

@@ -3,6 +3,7 @@ from thds import core
 from . import (  # noqa: F401
     abfss,
     blob_meta,
+    blobs,
     defaults,
     etag,
     fqn,

thds/adls/_etag.py

@@ -0,0 +1,46 @@
+# this module is for handling some new functionality related to using etags as a fallback
+# for file hashing when the file properties do not include locally-verifiable hash information.
+import typing as ty
+from pathlib import Path
+
+import xxhash
+
+from thds.core import config, hash_cache, home, log, types
+
+ETAG_FAKE_HASH_NAME = "adls-azure-etag-fake"
+logger = log.getLogger(__name__)
+
+
+def extract_etag_bytes(etag_str: str) -> bytes:
+    # ADLS etags may or may not be quoted depending on the API used:
+    # list_blobs returns unquoted, get_*_properties returns quoted.
+    # Strip quotes first, then calculate byte length from the stripped string.
+    stripped = etag_str.strip('"')
+    return int(stripped, 16).to_bytes((len(stripped) - 2 + 1) // 2, byteorder="big")
+
+
+_ETAG_CACHE = config.item("cache-path", home.HOMEDIR() / ".thds/adls/xxhash-onto-etag", parse=Path)
+
+
+def add_to_etag_cache(local_path: types.StrOrPath, etag: bytes) -> hash_cache.Hash:
+    xxh_bytes = hash_cache.hash_file(local_path, xxhash.xxh3_128())
+    etag_path = _ETAG_CACHE() / xxh_bytes.hex()
+    etag_path.parent.mkdir(parents=True, exist_ok=True)
+    etag_path.write_bytes(etag)
+    logger.debug("Writing etag 'hash' to path at %s", etag_path)
+    return hash_cache.Hash(ETAG_FAKE_HASH_NAME, etag)
+
+
+def hash_file_fake_etag(local_path: types.StrOrPath) -> ty.Optional[hash_cache.Hash]:
+    try:
+        xxh_bytes = hash_cache.hash_file(local_path, xxhash.xxh3_128())
+    except FileNotFoundError:
+        return None
+
+    etag_path = _ETAG_CACHE() / xxh_bytes.hex()
+    if etag_path.is_file():
+        etag_bytes = etag_path.read_bytes()
+        logger.debug("Reusing etag 'fake hash' from path at %s", etag_path)
+        return hash_cache.Hash(ETAG_FAKE_HASH_NAME, etag_bytes)
+
+    return None

thds/adls/blobs.py

@@ -0,0 +1,23 @@
+"""TODO: better organize the blobs-related modules in thds.adls"""
+
+import heapq
+
+from thds.core import log, source
+
+from .fqn import AdlsFqn
+from .impl import ADLSFileSystem
+from .source import get_with_hash
+
+_logger = log.getLogger(__name__)
+
+
+def most_recent_blobs(blobs_fqn: AdlsFqn, top_n: int = 1) -> list[source.Source]:
+    """Gets top n most recently-created blob in the directory at `blobs_fqn`."""
+    _logger.info(f"Enumerating the most recent blobs in {blobs_fqn}")
+    fs = ADLSFileSystem(blobs_fqn.sa, blobs_fqn.container)
+    snapshots = fs.get_directory_info(blobs_fqn.path, recursive=False)
+    if not snapshots:
+        raise ValueError(f"No blobs found in {blobs_fqn}")
+    top_blobs = heapq.nlargest(top_n, snapshots, key=lambda x: x.creation_time or -1)
+
+    return [get_with_hash(blobs_fqn.root() / item.name) for item in top_blobs if item.name]

thds/adls/copy.py

@@ -10,6 +10,7 @@ from azure.storage.blob import BlobSasPermissions, BlobServiceClient, UserDelega
 
 from thds.core import cache, log, parallel, thunks
 
+from ._etag import ETAG_FAKE_HASH_NAME
 from .file_properties import exists, get_blob_properties, get_file_properties, is_directory
 from .fqn import AdlsFqn
 from .global_client import get_global_blob_container_client, get_global_blob_service_client
@@ -60,7 +61,18 @@ def _copy_file(
     def hashes_exist_and_are_equal() -> bool:
         src_blob_props = src_blob_client.get_blob_properties()
         dest_blob_props = dest_blob_client.get_blob_properties()
-        return extract_hashes_from_props(src_blob_props) == extract_hashes_from_props(dest_blob_props)
+        # exclude etag from comparison since it's unique per blob and will always differ
+        src_hashes = {
+            k: v
+            for k, v in extract_hashes_from_props(src_blob_props).items()
+            if k != ETAG_FAKE_HASH_NAME
+        }
+        dest_hashes = {
+            k: v
+            for k, v in extract_hashes_from_props(dest_blob_props).items()
+            if k != ETAG_FAKE_HASH_NAME
+        }
+        return src_hashes == dest_hashes
 
     if dest_blob_client.exists():
         if hashes_exist_and_are_equal():

thds/adls/download.py

@@ -118,7 +118,7 @@ def _attempt_cache_hit(
     with log.logger_context(hash_for="before-download-dest"):
         local_hash = hash_path_if_exists(local_path)
     if local_hash == expected_hash:
-        logger.debug("Local path matches %s - no need to look further", expected_hash.algo)
+        logger.debug("Local path matches '%s' hash - no need to look further", expected_hash.algo)
         if cache:
             cache_path = cache.path(fqn)
             with log.logger_context(hash_for="before-download-cache"):
@@ -235,6 +235,12 @@ def _download_or_use_verified_cached_coroutine(  # noqa: C901
 
     # attempt cache hits before taking a lock, to avoid contention for existing files.
     if file_result := attempt_cache_hit():
+        logger.debug(
+            "No download - found cached version of %s using expected %s at %s",
+            fqn,
+            expected_hash,
+            file_result.hit,
+        )
         return file_result  # noqa: B901
 
     # No cache hit, so its time to prepare to download. if a cache was provided, we will
@@ -344,11 +350,15 @@ def download_or_use_verified(
     *,
     expected_hash: ty.Optional[hashing.Hash] = None,
     cache: ty.Optional[Cache] = None,
+    set_remote_hash: bool = True,
 ) -> ty.Optional[Path]:
     """Download a file or use the existing, cached copy if one exists in the cache and is verifiable.
 
     Note that you will get a logged warning if `local_path` already exists when you call
     this function.
+
+    If set_remote_hash is False, the function will not attempt to set hash metadata on the
+    remote file after download. This is useful when downloading from read-only locations.
     """
     file_properties = None
     try:
@@ -372,7 +382,9 @@ def download_or_use_verified(
             else:
                 raise ValueError(f"Unexpected coroutine request: {co_request}")
     except StopIteration as si:
-        if meta := hashes.create_hash_metadata_if_missing(file_properties, si.value.hash):
+        if set_remote_hash and (
+            meta := hashes.create_hash_metadata_if_missing(file_properties, si.value.hash)
+        ):
             try:
                 logger.info(f"Setting missing {si.value.hash.algo} hash for {remote_key}")
                 assert file_properties
@@ -399,6 +411,7 @@ async def async_download_or_use_verified(
     *,
     expected_hash: ty.Optional[hashing.Hash] = None,
     cache: ty.Optional[Cache] = None,
+    set_remote_hash: bool = True,
 ) -> ty.Optional[Path]:
     file_properties = None
     try:
@@ -429,7 +442,9 @@ async def async_download_or_use_verified(
                 raise ValueError(f"Unexpected coroutine request: {co_request}")
 
     except StopIteration as si:
-        if meta := hashes.create_hash_metadata_if_missing(file_properties, si.value.hash):
+        if set_remote_hash and (
+            meta := hashes.create_hash_metadata_if_missing(file_properties, si.value.hash)
+        ):
             try:
                 logger.info(f"Setting missing Hash for {remote_key}")
                 assert file_properties

thds/adls/file_properties.py

@@ -37,6 +37,10 @@ class PropertiesP(ty.Protocol):
     name: ty.Any
     metadata: ty.Any
 
+    @property
+    def etag(self) -> ty.Union[str, None, ty.Any]:
+        pass
+
     @property
     def content_settings(self) -> ContentSettingsP:
         pass

thds/adls/hashes.py

@@ -1,5 +1,6 @@
 import contextlib
 import os
+import sys
 import typing as ty
 from functools import partial
 
@@ -9,6 +10,7 @@ from thds.core import hash_cache, hashing, log, source, types
 from thds.core.hashing import Hash, SomehowReadable
 
 from . import errors, file_properties
+from ._etag import ETAG_FAKE_HASH_NAME, add_to_etag_cache, extract_etag_bytes, hash_file_fake_etag
 from .fqn import AdlsFqn
 
 logger = log.getLogger(__name__)
@@ -53,6 +55,9 @@ def hash_path_for_algo(
     algo: str,
 ) -> ty.Callable[[types.StrOrPath], ty.Optional[hashing.Hash]]:
     """Return a function that hashes a path for the given algorithm."""
+    if algo == ETAG_FAKE_HASH_NAME:
+        return hash_file_fake_etag
+
     return partial(_hash_path_if_exists, partial(hash_cache.filehash, algo))
 
 
@@ -77,6 +82,13 @@ def extract_hashes_from_props(
     hashes = list(extract_hashes_from_metadata(props.metadata or dict()))
     if props.content_settings and props.content_settings.content_md5:
         hashes.append(hashing.Hash("md5", bytes(props.content_settings.content_md5)))
+
+    if props.etag:
+        # this is the final fallback. it cannot be checked locally, but at least
+        # it can be checked against what exists remotely the next time we want to use it.
+        if etag_bytes := extract_etag_bytes(props.etag):
+            hashes.append(hashing.Hash(sys.intern(ETAG_FAKE_HASH_NAME), etag_bytes))
+
     return {h.algo: h for h in hashes}
 
 
@@ -87,10 +99,6 @@ def verify_hashes_before_and_after_download(
     fqn: AdlsFqn,
     local_dest: types.StrOrPath,
 ) -> ty.Iterator[None]:
-    # if expected_hash:
-    #     check_reasonable_md5b64(expected_md5b64)
-    # if remote_md5b64:
-    #     check_reasonable_md5b64(remote_md5b64)
     if remote_hash and expected_hash and remote_hash != expected_hash:
         raise errors.HashMismatchError(
             f"ADLS thinks the {remote_hash.algo} of {fqn} is {hashing.b64(remote_hash.bytes)},"
@@ -105,11 +113,16 @@ def verify_hashes_before_and_after_download(
         expected_algo = remote_hash.algo
 
     if not expected_algo:
-        # if we have neither a user-provided hash nor a remotely-foun9d hash, then we have nothing to check.
+        # if we have neither a user-provided hash nor a remotely-found hash, then we have nothing to check.
         return
 
+    assert expected_hash or remote_hash, "At least one of expected or remote hash must be present."
     with log.logger_context(hash_for="after-download"):
-        local_hash = hash_cache.filehash(expected_algo, local_dest)
+        if expected_algo == ETAG_FAKE_HASH_NAME:
+            assert remote_hash, f"An Etag hash should always originate remotely: {fqn}"
+            local_hash = add_to_etag_cache(local_dest, remote_hash.bytes)
+        else:
+            local_hash = hash_cache.filehash(expected_algo, local_dest)
 
     if remote_hash and remote_hash != local_hash:
         raise errors.HashMismatchError(
@@ -142,6 +155,10 @@ def create_hash_metadata_if_missing(
         # without file properties, we can't match the etag when we try to set this.
         return dict()
 
+    if new_hash.algo == ETAG_FAKE_HASH_NAME:
+        # we never want to write etag-based hashes into metadata.
+        return dict()
+
     existing_metadata = file_properties.metadata or dict()
     if metadata_hash_b64_key(new_hash.algo) not in existing_metadata:
         return {**existing_metadata, **metadata_hash_dict(new_hash)}

thds/adls/source.py

@@ -6,6 +6,7 @@ from thds.core import source
 from thds.core.hashing import Hash
 
 from . import cached, hashes, md5
+from .cached import upload_through_cache
 from .errors import blob_not_found_translation
 from .file_properties import get_file_properties
 from .fqn import AdlsFqn
@@ -71,3 +72,17 @@ def get_with_hash(fqn_or_uri: ty.Union[AdlsFqn, str]) -> source.Source:
 def with_md5b64(uri_or_fqn: ty.Union[str, AdlsFqn], *, md5b64: str = "") -> source.Source:
     """Meant for older use cases where we had an MD5"""
     return from_adls(uri_or_fqn, md5.to_hash(md5b64) if md5b64 else None)
+
+
+def _upload_handler(dest_uri: str) -> ty.Optional[source.Uploader]:
+    if dest_fqn := resolve_uri(dest_uri):
+
+        def upload_to_adls(local_path: Path, hash: ty.Optional[Hash]) -> None:
+            upload_through_cache(dest_fqn, local_path)
+
+        return upload_to_adls
+
+    return None
+
+
+source.register_upload_handler("thds.adls", _upload_handler)

{thds_adls-4.4.20251117191451.dist-info → thds_adls-4.5.20260110021526.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.adls
-Version: 4.4.20251117191451
+Version: 4.5.20260110021526
 Summary: ADLS tools
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

{thds_adls-4.4.20251117191451.dist-info → thds_adls-4.5.20260110021526.dist-info}/RECORD

@@ -1,22 +1,24 @@
-thds/adls/__init__.py,sha256=CzNcmTdP5RHrD-9K1boDU4Z3pbJAvEAFfFhXGFwpEXw,1088
+thds/adls/__init__.py,sha256=MXsKVIZ3uDayyKEXApWn-huhK9JcqlSl-12wE_lUcyo,1099
+thds/adls/_etag.py,sha256=amzbykSwmt5S426M_GXXr2vjwI1NhxYO_GvY-rA7E3Y,1779
 thds/adls/_fork_protector.py,sha256=WML1Ul-G_MCziBOyRp6GoNq1IGktn8-OQVjny1IZ1bs,950
 thds/adls/_progress.py,sha256=0cmFZ2sOEogQY25AkMAhcunXO69FdlbkD1rU_zmLAp0,6648
 thds/adls/_upload.py,sha256=rN-nuRZur9VvSUywaQezUi_qedj-xU30fTBG6rasuIA,4853
 thds/adls/abfss.py,sha256=nCFfcdwLiwtz_MdbrpLJ9WOhoz0zHIVxsf-tarorEwM,727
 thds/adls/blob_meta.py,sha256=DDhRK3pNeMaDUjiClXZ9g8nRD-ECZp21gLZGCKSxmfk,1404
+thds/adls/blobs.py,sha256=Rzw1gDlvI-CswUS8Wd-ebWxGxoKAkR7kC_OKn-QRxzc,869
 thds/adls/cached.py,sha256=up1F5JOVXdmwdZ8RAB2UDgiy6ooLg8IMULohBh75VpQ,3034
 thds/adls/conf.py,sha256=nTw3X1ilC3A_905jZH-rWXFsESeHAKQn5IghvfX2VIo,1991
-thds/adls/copy.py,sha256=aD6AquUR8r5W9SXd6Nm1qPrFH_fYpLC5dZk6HjPJnSQ,6611
+thds/adls/copy.py,sha256=-_5eDKRfhFfR7pGPs257cQL2x0JJTIXKDi3AB-fAtqc,7007
 thds/adls/dbfs.py,sha256=pPAjbIZRKJsaXKQljDMUgqS_zy1yKeEZHGMueXbuv3g,2219
 thds/adls/defaults.py,sha256=GGq5Pn4r-8cX4bZItp4nnwWAAz7S07pzPoOegw0y5Fw,676
-thds/adls/download.py,sha256=IPg5nz_sGE7dX8DUQyWjG2D9z54PXLScap-pZzTUFTk,19142
+thds/adls/download.py,sha256=jdg8t5lTHhJmH7qLbwxUCCPSErPdEtHUEVQFLSFgRe4,19672
 thds/adls/errors.py,sha256=6NMLHtVNsWBRDXaes9yzHj9cwKOD9t1dwL4BltdtjhU,1895
 thds/adls/etag.py,sha256=ct7jpHhNFcKzbekn5rZ3m6DhjK48A7qOZGwDiHkc-pc,242
 thds/adls/file_lock.py,sha256=yLak5XDpnIYwfUNdpGFbIGG64uEs98-yVscNpJlqMxM,3176
-thds/adls/file_properties.py,sha256=dhRtbsMNOYfExkEiy76wrLfrJ6IMQeN1Z3LIxgKceqY,2042
+thds/adls/file_properties.py,sha256=xtI2a0ahcqcJRernoDipeEbn2r_I_pMyR0ZSoapkDgc,2121
 thds/adls/fqn.py,sha256=pIfEw25SjZNGmtzOLwrYCblciK35VQ35ZWb_mRaZKK0,5915
 thds/adls/global_client.py,sha256=q6oG1OOsfl4fbti81u8TE9d4Jx9-phlYgsGSc4032w8,3721
-thds/adls/hashes.py,sha256=2x1zcT_87en_vqFrLFs6F_EZCGpn7hk_81dYAwcypm8,5459
+thds/adls/hashes.py,sha256=t2EZHWNN7N0VkkH1CyE1l5BNAjmn78F5k03fUFErWK0,6289
 thds/adls/impl.py,sha256=cNf1vmeS46X_wvyVdDJ8qFfowHn2QwtU5C80BmDtu5Y,43247
 thds/adls/list_fast.py,sha256=VTQ0C-78ucnywfCWRWSZ8Kfqhi2jt9VXArhpbiK0cII,4857
 thds/adls/md5.py,sha256=hGT8AIX32VUsnRCbm8cel9OlxAiRrgjwNWQTqRDHM_k,374
@@ -25,7 +27,7 @@ thds/adls/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/adls/ro_cache.py,sha256=P-UVFZhqnE5wojqYmRVWZcqjl-pM1xVMm9VAm3nXlnA,4769
 thds/adls/sas_tokens.py,sha256=mArbB_GYohevOmArw_1gKqVUWpv6kG8Hsbvdrhbtnbg,1957
 thds/adls/shared_credential.py,sha256=-x42aXoIM001KW59oS8PpuXQd4-F2vg-1gB6OMHlpk4,4602
-thds/adls/source.py,sha256=5t9bc3tUxdh4QdbRvvVd9xF9_Rv4v7CDAqE09M3rE1g,2657
+thds/adls/source.py,sha256=G9C5ncWSxbLCARDoPnhsQIgvTlFuNlSucMUiUoRmt60,3056
 thds/adls/source_tree.py,sha256=gJVpFUI3pjXRM5VmppyAcsmhJwtTl8tGLhed0TvlDOM,622
 thds/adls/upload.py,sha256=XJvygYzn1r6pZAl460s5j-kY9dgfQ6qeoddID7R6OQ0,6621
 thds/adls/uri.py,sha256=9MXuW_KfpPvzBc4ERxuTJ3vvi_6yr7e1kMAW9mx2zXM,1414
@@ -39,8 +41,8 @@ thds/adls/tools/download.py,sha256=CW2cWbCRdUqisVVVoqqvqk5Ved7pPGTkwnZj3uV0jy4,1
 thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
 thds/adls/tools/ls_fast.py,sha256=Nowc-efAL_Y4ybPwZzKIeh7KGIjfecRzdWvJZcBzq_8,585
 thds/adls/tools/upload.py,sha256=5WyWkpuVp2PETZ3O3ODlq8LXszSHU73ZMnIDZXPJdC8,442
-thds_adls-4.4.20251117191451.dist-info/METADATA,sha256=phV7EH6lnptlnQYY5TSfyZZk0Wiv0wqZ6L6o7pcP4UM,4586
-thds_adls-4.4.20251117191451.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_adls-4.4.20251117191451.dist-info/entry_points.txt,sha256=rtVF0A2MMTYUsBScF6b3AlOuk2Vm02QK7Tc2bDcDpk0,200
-thds_adls-4.4.20251117191451.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_adls-4.4.20251117191451.dist-info/RECORD,,
+thds_adls-4.5.20260110021526.dist-info/METADATA,sha256=f95s20SMLUIvjdtM6b4Y56_UWxEFVrwNskJ8fkEHkWY,4586
+thds_adls-4.5.20260110021526.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_adls-4.5.20260110021526.dist-info/entry_points.txt,sha256=rtVF0A2MMTYUsBScF6b3AlOuk2Vm02QK7Tc2bDcDpk0,200
+thds_adls-4.5.20260110021526.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_adls-4.5.20260110021526.dist-info/RECORD,,