thds.adls 4.1.20250701001205__py3-none-any.whl → 4.1.20250701190349__py3-none-any.whl

thds/adls/hashes.py

@@ -0,0 +1,147 @@
+import contextlib
+import os
+import typing as ty
+from functools import partial
+
+import xxhash
+
+from thds.core import hash_cache, hashing, log, source, types
+from thds.core.hashing import Hash, SomehowReadable
+
+from . import errors, file_properties
+from .fqn import AdlsFqn
+
+logger = log.getLogger(__name__)
+
+PREFERRED_ALGOS: ty.Final = ("xxh3_128", "blake3")
+AnyStrSrc = ty.Union[SomehowReadable, ty.Iterable[ty.AnyStr]]
+# this type closely corresponds to what the underlying DataLakeStorageClient will accept for upload_data.
+
+
+def default_hasher() -> hashing.Hasher:
+    return xxhash.xxh3_128()
+
+
+def _xxhash_hasher(algo: str) -> hashing.Hasher:
+    return getattr(xxhash, algo)()
+
+
+def register_hashes():
+    for algo in xxhash.algorithms_available:
+        hashing.add_named_hash(algo, _xxhash_hasher)
+    source.set_file_autohash(PREFERRED_ALGOS[0], _xxhash_hasher)
+
+    try:
+        from blake3 import blake3
+
+        hashing.add_named_hash("blake3", lambda _: blake3())  # type: ignore
+    except ModuleNotFoundError:
+        pass
+
+
+def _hash_path_if_exists(
+    file_hasher: ty.Callable[[types.StrOrPath], hashing.Hash], path: types.StrOrPath
+) -> ty.Optional[hashing.Hash]:
+    if not path or not os.path.exists(path):  # does not exist if it's a symlink with a bad referent.
+        return None
+    return file_hasher(path)
+
+
+def hash_path_for_algo(
+    algo: str,
+) -> ty.Callable[[types.StrOrPath], ty.Optional[hashing.Hash]]:
+    """Return a function that hashes a path for the given algorithm."""
+    return partial(_hash_path_if_exists, partial(hash_cache.filehash, algo))
+
+
+def metadata_hash_b64_key(algo: str) -> str:
+    return f"hash_{algo}_b64"
+
+
+def extract_hashes_from_metadata(metadata: dict) -> ty.Iterable[hashing.Hash]:
+    # NOTE! the order here is critical, because we want to _prefer_ the faster hash if it exists.
+    for hash_algo in PREFERRED_ALGOS:
+        md_key = metadata_hash_b64_key(hash_algo)
+        if metadata and md_key in metadata:
+            yield hashing.Hash(hash_algo, hashing.db64(metadata[md_key]))
+
+
+def extract_hashes_from_props(
+    props: ty.Optional[file_properties.PropertiesP],
+) -> dict[str, hashing.Hash]:
+    if not props:
+        return dict()
+
+    hashes = list(extract_hashes_from_metadata(props.metadata or dict()))
+    if props.content_settings and props.content_settings.content_md5:
+        hashes.append(hashing.Hash("md5", props.content_settings.content_md5))
+    return {h.algo: h for h in hashes}
+
+
+@contextlib.contextmanager
+def verify_hashes_before_and_after_download(
+    remote_hash: ty.Optional[Hash],
+    expected_hash: ty.Optional[Hash],
+    fqn: AdlsFqn,
+    local_dest: types.StrOrPath,
+) -> ty.Iterator[None]:
+    # if expected_hash:
+    #     check_reasonable_md5b64(expected_md5b64)
+    # if remote_md5b64:
+    #     check_reasonable_md5b64(remote_md5b64)
+    if remote_hash and expected_hash and remote_hash != expected_hash:
+        raise errors.HashMismatchError(
+            f"ADLS thinks the {remote_hash.algo} of {fqn} is {hashing.b64(remote_hash.bytes)},"
+            f" but we expected {hashing.b64(expected_hash.bytes)}."
+            " This may indicate that we need to update a hash in the codebase."
+        )
+
+    yield  # perform download
+
+    expected_algo = expected_hash.algo if expected_hash else None
+    if not expected_algo and remote_hash:
+        expected_algo = remote_hash.algo
+
+    if not expected_algo:
+        # if we have neither a user-provided hash nor a remotely-foun9d hash, then we have nothing to check.
+        return
+
+    with log.logger_context(hash_for="after-download"):
+        local_hash = hash_cache.filehash(expected_algo, local_dest)
+
+    if remote_hash and remote_hash != local_hash:
+        raise errors.HashMismatchError(
+            f"The {local_hash.algo} of the downloaded file {local_dest} is {hashing.b64(local_hash.bytes)},"
+            f" but the remote ({fqn}) says it should be {hashing.b64(remote_hash.bytes)}."
+            f" This may indicate that ADLS has an erroneous {remote_hash.algo} for {fqn}."
+        )
+
+    if expected_hash and local_hash != expected_hash:
+        raise errors.HashMismatchError(
+            f"The {local_hash.algo} of the downloaded file {local_dest} is {hashing.b64(local_hash.bytes)},"
+            f" but we expected it to be {hashing.b64(expected_hash.bytes)}."
+            f" This probably indicates a corrupted download of {fqn}"
+        )
+
+    all_hashes = dict(local=local_hash, remote=remote_hash, expected=expected_hash)
+    real_hashes = list(filter(None, all_hashes.values()))
+    assert len(real_hashes) > 0, all_hashes
+    assert all(real_hashes[0] == h for h in real_hashes), all_hashes
+
+
+def metadata_hash_dict(hash: Hash) -> dict[str, str]:
+    return {metadata_hash_b64_key(hash.algo): hashing.b64(hash.bytes)}
+
+
+def create_hash_metadata_if_missing(
+    file_properties: ty.Optional[file_properties.FileProperties], new_hash: ty.Optional[Hash]
+) -> dict:
+    if not (file_properties and new_hash):
+        # without file properties, we can't match the etag when we try to set this.
+        return dict()
+
+    existing_metadata = file_properties.metadata or dict()
+    if metadata_hash_b64_key(new_hash.algo) not in existing_metadata:
+        return {**existing_metadata, **metadata_hash_dict(new_hash)}
+
+    return dict()

thds/adls/impl.py

@@ -31,7 +31,7 @@ from azure.storage.filedatalake.aio import DataLakeServiceClient, FileSystemClie
 
 from thds.core import lazy, log
 
-from ._upload import async_upload_decision_and_settings, metadata_for_upload
+from ._upload import async_upload_decision_and_metadata
 from .conf import CONNECTION_TIMEOUT, UPLOAD_CHUNK_SIZE
 from .download import async_download_or_use_verified
 from .errors import translate_azure_error
@@ -330,15 +330,14 @@ class ADLSFileSystem:
 
         async with file_system_client.get_file_client(remote_path) as file_client:
             with open(local_path, "rb") as fp:
-                decision = await async_upload_decision_and_settings(file_client.get_file_properties, fp)
+                decision = await async_upload_decision_and_metadata(file_client.get_file_properties, fp)
                 if decision.upload_required:
                     await file_client.upload_data(
                         fp,
                         overwrite=True,
-                        content_settings=decision.content_settings,
                         connection_timeout=CONNECTION_TIMEOUT(),
                         chunk_size=UPLOAD_CHUNK_SIZE(),
-                        metadata={**metadata_for_upload(), **(metadata or {})},
+                        metadata={**decision.metadata, **(metadata or {})},
                     )
 
         return remote_path

thds/adls/md5.py

@@ -1,60 +1,13 @@
-"""Why MD5 when it's no longer a good choice for most use cases?
-Because Azure/ADLS support Content-MD5 but nothing else, and I don't
-want to lie to them and get us confused later.
-
-Thankfully, there are no real security concerns for us with purely
-internal code and data sets.
-
-That said, please _do not_ use MD5 for non-Azure things. Prefer SHA256
-if at all possible.
-"""
 import hashlib
-import typing as ty
-from pathlib import Path
-
-from thds.core.hash_cache import hash_file
-from thds.core.hashing import SomehowReadable, hash_anything, hash_using
-from thds.core.types import StrOrPath
-
-AnyStrSrc = ty.Union[SomehowReadable, ty.Iterable[ty.AnyStr]]
-# this type closely corresponds to what the underlying DataLakeStorageClient will accept for upload_data.
-
-
-def md5_file(file: StrOrPath) -> bytes:
-    """Raise exception if it cannot be read.
 
-    Safely caches the hash on your local filesystem (uses mtimes to
-    determine staleness).
-    """
-    return hash_file(file, hashlib.md5())
+from thds.core.hashing import Hash, db64, hash_using
 
 
 def hex_md5_str(string: str) -> str:
     return hash_using(string.encode(), hashlib.md5()).hexdigest()
 
 
-def try_md5(data: AnyStrSrc) -> ty.Optional[bytes]:
-    """Ideally, we calculate an MD5 sum for all data that we upload.
-
-    The only circumstances under which we cannot do this are if the
-    stream does not exist in its entirety before the upload begins.
-    """
-    if isinstance(data, Path):
-        return md5_file(data)
-    res = hash_anything(data, hashlib.md5())
-    if res:
-        return res.digest()
-    return None
-
-
-def is_reasonable_b64(md5: str):
-    if len(md5) == 22:
-        return True
-    if len(md5) == 24 and md5.endswith("=="):
-        return True
-    return False
-
-
-def check_reasonable_md5b64(maybe_md5: str):
-    if not is_reasonable_b64(maybe_md5):
-        raise ValueError(f"MD5 '{maybe_md5}' is not a reasonable MD5.")
+def to_hash(md5b64: str) -> Hash:
+    """Convert a base64-encoded MD5 hash to a hex string."""
+    assert md5b64, "MD5 base64 string cannot be empty"
+    return Hash(algo="md5", bytes=db64(md5b64))

thds/adls/ro_cache.py

@@ -12,7 +12,7 @@ from thds.core.link import LinkType, link_or_copy
 from .fqn import AdlsFqn
 from .md5 import hex_md5_str
 
-GLOBAL_CACHE_PATH = config.item("global-cache-path", HOMEDIR() / ".adls-md5-ro-cache", parse=Path)
+GLOBAL_CACHE_PATH = config.item("global-cache-path", HOMEDIR() / ".thds/adls/ro-cache", parse=Path)
 MAX_FILENAME_LEN = config.item("max-filename-len", 255, parse=int)  # safe on most local filesystems?
 MAX_TOTAL_PATH_LEN = config.item(
     "max-total-path-len", 1023 if sys.platform == "darwin" else 4095, parse=int
@@ -39,7 +39,6 @@ class Cache(ty.NamedTuple):
 
 
 def global_cache(link: LinkOpts = ("ref", "hard")) -> Cache:
-    """This is the recommended caching configuration."""
     return Cache(GLOBAL_CACHE_PATH(), link)
 
 

thds/adls/source.py

@@ -1,14 +1,14 @@
-import base64
 import typing as ty
 from functools import partial
 from pathlib import Path
 
 from thds.core import source
-from thds.core.hashing import Hash, b64
+from thds.core.hashing import Hash
 
-from .cached_up_down import download_to_cache
+from . import cached, hashes, md5
+from .errors import blob_not_found_translation
+from .file_properties import get_file_properties
 from .fqn import AdlsFqn
-from .resource import AdlsHashedResource
 from .uri import resolve_any, resolve_uri
 
 
@@ -19,17 +19,10 @@ def _adls_uri_source_download_handler(uri: str) -> ty.Optional[source.Downloader
 
     def download(hash: ty.Optional[Hash]) -> Path:
         assert fqn
-        if hash and hash.algo == "md5":
-            # this 'extra' check just allows us to short-circuit a download
-            # where the hash at this URI is known not to match what we expect.
-            # It's no safer than the non-md5 hash check that Source performs after download.
-            return download_to_cache(fqn, b64(hash.bytes))
-
-        # we don't validate this hash, because we already have md5 validation
-        # happening inside the download_to_cache function. the Source hash
-        # is actually mostly for use by systems that want to do content addressing,
-        # and not necessarily intended to be a runtime check in all scenarios.
-        return download_to_cache(fqn)
+        # this 'extra' check just allows us to short-circuit a download
+        # where the hash at this URI is known not to match what we expect.
+        # It's no safer than the non-md5 hash check that Source performs after download.
+        return cached.download_to_cache(fqn, expected_hash=hash)
 
     return download
 
@@ -37,30 +30,40 @@ def _adls_uri_source_download_handler(uri: str) -> ty.Optional[source.Downloader
 source.register_download_handler("thds.adls", _adls_uri_source_download_handler)
 
 
-def from_adls(
-    uri_or_fqn_or_ahr: ty.Union[str, AdlsFqn, AdlsHashedResource], hash: ty.Optional[Hash] = None
-) -> source.Source:
+def from_adls(uri_or_fqn: ty.Union[str, AdlsFqn], hash: ty.Optional[Hash] = None) -> source.Source:
     """Flexible, public interface to creating Sources from any ADLS-like reference.
 
-    Does NOT automatically fetch an MD5 hash from the ADLS URI if it's not provided. If
-    you know you want to include that, combine this with `resource.get`:
-    `source.from_adls(resource.get(uri))`
+    Does NOT automatically fetch a checksumming hash from the ADLS URI if it's not
+    provided. If you know you want to include that, instead call:
+    `source.get_with_hash(uri_or_fqn)`.
     """
-    if isinstance(uri_or_fqn_or_ahr, AdlsHashedResource):
-        fqn = uri_or_fqn_or_ahr.fqn
-        res_hash = Hash("md5", base64.b64decode(uri_or_fqn_or_ahr.md5b64))
-        if hash and hash != res_hash:
-            raise ValueError(f"Resource Hash mismatch for {fqn}: {hash} != {res_hash}")
-        hash = res_hash
-    else:
-        r_fqn = resolve_any(uri_or_fqn_or_ahr)
-        if not r_fqn:
-            raise ValueError(f"Could not resolve {uri_or_fqn_or_ahr} to an ADLS FQN")
-        fqn = r_fqn
-
-    return source.Source(str(fqn), hash)
+    r_fqn = resolve_any(uri_or_fqn)
+    if not r_fqn:
+        raise ValueError(f"Could not resolve {uri_or_fqn} to an ADLS FQN")
+    return source.Source(str(r_fqn), hash)
 
 
 source.register_from_uri_handler(
     "thds.adls", lambda uri: partial(from_adls, uri) if resolve_uri(uri) else None
 )
+
+
+def get_with_hash(fqn_or_uri: ty.Union[AdlsFqn, str]) -> source.Source:
+    """Creates a Source from a remote-only file, with MD5 or other hash.
+
+    The file _must_ have a pre-existing hash!
+    """
+    fqn = AdlsFqn.parse(fqn_or_uri) if isinstance(fqn_or_uri, str) else fqn_or_uri
+    with blob_not_found_translation(fqn):
+        uri_hashes = hashes.extract_hashes_from_props(get_file_properties(fqn))
+        if not uri_hashes:
+            raise ValueError(
+                f"ADLS file {fqn} must have a hash to use this function. "
+                "If you know the hash, use `from_adls` with the hash parameter."
+            )
+        return from_adls(fqn, next(iter(uri_hashes.values())))
+
+
+def with_md5b64(uri_or_fqn: ty.Union[str, AdlsFqn], *, md5b64: str = "") -> source.Source:
+    """Meant for older use cases where we had an MD5"""
+    return from_adls(uri_or_fqn, md5.to_hash(md5b64) if md5b64 else None)

thds/adls/tools/download.py

@@ -1,7 +1,7 @@
 import argparse
 from pathlib import Path
 
-from thds.adls.cached_up_down import download_directory, download_to_cache
+from thds.adls import cached
 from thds.adls.file_properties import get_file_properties, is_directory
 from thds.adls.impl import ADLSFileSystem
 from thds.adls.uri import resolve_uri
@@ -39,9 +39,9 @@ def main():
             cache_path = fs.fetch_file(args.adls_fqn.path)
     else:
         if is_dir:
-            cache_path = download_directory(args.adls_fqn)
+            cache_path = cached.download_directory(args.adls_fqn)
         else:
-            cache_path = download_to_cache(args.adls_fqn)
+            cache_path = cached.download_to_cache(args.adls_fqn)
 
     if args.copy_to:
         link(cache_path, args.copy_to)

thds/adls/tools/upload.py

@@ -1,17 +1,16 @@
 import argparse
 from pathlib import Path
 
-from thds.adls.cached_up_down import upload_through_cache
-from thds.adls.uri import resolve_uri
+from thds.adls import cached, uri
 
 
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("path", type=Path, help="A local file you want to upload.")
-    parser.add_argument("uri", type=resolve_uri, help="A fully qualified path to an ADLS location")
+    parser.add_argument("uri", type=uri.resolve_uri, help="A fully qualified path to an ADLS location")
     args = parser.parse_args()
 
-    upload_through_cache(args.uri, args.path)
+    cached.upload_through_cache(args.uri, args.path)
 
 
 if __name__ == "__main__":

thds/adls/upload.py

@@ -0,0 +1,162 @@
+"""API for uploading files to Azure Data Lake Storage (ADLS) Gen2.
+
+We hash anything that we possibly can, since it's a fast verification step that we
+can do later during downloads.
+"""
+
+import subprocess
+import typing as ty
+from pathlib import Path
+
+from azure.core.exceptions import ResourceModifiedError
+from azure.storage.blob import ContentSettings
+
+from thds.core import files, fretry, link, log, scope, source, tmp
+
+from . import azcopy, hashes
+from ._progress import report_upload_progress
+from ._upload import upload_decision_and_metadata
+from .conf import UPLOAD_FILE_MAX_CONCURRENCY
+from .fqn import AdlsFqn
+from .global_client import get_global_blob_container_client
+from .ro_cache import Cache
+
+logger = log.getLogger(__name__)
+_SLOW_CONNECTION_WORKAROUND = 14400  # seconds
+
+
+UploadSrc = ty.Union[Path, bytes, ty.IO[ty.AnyStr], ty.Iterable[bytes]]
+
+
+def _write_through_local_cache(local_cache_path: Path, data: UploadSrc) -> ty.Optional[Path]:
+    @scope.bound
+    def _try_write_through() -> bool:
+        if isinstance(data, Path) and data.exists():
+            link.link_or_copy(data, local_cache_path, "ref")
+            return True
+        out = scope.enter(tmp.temppath_same_fs(local_cache_path))
+        if hasattr(data, "read") and hasattr(data, "seek"):
+            with open(out, "wb") as f:
+                f.write(data.read())  # type: ignore
+            data.seek(0)  # type: ignore
+            link.link_or_copy(out, local_cache_path)
+            return True
+        if isinstance(data, bytes):
+            with open(out, "wb") as f:
+                f.write(data)
+            link.link_or_copy(out, local_cache_path)
+            return True
+        return False
+
+    if _try_write_through():
+        try:
+            # it's a reflink or a copy, so the cache now owns its copy
+            # and we don't want to allow anyone to write to its copy.
+            files.set_read_only(local_cache_path)
+            return local_cache_path
+        except FileNotFoundError:
+            # may have hit a race condition.
+            # don't fail upload just because we couldn't write through the cache.
+            pass
+    return None
+
+
+@scope.bound
+@fretry.retry_sleep(
+    # ADLS lib has a bug where parallel uploads of the same thing will
+    # hit a race condition and error.  this will detect that scenario
+    # and avoid re-uploading as well.
+    fretry.is_exc(ResourceModifiedError),
+    fretry.expo(retries=5),
+)
+def upload(
+    dest: ty.Union[AdlsFqn, str],
+    src: UploadSrc,
+    write_through_cache: ty.Optional[Cache] = None,
+    *,
+    content_type: str = "",
+    **upload_data_kwargs: ty.Any,
+) -> source.Source:
+    """Uploads only if the remote does not exist or does not match
+    xxhash.
+
+    Always embeds xxhash in the blob metadata if at all possible. In very rare cases
+    it may not be possible for us to calculate one. Will always be possible if the passed
+    data was a Path. If one can be calculated, it will be returned in the Source.
+
+    Can write through a local cache, which may save you a download later.
+
+    content_type and all upload_data_kwargs will be ignored if the file
+    has already been uploaded and the hash matches.
+    """
+    dest_ = AdlsFqn.parse(dest) if isinstance(dest, str) else dest
+    if write_through_cache:
+        _write_through_local_cache(write_through_cache.path(dest_), src)
+        # we always use the original source file to upload, not the cached path,
+        # because uploading from a shared location risks race conditions.
+
+    blob_container_client = get_global_blob_container_client(dest_.sa, dest_.container)
+    blob_client = blob_container_client.get_blob_client(dest_.path)
+    decision = upload_decision_and_metadata(blob_client.get_blob_properties, src)  # type: ignore [arg-type]
+
+    def source_from_meta() -> source.Source:
+        best_hash = next(iter(hashes.extract_hashes_from_metadata(decision.metadata)), None)
+        if isinstance(src, Path):
+            assert best_hash, "A hash should always be calculable for a local path."
+            return source.from_file(src, hash=best_hash, uri=str(dest_))
+
+        return source.from_uri(str(dest_), hash=best_hash)
+
+    if decision.upload_required:
+        # set up some bookkeeping
+        n_bytes = None  # if we pass 0 to upload_blob, it truncates the write now
+        bytes_src: ty.Union[bytes, ty.IO, ty.Iterable[bytes]]
+        if isinstance(src, Path):
+            n_bytes = src.stat().st_size
+            bytes_src = scope.enter(open(src, "rb"))
+        elif isinstance(src, bytes):
+            n_bytes = len(src)
+            bytes_src = src
+        else:
+            bytes_src = src
+
+        if "metadata" in upload_data_kwargs:
+            decision.metadata.update(upload_data_kwargs.pop("metadata"))
+
+        if azcopy.upload.should_use_azcopy(n_bytes or 0) and isinstance(src, Path):
+            logger.info("Using azcopy to upload %s to %s", src, dest_)
+            try:
+                azcopy.upload.run(
+                    azcopy.upload.build_azcopy_upload_command(
+                        src, dest_, content_type=content_type, metadata=decision.metadata, overwrite=True
+                    ),
+                    dest_,
+                    n_bytes or 0,
+                )
+                return source_from_meta()
+
+            except subprocess.SubprocessError:
+                logger.warning("Azcopy upload failed, falling back to SDK upload")
+
+        upload_content_settings = ContentSettings()
+        if content_type:
+            upload_content_settings.content_type = content_type
+
+        # we are now using blob_client instead of file system client
+        # because blob client (as of 2024-06-24) does actually do
+        # some one-step, atomic uploads, wherein there is not a separate
+        # create/truncate action associated with an overwrite.
+        # This is both faster, as well as simpler to reason about, and
+        # in fact was the behavior I had been assuming all along...
+        blob_client.upload_blob(
+            report_upload_progress(ty.cast(ty.IO, bytes_src), str(dest_), n_bytes or 0),
+            overwrite=True,
+            length=n_bytes,
+            content_settings=upload_content_settings,
+            connection_timeout=_SLOW_CONNECTION_WORKAROUND,
+            max_concurrency=UPLOAD_FILE_MAX_CONCURRENCY(),
+            metadata=decision.metadata,
+            **upload_data_kwargs,
+        )
+
+    return source_from_meta()

thds/adls/uri.py

@@ -36,3 +36,9 @@ def resolve_any(fqn_or_uri: UriIsh) -> ty.Optional[fqn.AdlsFqn]:
 
 def parse_any(fqn_or_uri: UriIsh) -> fqn.AdlsFqn:
     return parse_uri(fqn_or_uri) if isinstance(fqn_or_uri, str) else fqn_or_uri
+
+
+def to_blob_windows_url(uri: UriIsh) -> str:
+    """Convert an ADLS URI to a Windows network path."""
+    fqn = parse_any(uri)
+    return f"https://{fqn.sa}.blob.core.windows.net/{fqn.container}/{fqn.path}"

{thds_adls-4.1.20250701001205.dist-info → thds_adls-4.1.20250701190349.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.adls
-Version: 4.1.20250701001205
+Version: 4.1.20250701190349
 Summary: ADLS tools
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

thds_adls-4.1.20250701190349.dist-info/RECORD

@@ -0,0 +1,42 @@
+thds/adls/__init__.py,sha256=g2Zb0EAAH-JzPMYHAub9liU4qa5pfqQDnILfEhmObGo,1036
+thds/adls/_progress.py,sha256=ZzCHn_G7nHakioNFxdvoJZRr-jN6ymsp5JXf-iReROM,6580
+thds/adls/_upload.py,sha256=mhTdWiQroaugYuwQg7R8CEgdfCYF4xvJthlsqO0jlnE,4692
+thds/adls/abfss.py,sha256=ZRJOLjDuXmS4bIbQAQpQxWWWeu74N9NKEKCNfXQek80,726
+thds/adls/cached.py,sha256=up1F5JOVXdmwdZ8RAB2UDgiy6ooLg8IMULohBh75VpQ,3034
+thds/adls/conf.py,sha256=nTw3X1ilC3A_905jZH-rWXFsESeHAKQn5IghvfX2VIo,1991
+thds/adls/copy.py,sha256=jUWbGvTpb4B3yRGS0nhGSbDzqRPzUqYgH0z1lFRJB3k,6365
+thds/adls/dbfs.py,sha256=pPAjbIZRKJsaXKQljDMUgqS_zy1yKeEZHGMueXbuv3g,2219
+thds/adls/defaults.py,sha256=GGq5Pn4r-8cX4bZItp4nnwWAAz7S07pzPoOegw0y5Fw,676
+thds/adls/download.py,sha256=N8JqNqD5ioHsEHcTl2bNJt3Bb187yyvZAXn4xW3flfU,18090
+thds/adls/download_lock.py,sha256=tgT48l4C5_qmArGeq05gl7VlxT22dZBH2Xwxx0itE9o,3176
+thds/adls/errors.py,sha256=6cLg2E4SB8ic46PBzA3ynRH4b1oR8qRb07RBgKGJRxY,1783
+thds/adls/etag.py,sha256=ct7jpHhNFcKzbekn5rZ3m6DhjK48A7qOZGwDiHkc-pc,242
+thds/adls/file_properties.py,sha256=C9Kl3a5wuBNWYgZYnZbkH04u8uxadEcjVJIm3UevUM0,1912
+thds/adls/fqn.py,sha256=0zHmHhBWN7GEfKRB3fBC1NVhaiIHHifBdCRanyT01X8,5822
+thds/adls/global_client.py,sha256=f4VJw5y_Yh__8gQUcdSYTh1aU6iEPlauMchVirSAwDQ,3716
+thds/adls/hashes.py,sha256=RDQS-C38wskUhxXGFGLJ4ox8vm7ofurxSsUk13Ywijo,5309
+thds/adls/impl.py,sha256=4rZAGlhU_UojPy1FC7j3KEFIj6BWSbCDAVV1FCyki3s,42496
+thds/adls/md5.py,sha256=hGT8AIX32VUsnRCbm8cel9OlxAiRrgjwNWQTqRDHM_k,374
+thds/adls/named_roots.py,sha256=7SLbAoQQpV_mrFZaUPjYoS-F9dxQxN5Hg4M3YPirF_w,751
+thds/adls/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+thds/adls/ro_cache.py,sha256=P-UVFZhqnE5wojqYmRVWZcqjl-pM1xVMm9VAm3nXlnA,4769
+thds/adls/sas_tokens.py,sha256=mArbB_GYohevOmArw_1gKqVUWpv6kG8Hsbvdrhbtnbg,1957
+thds/adls/shared_credential.py,sha256=-x42aXoIM001KW59oS8PpuXQd4-F2vg-1gB6OMHlpk4,4602
+thds/adls/source.py,sha256=8HVMYuxDn1XYGwFFSBowMlvQ6r2Jm2CQlpu4h85JvsE,2559
+thds/adls/source_tree.py,sha256=yP_v2XrKxXqUOdZ-x8kqHhBFAuur3AlAq3zi4hHj4AE,2235
+thds/adls/upload.py,sha256=gS_S66gorzdW83eavPUVJ3UYrv5u3HnftDXjdwEZOo8,6441
+thds/adls/uri.py,sha256=9MXuW_KfpPvzBc4ERxuTJ3vvi_6yr7e1kMAW9mx2zXM,1414
+thds/adls/azcopy/__init__.py,sha256=qn2dmT92EHcrtaQ8uwRoUgvtF6Fu3NQbhZItOBdIBmY,45
+thds/adls/azcopy/download.py,sha256=J7QAoBehpxsY58ofgGQur-MtIwM0NEnV9_Cw4i_X3y8,6007
+thds/adls/azcopy/login.py,sha256=923UaewVMPFzkDSgCQsbl-_g7qdFhpXpF0MGNIy3T_A,1538
+thds/adls/azcopy/progress.py,sha256=K7TVmSiWfu561orL3GuOnlQX9VtVxWVECAq9NiweYNo,1387
+thds/adls/azcopy/system_resources.py,sha256=okgDEKAp0oWGQF7OKikbgJ9buBeiOgNaDYy-36j6dHo,761
+thds/adls/azcopy/upload.py,sha256=bvtYdbaFsZkOHFLDpeBlTKqw63P3_kbImInI04ZlekM,2601
+thds/adls/tools/download.py,sha256=Dmt-EBZUEF-gVfUcwjAD8VRKR5rhw-oozxl40lZHmdw,1562
+thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
+thds/adls/tools/upload.py,sha256=5WyWkpuVp2PETZ3O3ODlq8LXszSHU73ZMnIDZXPJdC8,442
+thds_adls-4.1.20250701190349.dist-info/METADATA,sha256=gJNup1vZrpFp-0nor96kwmz__Ij_Zc5pkytWoEslYMU,587
+thds_adls-4.1.20250701190349.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_adls-4.1.20250701190349.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
+thds_adls-4.1.20250701190349.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_adls-4.1.20250701190349.dist-info/RECORD,,

thds/adls/resource/__init__.py

@@ -1,36 +0,0 @@
-"""The reason for a hashed resource is that it enables worry-free caching.
-
-If under any circumstances we re-use a name/URI with different bytes,
-then having captured a hash will enable us to transparently detect the
-situation and re-download.
-
-It is strongly recommended that you construct these using `of`, as
-that will avoid the accidental, invalid creation of an
-AdlsHashedResource containing an empty hash.
-
-How to get the hash itself?
-
-From our experience, it seems that any file uploaded using Azure
-Storage Explorer will have an MD5 calculated locally before upload and
-that will be embedded in the remote file. You can look in the
-properties of the uploaded file for Content-MD5 and copy-paste that
-into whatever you're writing.
-
-Programmatically, you can instead use `resource.upload`, which will
-return to you an in-memory AdlsHashedResource object. If you want to
-store it programmatically rather than in the source code, it's
-recommended that you use `resource.to_path`, and then load it using
-`resource.from_path`.
-
-Prefer importing this module `as resource` or `from thds.adls
-import resource`, and then using it as a namespace,
-e.g. `resource.of(uri)`.
-"""
-from .core import AdlsHashedResource, from_source, get, of, parse, serialize, to_source  # noqa: F401
-from .file_pointers import resource_from_path as from_path  # noqa: F401
-from .file_pointers import resource_to_path as to_path  # noqa: F401
-from .file_pointers import validate_resource as validate  # noqa: F401
-from .up_down import get_read_only, upload  # noqa: F401
-from .up_down import verify_or_create_resource as verify_or_create  # noqa: F401
-
-AHR = AdlsHashedResource  # just an alias