thds.adls 4.1.20250701001205__py3-none-any.whl → 4.1.20250701190349__py3-none-any.whl

thds/adls/resource/core.py

@@ -1,77 +0,0 @@
-import json
-import typing as ty
-
-from thds.core import hashing, log, source
-from thds.core.hashing import b64
-
-from ..errors import blob_not_found_translation
-from ..file_properties import get_file_properties
-from ..fqn import AdlsFqn
-from ..md5 import check_reasonable_md5b64
-
-logger = log.getLogger(__name__)
-
-
-class AdlsHashedResource(ty.NamedTuple):
-    """See the containing package for documentation on how to use this and its motivation."""
-
-    fqn: AdlsFqn
-    md5b64: str
-
-    @property
-    def serialized(self) -> str:
-        return serialize(self)
-
-    @staticmethod
-    def of(fqn_or_uri: ty.Union[AdlsFqn, str], md5b64: str) -> "AdlsHashedResource":
-        return of(fqn_or_uri, md5b64)
-
-    @staticmethod
-    def parse(serialized_dict: str) -> "AdlsHashedResource":
-        return parse(serialized_dict)
-
-
-def of(fqn_or_uri: ty.Union[AdlsFqn, str], md5b64: str) -> AdlsHashedResource:
-    assert md5b64, "md5b64 must be non-empty"
-    fqn = AdlsFqn.parse(fqn_or_uri) if isinstance(fqn_or_uri, str) else fqn_or_uri
-    return AdlsHashedResource(fqn, md5b64)
-
-
-def from_source(source: source.Source) -> AdlsHashedResource:
-    assert source.hash, "Source must have a hash"
-    assert source.hash.algo == "md5", f"Source Hash type must be MD5! Got: {source.hash.algo}"
-    return of(source.uri, hashing.b64(source.hash.bytes))
-
-
-def to_source(resource: AdlsHashedResource) -> source.Source:
-    return source.from_uri(
-        str(resource.fqn),
-        hash=hashing.Hash("md5", hashing.db64(resource.md5b64)),
-    )
-
-
-def serialize(resource: AdlsHashedResource) -> str:
-    d = resource._asdict()
-    # we use uri instead of fqn in order to make these a more generic format
-    return json.dumps(dict(uri=str(d["fqn"]), md5b64=d["md5b64"]))
-
-
-def parse(serialized_dict: str) -> AdlsHashedResource:
-    actual_dict = json.loads(serialized_dict)
-    # accept either uri or fqn
-    uri = actual_dict["uri"] if "uri" in actual_dict else actual_dict["fqn"]
-    md5b64 = actual_dict["md5b64"]
-    check_reasonable_md5b64(md5b64)
-    return AdlsHashedResource.of(AdlsFqn.parse(uri), md5b64)
-
-
-def get(fqn_or_uri: ty.Union[AdlsFqn, str]) -> AdlsHashedResource:
-    """Creates an AdlsHashedResource from a remote-only file.
-
-    The file _must_ have a pre-existing Content MD5!
-    """
-    fqn = AdlsFqn.parse(fqn_or_uri) if isinstance(fqn_or_uri, str) else fqn_or_uri
-    with blob_not_found_translation(fqn):
-        props = get_file_properties(fqn)
-        assert props.content_settings.content_md5, "ADLS file has empty Content-MD5!"
-        return AdlsHashedResource.of(fqn, b64(props.content_settings.content_md5))

thds/adls/resource/file_pointers.py

@@ -1,54 +0,0 @@
-import os
-import typing as ty
-from pathlib import Path
-
-from thds.core.hashing import b64
-
-from ..file_properties import get_file_properties
-from .core import AdlsHashedResource, parse, serialize
-
-_AZURE_PLACEHOLDER_SIZE_LIMIT = 4096
-# it is assumed that no placeholder will ever need to be larger than 4 KB.
-
-
-def resource_from_path(path: ty.Union[str, Path]) -> AdlsHashedResource:
-    """Raises if the path does not represent a serialized AdlsHashedResource."""
-    with open(path) as maybe_json_file:
-        json_str = maybe_json_file.read(_AZURE_PLACEHOLDER_SIZE_LIMIT)
-        return parse(json_str)
-
-
-class MustCommitResourceLocally(Exception):
-    """Means you need to read the message and commit the change to the repo."""
-
-
-def _check_ci(resource_json_path: ty.Union[str, Path], resource: AdlsHashedResource):
-    if os.getenv("CI"):
-        print("\n" + resource.serialized + "\n")
-        # Because we can't modify+commit in CI, this should fail
-        # if it's recreating a resource in a CI environment.
-        raise MustCommitResourceLocally(
-            "In CI, a newly built resource cannot be recorded."
-            " However, it did successfully build and upload!"
-            f" You should paste '{resource.serialized}'"
-            f" into '{resource_json_path}' locally, and commit."
-        )
-
-
-def resource_to_path(
-    path: ty.Union[str, Path], resource: AdlsHashedResource, *, check_ci: bool = False
-) -> None:
-    if check_ci:
-        _check_ci(path, resource)
-    with open(path, "w") as json_file:
-        json_file.write(serialize(resource) + "\n")
-
-
-def validate_resource(srcfile: ty.Union[str, Path]) -> AdlsHashedResource:
-    res = resource_from_path(srcfile)
-    fqn, md5b64 = res
-    props = get_file_properties(fqn)
-    md5 = props.content_settings.content_md5
-    assert md5, f"{fqn} was incorrectly uploaded to ADLS without an MD5 embedded."
-    assert md5b64 == b64(md5), f"You probably need to update the MD5 in {srcfile}"
-    return res

thds/adls/resource/up_down.py

@@ -1,242 +0,0 @@
-"""Tools for using and creating locally-cached resources."""
-
-import typing as ty
-from pathlib import Path
-
-from azure.core.exceptions import HttpResponseError, ResourceModifiedError
-from azure.storage.blob import ContentSettings
-
-from thds.core import files, fretry, hashing, link, log, scope, tmp
-
-from .._progress import report_upload_progress
-from .._upload import metadata_for_upload, upload_decision_and_settings
-from ..conf import UPLOAD_FILE_MAX_CONCURRENCY
-from ..download import download_or_use_verified
-from ..errors import BlobNotFoundError
-from ..file_properties import get_file_properties
-from ..fqn import AdlsFqn
-from ..global_client import get_global_blob_container_client, get_global_fs_client
-from ..ro_cache import Cache, global_cache
-from .file_pointers import AdlsHashedResource, resource_from_path, resource_to_path
-
-logger = log.getLogger(__name__)
-_SLOW_CONNECTION_WORKAROUND = 14400  # seconds
-
-
-# DOWNLOAD
-def get_read_only(
-    resource: AdlsHashedResource,
-    local_path: ty.Optional[Path] = None,
-    cache: Cache = global_cache(),
-) -> Path:
-    """Downloads a read-only resource if it is not already present in
-    the cache or at the local_path.
-
-    Because the resource includes a hash, we can save a lot of
-    bandwidth if we can detect that it already is present locally.
-
-    By default, downloads through the machine-global cache. Caching
-    cannot be disabled, but the location of the cache can be changed.
-    """
-    cache = cache or global_cache()
-    local_path = local_path or cache.path(resource.fqn)
-    download_or_use_verified(
-        get_global_fs_client(resource.fqn.sa, resource.fqn.container),
-        resource.fqn.path,
-        local_path,
-        md5b64=resource.md5b64,
-        cache=cache,
-    )
-    return local_path
-
-
-# UPLOAD
-UploadSrc = ty.Union[Path, bytes, ty.IO[ty.AnyStr], ty.Iterable[bytes]]
-
-
-def _write_through_local_cache(local_cache_path: Path, data: UploadSrc) -> ty.Optional[Path]:
-    @scope.bound
-    def _try_write_through() -> bool:
-        if isinstance(data, Path) and data.exists():
-            link.link_or_copy(data, local_cache_path, "ref")
-            return True
-        out = scope.enter(tmp.temppath_same_fs(local_cache_path))
-        if hasattr(data, "read") and hasattr(data, "seek"):
-            with open(out, "wb") as f:
-                f.write(data.read())  # type: ignore
-            data.seek(0)  # type: ignore
-            link.link_or_copy(out, local_cache_path)
-            return True
-        if isinstance(data, bytes):
-            with open(out, "wb") as f:
-                f.write(data)
-            link.link_or_copy(out, local_cache_path)
-            return True
-        return False
-
-    if _try_write_through():
-        try:
-            # it's a reflink or a copy, so the cache now owns its copy
-            # and we don't want to allow anyone to write to its copy.
-            files.set_read_only(local_cache_path)
-            return local_cache_path
-        except FileNotFoundError:
-            # may have hit a race condition.
-            # don't fail upload just because we couldn't write through the cache.
-            pass
-    return None
-
-
-@scope.bound
-@fretry.retry_sleep(
-    # ADLS lib has a bug where parallel uploads of the same thing will
-    # hit a race condition and error.  this will detect that scenario
-    # and avoid re-uploading as well.
-    fretry.is_exc(ResourceModifiedError),
-    fretry.expo(retries=5),
-)
-def upload(
-    dest: ty.Union[AdlsFqn, str],
-    src: UploadSrc,
-    write_through_cache: ty.Optional[Cache] = None,
-    *,
-    content_type: str = "",
-    **upload_data_kwargs: ty.Any,
-) -> ty.Optional[AdlsHashedResource]:
-    """Uploads only if the remote does not exist or does not match
-    md5.
-
-    Always embeds md5 in upload if at all possible. In very rare cases
-    it may not be possible for us to calculate one. Will always
-    be possible if the passed data was a Path. If one can be
-    calculated, an AdlsHashedResource is returned.
-
-    Can write through a local cache, which may save you a download later.
-
-    content_type and all upload_data_kwargs will be ignored if the file
-    has already been uploaded and the md5 matches.
-    """
-    dest_ = AdlsFqn.parse(dest) if isinstance(dest, str) else dest
-    if write_through_cache:
-        _write_through_local_cache(write_through_cache.path(dest_), src)
-        # we always use the original source file to upload, not the cached path,
-        # because uploading from a shared location risks race conditions.
-
-    blob_container_client = get_global_blob_container_client(dest_.sa, dest_.container)
-    blob_client = blob_container_client.get_blob_client(dest_.path)
-    decision = upload_decision_and_settings(blob_client.get_blob_properties, src)  # type: ignore [arg-type]
-    if decision.upload_required:
-        # set up some bookkeeping
-        n_bytes = None  # if we pass 0 to upload_blob, it truncates the write now
-        if isinstance(src, Path):
-            n_bytes = src.stat().st_size
-            src = scope.enter(open(src, "rb"))
-        elif isinstance(src, bytes):
-            n_bytes = len(src)
-
-        adls_meta = metadata_for_upload()
-        if "metadata" in upload_data_kwargs:
-            adls_meta.update(upload_data_kwargs.pop("metadata"))
-
-        upload_content_settings = decision.content_settings or ContentSettings()
-        if content_type:
-            upload_content_settings.content_type = content_type
-
-        # we are now using blob_client instead of file system client
-        # because blob client (as of 2024-06-24) does actually do
-        # some one-step, atomic uploads, wherein there is not a separate
-        # create/truncate action associated with an overwrite.
-        # This is both faster, as well as simpler to reason about, and
-        # in fact was the behavior I had been assuming all along...
-        blob_client.upload_blob(
-            report_upload_progress(ty.cast(ty.IO, src), str(dest_), n_bytes or 0),
-            overwrite=True,
-            length=n_bytes,
-            content_settings=upload_content_settings,
-            connection_timeout=_SLOW_CONNECTION_WORKAROUND,
-            max_concurrency=UPLOAD_FILE_MAX_CONCURRENCY(),
-            metadata=adls_meta,
-            **upload_data_kwargs,
-        )
-
-    # if at all possible (if the md5 is known), return a resource containing it.
-    if decision.content_settings and decision.content_settings.content_md5:
-        return AdlsHashedResource.of(dest_, hashing.b64(decision.content_settings.content_md5))
-    return None
-
-
-def verify_remote_md5(resource: AdlsHashedResource) -> bool:
-    try:
-        props = get_file_properties(resource.fqn)
-        if props.content_settings.content_md5:
-            return hashing.b64(props.content_settings.content_md5) == resource.md5b64
-    except HttpResponseError:
-        return False
-    except Exception:
-        logger.exception("Unable to verify remote md5")
-    return False
-
-
-# DOWNLOAD if exists, else CREATE and UPLOAD
-
-
-def verify_or_create_resource(
-    resource_json_path: Path,
-    get_adls_fqn: ty.Callable[[], AdlsFqn],
-    creator: ty.Callable[[], Path],
-    cache: ty.Optional[Cache] = global_cache(),
-) -> AdlsHashedResource:
-    """Return an MD5-verified resource if it already exists and
-    matches the FQN _and_ the MD5 embedded in the resource JSON file.
-
-    Does not download the actual resource if it can be verified to
-    exist and match what is expected.
-
-    If if does not exist or does not match, creates the resource and
-    uploads it to the requested path as well.
-
-    Basically an idempotent get-or-create pattern, applied to things
-    that are more expensive to build than to upload, and that result
-    in a single file.
-
-    For this to work correctly, you will need the FQN itself to change
-    based on some kind of key that can be considered unique to a
-    particular version of the resource. Think of it like a cache
-    key. For instance, if you have a resource that gets changed every
-    time your library gets built, then your library version could be
-    part of the Adls FQN.
-
-    If running in CI, will create and upload your resource from CI,
-    but will then raise an exception, since you need to commit the
-    serialized resource to the resource_json_path.
-    """
-    remote_fqn = get_adls_fqn()  # this is lazy to allow partial application at a module level.
-    if resource_json_path.exists():
-        try:
-            resource = resource_from_path(resource_json_path)
-            try:
-                if resource.fqn == remote_fqn:
-                    if verify_remote_md5(resource):
-                        return resource
-                    else:
-                        logger.info("Resource MD5 does not match; must recreate.")
-                logger.info("Resource FQN does not match - it needs to be recreated.")
-            except BlobNotFoundError:
-                logger.info(f"Resource does not exist at {resource.fqn}; will create.")
-            except Exception:
-                logger.exception(f"Failed to get resource from {resource.fqn}, will recreate.")
-        except Exception:
-            logger.exception(f"Unable to parse a resource from {resource_json_path}; must recreate.")
-
-    logger.info(f"Creating resource for {remote_fqn}...")
-    created_path = creator()
-    sz_mb = created_path.stat().st_size / 2**20  # 1 MB
-    logger.info(
-        f"Uploading created resource of size {sz_mb:.1f} MB to {remote_fqn} from {created_path} ..."
-    )
-    uploaded_resource = upload(remote_fqn, created_path, write_through_cache=cache)
-    assert (
-        uploaded_resource
-    ), "Cannot create a shared resource without being able to calculate MD5 prior to upload."
-    resource_to_path(resource_json_path, uploaded_resource, check_ci=True)
-    return uploaded_resource

thds_adls-4.1.20250701001205.dist-info/RECORD

@@ -1,40 +0,0 @@
-thds/adls/__init__.py,sha256=er14MoCC9PlJMxWVS4G1hAeMJItaJj4EAsrTZlvlb0M,797
-thds/adls/_progress.py,sha256=ZzCHn_G7nHakioNFxdvoJZRr-jN6ymsp5JXf-iReROM,6580
-thds/adls/_upload.py,sha256=XyP6tDM7s-A3G0SPSVlXRT4IZYsPqpOE4TeqtxP5i-I,4375
-thds/adls/abfss.py,sha256=ZRJOLjDuXmS4bIbQAQpQxWWWeu74N9NKEKCNfXQek80,726
-thds/adls/cached_up_down.py,sha256=3rdTuD_qKodFEo7vsl2mN5QMHSjDCGf3UaRKta6BTaI,2955
-thds/adls/conf.py,sha256=q1SPrgb46NpobVzwt_Oyv71-BvsIbZLq9nRWS3LZjz0,1990
-thds/adls/copy.py,sha256=jUWbGvTpb4B3yRGS0nhGSbDzqRPzUqYgH0z1lFRJB3k,6365
-thds/adls/dbfs.py,sha256=pPAjbIZRKJsaXKQljDMUgqS_zy1yKeEZHGMueXbuv3g,2219
-thds/adls/defaults.py,sha256=GGq5Pn4r-8cX4bZItp4nnwWAAz7S07pzPoOegw0y5Fw,676
-thds/adls/download.py,sha256=c1GapH8e7H5AmGR7S2u6CJ-XqQaDdtVQ6tKIwjuMTio,18510
-thds/adls/download_lock.py,sha256=ttD2GhPNRnITNoV1XH2PvKbMsHppZirjy3RZ4P4kgKM,2826
-thds/adls/errors.py,sha256=B_rMsQvQnNmP_sf-x8kmGsv2vIeOh4G9kVbdNVyk350,1469
-thds/adls/etag.py,sha256=ct7jpHhNFcKzbekn5rZ3m6DhjK48A7qOZGwDiHkc-pc,242
-thds/adls/file_properties.py,sha256=DYUu4zI_5amwTSMbhvUVimSEfEsfpgoqOawQ4Z4W-nY,1790
-thds/adls/fqn.py,sha256=0zHmHhBWN7GEfKRB3fBC1NVhaiIHHifBdCRanyT01X8,5822
-thds/adls/global_client.py,sha256=f4VJw5y_Yh__8gQUcdSYTh1aU6iEPlauMchVirSAwDQ,3716
-thds/adls/impl.py,sha256=4qa70w1sehzp60CI6lW82NLDK-lsM1uUfhPmZnYJaw0,42589
-thds/adls/md5.py,sha256=qOX4_7WUj1QkbH_IwREcQNHvvZccOj-HpHZBfsKn1gY,1846
-thds/adls/named_roots.py,sha256=7SLbAoQQpV_mrFZaUPjYoS-F9dxQxN5Hg4M3YPirF_w,751
-thds/adls/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-thds/adls/ro_cache.py,sha256=F0uXol0t95mcRuBukNg3A7wt7XXQxpD5Sy09d9sl8f0,4825
-thds/adls/sas_tokens.py,sha256=mArbB_GYohevOmArw_1gKqVUWpv6kG8Hsbvdrhbtnbg,1957
-thds/adls/shared_credential.py,sha256=-x42aXoIM001KW59oS8PpuXQd4-F2vg-1gB6OMHlpk4,4602
-thds/adls/source.py,sha256=1JYliDqafqpLewOQ2XMKp4T2WD7b5Rl7XYViyLYtzX8,2437
-thds/adls/source_tree.py,sha256=yP_v2XrKxXqUOdZ-x8kqHhBFAuur3AlAq3zi4hHj4AE,2235
-thds/adls/uri.py,sha256=pDH956p_VEHnjLLUnjWY6sGgRqksp9gdpc9KOW4gEP0,1205
-thds/adls/azcopy/__init__.py,sha256=nTNbgz2GcEiGeswYbAgy4oPhivnzl_5crF3HqCdWWiw,31
-thds/adls/azcopy/download.py,sha256=49Vv5z_bBTzkcCg8IP2fvQnMvrxqHS4AfxPWQ6CSEh0,6610
-thds/adls/resource/__init__.py,sha256=IZ7_aRf1b3jEp7wXOxqHop0gV2gUcf9SOLeEEjIWlCU,1669
-thds/adls/resource/core.py,sha256=u6iaJAbnk88Xfn6SQBgITDnvHpEFdMjUtE_fOgebbTU,2627
-thds/adls/resource/file_pointers.py,sha256=2g5lUtauA5vCZ4-skxIdmLeOp4cLZ7Mtv-MAhq6nIsA,1983
-thds/adls/resource/up_down.py,sha256=eTQMbeqM6edfVz18uHXRiOe0oEzOAaW1TBvym6lctNM,9822
-thds/adls/tools/download.py,sha256=vvBO8lSDl9oPugv75qpCkoemT9pOM9BV6yeExlkyG08,1594
-thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
-thds/adls/tools/upload.py,sha256=eMk4pdug1aCMPDDWpIE3Zoq77i5APp9Uuh-sVCCDNJE,493
-thds_adls-4.1.20250701001205.dist-info/METADATA,sha256=86BiNStEl525bbhbeqOzlIOzXy9-v-sESVUD9vQJuK0,587
-thds_adls-4.1.20250701001205.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_adls-4.1.20250701001205.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
-thds_adls-4.1.20250701001205.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_adls-4.1.20250701001205.dist-info/RECORD,,