thds.adls 4.2.20250827225749__py3-none-any.whl → 4.2.20250902211626__py3-none-any.whl

thds/adls/__init__.py

@@ -1,6 +1,17 @@
 from thds import core
 
-from . import abfss, defaults, etag, fqn, hashes, named_roots, source, source_tree, uri  # noqa: F401
+from . import (  # noqa: F401
+    abfss,
+    defaults,
+    etag,
+    fqn,
+    hashes,
+    list_fast,
+    named_roots,
+    source,
+    source_tree,
+    uri,
+)
 from .cached import download_directory, download_to_cache, upload_through_cache  # noqa: F401
 from .copy import copy_file, copy_files, wait_for_copy  # noqa: F401
 from .errors import BlobNotFoundError  # noqa: F401

thds/adls/_progress.py

@@ -53,8 +53,7 @@ def _blobs(n: list) -> str:
 
 
 class _Reporter(ty.Protocol):
-    def __call__(self, states: ty.List[ProgressState]):
-        ...
+    def __call__(self, states: ty.List[ProgressState]): ...
 
 
 class DumbReporter:

thds/adls/abfss.py

@@ -1,4 +1,5 @@
 """Translate ADLS URIs to ABFSS URIs (for use with Spark/Hadoop)."""
+
 from .fqn import AdlsFqn
 
 ABFSS_SCHEME = "abfss://"

thds/adls/copy.py

@@ -1,4 +1,5 @@
 """Functions for copying blobs across remote locations."""
+
 import datetime
 import random
 import time

thds/adls/list_fast.py

@@ -0,0 +1,95 @@
+"""This module is roughly 10x as fast as ADLSFileSystem.get_directory_info.
+
+Part of that is the parallelism, but part of it seems to be using the blob container
+client instead of the file system client.
+"""
+
+import typing as ty
+
+from thds.core import parallel, thunks
+
+from . import global_client
+from .fqn import AdlsFqn
+from .source_tree import BlobMeta, to_blob_meta, yield_blob_meta
+
+R = ty.TypeVar("R")
+
+
+def _failfast_parallel(thunks: ty.Iterable[ty.Callable[[], R]]) -> ty.Iterator[R]:
+    yield from (res for _, res in parallel.failfast(parallel.yield_all(parallel.create_keys(thunks))))
+
+
+def multilayer_yield_blob_meta(fqn: AdlsFqn, layers: int = 1) -> ty.Iterator[BlobMeta]:
+    """A fast way to find all blobs in a directory tree; we do this in parallel on
+    subdirs, with as much nesting as necessary (though 1 level should be enough for most cases).
+
+    Does not maintain order.
+
+    Does return directories; if you want to filter those out, you can use filter with is_dir.
+    """
+    if layers <= 0:
+        # directly yield the blobs
+        yield from yield_blob_meta(
+            global_client.get_global_blob_container_client(fqn.sa, fqn.container),
+            fqn.path,
+        )
+        return
+
+    # This code seems more complex than it needs to be...  Why is it like this?
+    #
+    # Well, it's an optimization.  The underlying Azure API being used (i.e.,
+    # call to list_blobs) limits the number of paths returned with each
+    # invocation and each subsequent request must use a continuation token
+    # received from the previous request to get the next "page" (it paginates serially).
+    #
+    # Running this sequentially for tens of thousands of blobs is slow if we just
+    # iterate over the paths recursively. Therefore, we first list/get the
+    # top-level subdirectories (not recursively), and then in parallel ask
+    # the API to list the parquet paths of each subdirectory in parallel,
+    # greatly speeding things up.
+    #
+    # Because there may be parquet files in the top-level directory, we also
+    # gather those up and get `BlobProperties` for those individually (this is
+    # what azure's SDK offers us as an option -_-.  At least we can do each of
+    # these gets in parallel...), then convert each of them to a adls.source_tree.BlobMeta.
+    #
+    # API reference: https://learn.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/list?view=rest-storageservices-datalakestoragegen2-2019-12-12
+
+    # you cannot limit recursion depth using the container client, so to speed things up
+    # we use the file system client to get the direct children, and then use the container client
+    # to list within each final subdir
+    subdirs, files = set(), set()  # direct children of the top-level directory
+    for child_path_props in global_client.get_global_fs_client(fqn.sa, fqn.container).get_paths(
+        path=fqn.path, recursive=False
+    ):
+        if child_path_props.is_directory:
+            subdirs.add(child_path_props.name)
+        else:
+            files.add(child_path_props.name)
+
+    blob_container_client = global_client.get_global_blob_container_client(fqn.sa, fqn.container)
+
+    def _get_blob_meta(blob_name: str) -> BlobMeta:
+        return to_blob_meta(blob_container_client.get_blob_client(blob_name).get_blob_properties())
+
+    for blob_meta_iter in (
+        _failfast_parallel((thunks.thunking(_get_blob_meta)(file) for file in files)),
+        *(
+            _failfast_parallel(
+                # we use list_blobs (defined below) rather than yield here because returning
+                # an iterator across a thread boundary doesn't work
+                thunks.thunking(_list_blob_meta)(AdlsFqn(fqn.sa, fqn.container, subdir), layers - 1)
+                for subdir in subdirs
+            )
+        ),
+    ):
+        yield from blob_meta_iter
+
+
+def is_dir(blob_meta: BlobMeta) -> bool:  # TODO move to blob_meta.py once it exists
+    return blob_meta.metadata.get("hdi_isfolder", "false") == "true"
+
+
+def _list_blob_meta(fqn: AdlsFqn, layers: int = 1) -> list[BlobMeta]:
+    """Only for use within multi_layer_yield_blobs."""
+    return list(multilayer_yield_blob_meta(fqn, layers))

thds/adls/source_tree.py

@@ -8,6 +8,8 @@ from thds.core.source.tree import SourceTree
 
 from . import fqn, global_client, hashes, source, uri
 
+# TODO refactor BlobMeta into its own module.
+
 
 @dataclass
 class BlobMeta:

thds/adls/tools/ls_fast.py

@@ -0,0 +1,22 @@
+import argparse
+
+from thds.adls import list_fast, uri
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("uri", type=uri.parse_any, help="A fully qualified path to an ADLS location")
+    parser.add_argument("--verbose", "-v", action="store_true")
+    args = parser.parse_args()
+
+    for f in list_fast.multilayer_yield_blob_meta(args.uri):
+        print(f.path)
+        if args.verbose:
+            print("      ", f.size)
+            print("      ", f.hash)
+            print("      ", f.metadata)
+            print()
+
+
+if __name__ == "__main__":
+    main()

{thds_adls-4.2.20250827225749.dist-info → thds_adls-4.2.20250902211626.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.adls
-Version: 4.2.20250827225749
+Version: 4.2.20250902211626
 Summary: ADLS tools
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

{thds_adls-4.2.20250827225749.dist-info → thds_adls-4.2.20250902211626.dist-info}/RECORD

@@ -1,11 +1,11 @@
-thds/adls/__init__.py,sha256=PL0BRhiLhW_xY_2hhBgd8v3_NS1zpR4Kdd28zjNHBgo,1017
+thds/adls/__init__.py,sha256=Z450uhM5BIxgve9qkz_Lw7biVWFKp3Uz_iKaSPYEf1Y,1073
 thds/adls/_fork_protector.py,sha256=WML1Ul-G_MCziBOyRp6GoNq1IGktn8-OQVjny1IZ1bs,950
-thds/adls/_progress.py,sha256=D6XIipzG_xwmxs_08LuiYFfThGqHTU2KiIyjNduiOFY,6656
+thds/adls/_progress.py,sha256=0cmFZ2sOEogQY25AkMAhcunXO69FdlbkD1rU_zmLAp0,6648
 thds/adls/_upload.py,sha256=mhTdWiQroaugYuwQg7R8CEgdfCYF4xvJthlsqO0jlnE,4692
-thds/adls/abfss.py,sha256=ZRJOLjDuXmS4bIbQAQpQxWWWeu74N9NKEKCNfXQek80,726
+thds/adls/abfss.py,sha256=nCFfcdwLiwtz_MdbrpLJ9WOhoz0zHIVxsf-tarorEwM,727
 thds/adls/cached.py,sha256=up1F5JOVXdmwdZ8RAB2UDgiy6ooLg8IMULohBh75VpQ,3034
 thds/adls/conf.py,sha256=nTw3X1ilC3A_905jZH-rWXFsESeHAKQn5IghvfX2VIo,1991
-thds/adls/copy.py,sha256=jUWbGvTpb4B3yRGS0nhGSbDzqRPzUqYgH0z1lFRJB3k,6365
+thds/adls/copy.py,sha256=BGtzroE46ZrL10eFsW_a2ng6W9dUgG2tlk1wk0UNeDI,6366
 thds/adls/dbfs.py,sha256=pPAjbIZRKJsaXKQljDMUgqS_zy1yKeEZHGMueXbuv3g,2219
 thds/adls/defaults.py,sha256=GGq5Pn4r-8cX4bZItp4nnwWAAz7S07pzPoOegw0y5Fw,676
 thds/adls/download.py,sha256=LHxkv073T-pCmIRmiXIgYEXIEpEm_OIZKvNHp1e4_k4,19124
@@ -17,6 +17,7 @@ thds/adls/fqn.py,sha256=pIfEw25SjZNGmtzOLwrYCblciK35VQ35ZWb_mRaZKK0,5915
 thds/adls/global_client.py,sha256=q6oG1OOsfl4fbti81u8TE9d4Jx9-phlYgsGSc4032w8,3721
 thds/adls/hashes.py,sha256=2x1zcT_87en_vqFrLFs6F_EZCGpn7hk_81dYAwcypm8,5459
 thds/adls/impl.py,sha256=cNf1vmeS46X_wvyVdDJ8qFfowHn2QwtU5C80BmDtu5Y,43247
+thds/adls/list_fast.py,sha256=7jHnln4DMWYVLHhejj-fdWMBWflBiWfynegKxcUlNDY,4189
 thds/adls/md5.py,sha256=hGT8AIX32VUsnRCbm8cel9OlxAiRrgjwNWQTqRDHM_k,374
 thds/adls/named_roots.py,sha256=7SLbAoQQpV_mrFZaUPjYoS-F9dxQxN5Hg4M3YPirF_w,751
 thds/adls/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -24,7 +25,7 @@ thds/adls/ro_cache.py,sha256=P-UVFZhqnE5wojqYmRVWZcqjl-pM1xVMm9VAm3nXlnA,4769
 thds/adls/sas_tokens.py,sha256=mArbB_GYohevOmArw_1gKqVUWpv6kG8Hsbvdrhbtnbg,1957
 thds/adls/shared_credential.py,sha256=-x42aXoIM001KW59oS8PpuXQd4-F2vg-1gB6OMHlpk4,4602
 thds/adls/source.py,sha256=8HVMYuxDn1XYGwFFSBowMlvQ6r2Jm2CQlpu4h85JvsE,2559
-thds/adls/source_tree.py,sha256=FFcSJpbb7w5y6v1kzRQl0cUjjuSb34V8J2iHrrm9zDA,2542
+thds/adls/source_tree.py,sha256=gl2JLjxAduo4cGQBb8LqBnmRHHk2wqIC5yt-sqkXOEo,2589
 thds/adls/upload.py,sha256=MRHK9Am-x5FKBPh1SXLTbPC1r0Xk0bGWNU8CcNuUMLo,6602
 thds/adls/uri.py,sha256=9MXuW_KfpPvzBc4ERxuTJ3vvi_6yr7e1kMAW9mx2zXM,1414
 thds/adls/azcopy/__init__.py,sha256=qn2dmT92EHcrtaQ8uwRoUgvtF6Fu3NQbhZItOBdIBmY,45
@@ -35,9 +36,10 @@ thds/adls/azcopy/system_resources.py,sha256=okgDEKAp0oWGQF7OKikbgJ9buBeiOgNaDYy-
 thds/adls/azcopy/upload.py,sha256=RQLDJzS6qsMM12t5bykWJWBXs0UrmImrEFnPMxX2UlM,2767
 thds/adls/tools/download.py,sha256=CW2cWbCRdUqisVVVoqqvqk5Ved7pPGTkwnZj3uV0jy4,1587
 thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
+thds/adls/tools/ls_fast.py,sha256=Nowc-efAL_Y4ybPwZzKIeh7KGIjfecRzdWvJZcBzq_8,585
 thds/adls/tools/upload.py,sha256=5WyWkpuVp2PETZ3O3ODlq8LXszSHU73ZMnIDZXPJdC8,442
-thds_adls-4.2.20250827225749.dist-info/METADATA,sha256=sRSt8S6xJPAVYqh9z4XCnVoGI9HB8WOhmuDZuhXiQrI,587
-thds_adls-4.2.20250827225749.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_adls-4.2.20250827225749.dist-info/entry_points.txt,sha256=rtVF0A2MMTYUsBScF6b3AlOuk2Vm02QK7Tc2bDcDpk0,200
-thds_adls-4.2.20250827225749.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_adls-4.2.20250827225749.dist-info/RECORD,,
+thds_adls-4.2.20250902211626.dist-info/METADATA,sha256=UiTXMLzpZi6uvnbHbBJ_h4wOwG5cF5XvtgbK3SyWAMg,587
+thds_adls-4.2.20250902211626.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_adls-4.2.20250902211626.dist-info/entry_points.txt,sha256=rtVF0A2MMTYUsBScF6b3AlOuk2Vm02QK7Tc2bDcDpk0,200
+thds_adls-4.2.20250902211626.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_adls-4.2.20250902211626.dist-info/RECORD,,