thds.adls 3.2.20250618235513__py3-none-any.whl → 3.2.20250619170701__py3-none-any.whl

thds/adls/cached_up_down.py

@@ -2,8 +2,9 @@ import typing as ty
 from pathlib import Path
 
 from thds.adls import source
-from thds.core.source import Source
+from thds.core import parallel
 from thds.core.source.tree import SourceTree
+from thds.core.thunks import thunking
 
 from .download import download_or_use_verified
 from .fqn import AdlsFqn
@@ -53,18 +54,30 @@ def download_directory(fqn: AdlsFqn) -> Path:
     return cached_dir_root
 
 
+def _yield_all_file_paths(dir_path: Path) -> ty.Iterator[Path]:
+    for item in dir_path.iterdir():
+        if item.is_dir():  # recur
+            yield from _yield_all_file_paths(item)
+        elif item.is_file():  # yield
+            yield item
+
+
 def upload_directory_through_cache(dest: UriIsh, src_path: Path) -> SourceTree:
     if not src_path.is_dir():
         raise ValueError(f"If you want to upload a file, use {upload_through_cache.__name__} instead")
 
     dest = parse_any(dest)
 
-    def _upload_directory(dir_path: Path) -> ty.Iterable[Source]:
-        for item in dir_path.iterdir():
-            if item.is_dir():  # recur
-                yield from _upload_directory(item)
-            elif item.is_file():  # upload
-                file_dest = dest / str(item.relative_to(src_path))
-                yield source.from_adls(upload_through_cache(file_dest, item))
-
-    return SourceTree(sources=list(_upload_directory(src_path)))
+    upload_thunks = [
+        thunking(upload_through_cache)(dest / str(file_path.relative_to(src_path)), file_path)
+        for file_path in _yield_all_file_paths(src_path)
+    ]
+
+    return SourceTree(
+        sources=list(
+            map(
+                source.from_adls,
+                parallel.yield_results(upload_thunks, named="upload_directory_through_cache"),
+            )
+        )
+    )

{thds_adls-3.2.20250618235513.dist-info → thds_adls-3.2.20250619170701.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.adls
-Version: 3.2.20250618235513
+Version: 3.2.20250619170701
 Summary: ADLS tools
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

{thds_adls-3.2.20250618235513.dist-info → thds_adls-3.2.20250619170701.dist-info}/RECORD

@@ -2,7 +2,7 @@ thds/adls/__init__.py,sha256=er14MoCC9PlJMxWVS4G1hAeMJItaJj4EAsrTZlvlb0M,797
 thds/adls/_progress.py,sha256=ZzCHn_G7nHakioNFxdvoJZRr-jN6ymsp5JXf-iReROM,6580
 thds/adls/_upload.py,sha256=XyP6tDM7s-A3G0SPSVlXRT4IZYsPqpOE4TeqtxP5i-I,4375
 thds/adls/abfss.py,sha256=ZRJOLjDuXmS4bIbQAQpQxWWWeu74N9NKEKCNfXQek80,726
-thds/adls/cached_up_down.py,sha256=CEqg-Q9FB_fvgCsN0C_dn9CJqnEpnF0lA7gCdd9s6gg,2721
+thds/adls/cached_up_down.py,sha256=3rdTuD_qKodFEo7vsl2mN5QMHSjDCGf3UaRKta6BTaI,2955
 thds/adls/conf.py,sha256=q1SPrgb46NpobVzwt_Oyv71-BvsIbZLq9nRWS3LZjz0,1990
 thds/adls/copy.py,sha256=jUWbGvTpb4B3yRGS0nhGSbDzqRPzUqYgH0z1lFRJB3k,6365
 thds/adls/dbfs.py,sha256=pPAjbIZRKJsaXKQljDMUgqS_zy1yKeEZHGMueXbuv3g,2219
@@ -33,8 +33,8 @@ thds/adls/resource/up_down.py,sha256=eTQMbeqM6edfVz18uHXRiOe0oEzOAaW1TBvym6lctNM
 thds/adls/tools/download.py,sha256=vvBO8lSDl9oPugv75qpCkoemT9pOM9BV6yeExlkyG08,1594
 thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
 thds/adls/tools/upload.py,sha256=eMk4pdug1aCMPDDWpIE3Zoq77i5APp9Uuh-sVCCDNJE,493
-thds_adls-3.2.20250618235513.dist-info/METADATA,sha256=jVlp7EQRZMo-3rGvM02FL8Eoe4qoH9wdvhydB6g_-o0,543
-thds_adls-3.2.20250618235513.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_adls-3.2.20250618235513.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
-thds_adls-3.2.20250618235513.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_adls-3.2.20250618235513.dist-info/RECORD,,
+thds_adls-3.2.20250619170701.dist-info/METADATA,sha256=Ij-9J-Q0YxxL0bpzxe-Yvq_EbILEFzAiqWshTUpjvXo,543
+thds_adls-3.2.20250619170701.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_adls-3.2.20250619170701.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
+thds_adls-3.2.20250619170701.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_adls-3.2.20250619170701.dist-info/RECORD,,