thds.adls 4.1.20250711201240__py3-none-any.whl → 4.1.20250715194412__py3-none-any.whl

thds/adls/_progress.py

@@ -161,6 +161,10 @@ def get_global_download_tracker() -> Tracker:
     return _GLOBAL_DN_TRACKER
 
 
+def get_global_upload_tracker() -> Tracker:
+    return _GLOBAL_UP_TRACKER
+
+
 def _proxy_io(io_type: str, stream: T, key: str, total_len: int) -> T:
     assert io_type in ("read", "write"), io_type
 

thds/adls/azcopy/download.py

@@ -74,7 +74,9 @@ def sync_fastpath(
             )
             assert process.stdout
             output_lines = list()
-            with progress.azcopy_tracker(dl_file_client.url, download_request.size_bytes or 0) as track:
+            with progress.azcopy_tracker(
+                "down", dl_file_client.url, download_request.size_bytes or 0
+            ) as track:
                 for line in process.stdout:
                     track(line)
                     output_lines.append(line.strip())
@@ -127,7 +129,9 @@ async def async_fastpath(
 
             # Feed lines to the tracker asynchronously
             output_lines = list()
-            with progress.azcopy_tracker(dl_file_client.url, download_request.size_bytes or 0) as track:
+            with progress.azcopy_tracker(
+                "down", dl_file_client.url, download_request.size_bytes or 0
+            ) as track:
                 while True:
                     line = await copy_proc.stdout.readline()
                     if not line:  # EOF

thds/adls/azcopy/progress.py

@@ -25,9 +25,15 @@ def _parse_azcopy_json_output(line: str) -> AzCopyJsonLine:
 
 
 @contextmanager
-def azcopy_tracker(http_url: str, size_bytes: int) -> ty.Iterator[ty.Callable[[str], None]]:
+def azcopy_tracker(
+    direction: ty.Literal["up", "down"], http_url: str, size_bytes: int
+) -> ty.Iterator[ty.Callable[[str], None]]:
     """Context manager that tracks progress from AzCopy JSON lines. This works for both async and sync impls."""
-    tracker = _progress.get_global_download_tracker()
+    tracker = (
+        _progress.get_global_download_tracker
+        if direction == "down"
+        else _progress.get_global_upload_tracker
+    )()
     adls_uri = urllib.parse.unquote(str(uri.parse_uri(http_url)))
     if size_bytes:
         tracker.add(adls_uri, total=size_bytes)

thds/adls/azcopy/upload.py

@@ -87,7 +87,7 @@ def run(
     )
     assert process.stdout
     output_lines = list()
-    with progress.azcopy_tracker(uri.to_blob_windows_url(dest), size_bytes) as track:
+    with progress.azcopy_tracker("up", uri.to_blob_windows_url(dest), size_bytes) as track:
         for line in process.stdout:
             track(line)
             output_lines.append(line.strip())

thds/adls/file_properties.py

@@ -29,10 +29,17 @@ def get_blob_properties(fqn: AdlsFqn) -> BlobProperties:
     )
 
 
+class ContentSettingsP(ty.Protocol):
+    content_md5: ty.Optional[bytearray]
+
+
 class PropertiesP(ty.Protocol):
     name: ty.Any
     metadata: ty.Any
-    content_settings: ty.Any
+
+    @property
+    def content_settings(self) -> ContentSettingsP:
+        pass
 
 
 # At some point it may make sense to separate file and blob property modules,

thds/adls/hashes.py

@@ -74,7 +74,7 @@ def extract_hashes_from_props(
 
     hashes = list(extract_hashes_from_metadata(props.metadata or dict()))
     if props.content_settings and props.content_settings.content_md5:
-        hashes.append(hashing.Hash("md5", props.content_settings.content_md5))
+        hashes.append(hashing.Hash("md5", bytes(props.content_settings.content_md5)))
     return {h.algo: h for h in hashes}
 
 

thds/adls/source_tree.py

@@ -6,25 +6,21 @@ from azure.storage.blob import BlobProperties, ContainerClient
 from thds.core import hashing
 from thds.core.source.tree import SourceTree
 
-from . import fqn, global_client, source, uri
+from . import fqn, global_client, hashes, source, uri
 
 
 @dataclass
 class BlobMeta:
     path: str
     size: int
-    md5: ty.Optional[hashing.Hash]
+    hash: ty.Optional[hashing.Hash]
 
 
 def to_blob_meta(blob_props: BlobProperties) -> BlobMeta:
     return BlobMeta(
         blob_props.name,
         blob_props.size,
-        (
-            hashing.Hash("md5", bytes(blob_props.content_settings.content_md5))
-            if blob_props.content_settings.content_md5
-            else None
-        ),
+        next(iter(hashes.extract_hashes_from_props(blob_props).values()), None),
     )
 
 
@@ -34,10 +30,11 @@ def to_blob_meta(blob_props: BlobProperties) -> BlobMeta:
 def list_blob_meta(
     container_client: ContainerClient, root_dir: str, match_suffix: str = ""
 ) -> ty.List[BlobMeta]:
-    """Gets the path (relative to the SA/container root), size, and MD5 hash of all blobs in a directory."""
+    """Gets the path (relative to the SA/container root), size, and _a_ hash (if available) of all blobs in a directory."""
     return [
         to_blob_meta(blob_props)
-        for blob_props in container_client.list_blobs(name_starts_with=root_dir)
+        for blob_props in container_client.list_blobs(name_starts_with=root_dir, include=["metadata"])
+        # `list_blobs` does not include metadata by default, so we need to explicitly specify including it
         if blob_props.size > 0
         # container client lists directories as blobs with size 0
         and blob_props.name.endswith(match_suffix)
@@ -54,7 +51,7 @@ def from_path(adls_path: uri.UriIsh, match_suffix: str = "") -> SourceTree:
     container_root = root_fqn.root()
     return SourceTree(
         sources=[
-            source.from_adls(container_root / blob_meta.path, hash=blob_meta.md5)
+            source.from_adls(container_root / blob_meta.path, hash=blob_meta.hash)
             for blob_meta in list_blob_meta(container_client, root_fqn.path, match_suffix=match_suffix)
         ],
         higher_logical_root=fqn.split(root_fqn)[-1],

{thds_adls-4.1.20250711201240.dist-info → thds_adls-4.1.20250715194412.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.adls
-Version: 4.1.20250711201240
+Version: 4.1.20250715194412
 Summary: ADLS tools
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

{thds_adls-4.1.20250711201240.dist-info → thds_adls-4.1.20250715194412.dist-info}/RECORD

@@ -1,5 +1,5 @@
 thds/adls/__init__.py,sha256=g2Zb0EAAH-JzPMYHAub9liU4qa5pfqQDnILfEhmObGo,1036
-thds/adls/_progress.py,sha256=ZzCHn_G7nHakioNFxdvoJZRr-jN6ymsp5JXf-iReROM,6580
+thds/adls/_progress.py,sha256=D6XIipzG_xwmxs_08LuiYFfThGqHTU2KiIyjNduiOFY,6656
 thds/adls/_upload.py,sha256=mhTdWiQroaugYuwQg7R8CEgdfCYF4xvJthlsqO0jlnE,4692
 thds/adls/abfss.py,sha256=ZRJOLjDuXmS4bIbQAQpQxWWWeu74N9NKEKCNfXQek80,726
 thds/adls/cached.py,sha256=up1F5JOVXdmwdZ8RAB2UDgiy6ooLg8IMULohBh75VpQ,3034
@@ -11,10 +11,10 @@ thds/adls/download.py,sha256=WOpMXGUbWImBdkM4tSW7qnCbu7G_cRXKF5pFQVLPPxs,18772
 thds/adls/download_lock.py,sha256=tgT48l4C5_qmArGeq05gl7VlxT22dZBH2Xwxx0itE9o,3176
 thds/adls/errors.py,sha256=6NMLHtVNsWBRDXaes9yzHj9cwKOD9t1dwL4BltdtjhU,1895
 thds/adls/etag.py,sha256=ct7jpHhNFcKzbekn5rZ3m6DhjK48A7qOZGwDiHkc-pc,242
-thds/adls/file_properties.py,sha256=C9Kl3a5wuBNWYgZYnZbkH04u8uxadEcjVJIm3UevUM0,1912
+thds/adls/file_properties.py,sha256=dhRtbsMNOYfExkEiy76wrLfrJ6IMQeN1Z3LIxgKceqY,2042
 thds/adls/fqn.py,sha256=0zHmHhBWN7GEfKRB3fBC1NVhaiIHHifBdCRanyT01X8,5822
 thds/adls/global_client.py,sha256=f4VJw5y_Yh__8gQUcdSYTh1aU6iEPlauMchVirSAwDQ,3716
-thds/adls/hashes.py,sha256=RDQS-C38wskUhxXGFGLJ4ox8vm7ofurxSsUk13Ywijo,5309
+thds/adls/hashes.py,sha256=-wRRATGmww7k2RD5Zmhq_Fq7Z2JihLy1njeHFekU15c,5316
 thds/adls/impl.py,sha256=cNf1vmeS46X_wvyVdDJ8qFfowHn2QwtU5C80BmDtu5Y,43247
 thds/adls/md5.py,sha256=hGT8AIX32VUsnRCbm8cel9OlxAiRrgjwNWQTqRDHM_k,374
 thds/adls/named_roots.py,sha256=7SLbAoQQpV_mrFZaUPjYoS-F9dxQxN5Hg4M3YPirF_w,751
@@ -23,20 +23,20 @@ thds/adls/ro_cache.py,sha256=P-UVFZhqnE5wojqYmRVWZcqjl-pM1xVMm9VAm3nXlnA,4769
 thds/adls/sas_tokens.py,sha256=mArbB_GYohevOmArw_1gKqVUWpv6kG8Hsbvdrhbtnbg,1957
 thds/adls/shared_credential.py,sha256=-x42aXoIM001KW59oS8PpuXQd4-F2vg-1gB6OMHlpk4,4602
 thds/adls/source.py,sha256=8HVMYuxDn1XYGwFFSBowMlvQ6r2Jm2CQlpu4h85JvsE,2559
-thds/adls/source_tree.py,sha256=yP_v2XrKxXqUOdZ-x8kqHhBFAuur3AlAq3zi4hHj4AE,2235
+thds/adls/source_tree.py,sha256=exBi7jNrOyj9nH5rzxZmZOqhRW9WXnmZLdLt4weoXYc,2292
 thds/adls/upload.py,sha256=MRHK9Am-x5FKBPh1SXLTbPC1r0Xk0bGWNU8CcNuUMLo,6602
 thds/adls/uri.py,sha256=9MXuW_KfpPvzBc4ERxuTJ3vvi_6yr7e1kMAW9mx2zXM,1414
 thds/adls/azcopy/__init__.py,sha256=qn2dmT92EHcrtaQ8uwRoUgvtF6Fu3NQbhZItOBdIBmY,45
-thds/adls/azcopy/download.py,sha256=MwkUaQTrrXRX9yip_hiLZXzSgGqyTDUxjr0MeXitWuo,6450
+thds/adls/azcopy/download.py,sha256=FOtYyYh7ZXNWNdkj04yTV26lxcKOVj-YhS2p_EclYxA,6526
 thds/adls/azcopy/login.py,sha256=923UaewVMPFzkDSgCQsbl-_g7qdFhpXpF0MGNIy3T_A,1538
-thds/adls/azcopy/progress.py,sha256=K7TVmSiWfu561orL3GuOnlQX9VtVxWVECAq9NiweYNo,1387
+thds/adls/azcopy/progress.py,sha256=U9JzkHKyCglyzq3qMemcRKhmtDJhF1zXxHxlO_P3F9w,1526
 thds/adls/azcopy/system_resources.py,sha256=okgDEKAp0oWGQF7OKikbgJ9buBeiOgNaDYy-36j6dHo,761
-thds/adls/azcopy/upload.py,sha256=0l5FzV9IgZ2iQhm4eKZjTdw4SO17bHd8VnwcTev1lUs,2761
+thds/adls/azcopy/upload.py,sha256=RQLDJzS6qsMM12t5bykWJWBXs0UrmImrEFnPMxX2UlM,2767
 thds/adls/tools/download.py,sha256=CW2cWbCRdUqisVVVoqqvqk5Ved7pPGTkwnZj3uV0jy4,1587
 thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
 thds/adls/tools/upload.py,sha256=5WyWkpuVp2PETZ3O3ODlq8LXszSHU73ZMnIDZXPJdC8,442
-thds_adls-4.1.20250711201240.dist-info/METADATA,sha256=6loFnjW02fNJwgfTwSw5FLgsHe5SmX1AN4mgNRpCsts,587
-thds_adls-4.1.20250711201240.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_adls-4.1.20250711201240.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
-thds_adls-4.1.20250711201240.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_adls-4.1.20250711201240.dist-info/RECORD,,
+thds_adls-4.1.20250715194412.dist-info/METADATA,sha256=BWc8D2IzGxH03rsA-f1M2KpCi0S_Fzjm7eWRMhDjsUs,587
+thds_adls-4.1.20250715194412.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_adls-4.1.20250715194412.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
+thds_adls-4.1.20250715194412.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_adls-4.1.20250715194412.dist-info/RECORD,,