thds.adls 4.1.20250702194306__py3-none-any.whl → 4.1.20250709174808__py3-none-any.whl

thds/adls/azcopy/download.py

@@ -73,19 +73,24 @@ def sync_fastpath(
                 env=system_resources.restrict_usage(),
             )
             assert process.stdout
+            output_lines = list()
             with progress.azcopy_tracker(dl_file_client.url, download_request.size_bytes or 0) as track:
                 for line in process.stdout:
                     track(line)
+                    output_lines.append(line.strip())
 
             process.wait()
             if process.returncode != 0:
-                raise subprocess.SubprocessError(f"AzCopy failed with return code {process.returncode}")
+                raise subprocess.CalledProcessError(
+                    process.returncode,
+                    f"AzCopy failed with return code {process.returncode}\n\n" + "\n".join(output_lines),
+                )
             assert (
                 download_request.temp_path.exists()
             ), f"AzCopy did not create the file at {download_request.temp_path}"
             return
 
-        except (subprocess.CalledProcessError, FileNotFoundError):
+        except (subprocess.SubprocessError, FileNotFoundError):
             logger.warning("Falling back to Python SDK for download")
 
     logger.debug("Downloading %s using Python SDK", dl_file_client.url)
@@ -121,17 +126,22 @@ async def async_fastpath(
             assert copy_proc.stdout
 
             # Feed lines to the tracker asynchronously
+            output_lines = list()
             with progress.azcopy_tracker(dl_file_client.url, download_request.size_bytes or 0) as track:
                 while True:
                     line = await copy_proc.stdout.readline()
                     if not line:  # EOF
                         break
                     track(line.decode().strip())
+                    output_lines.append(line.decode().strip())
 
             # Wait for process completion
             exit_code = await copy_proc.wait()
             if exit_code != 0:
-                raise subprocess.SubprocessError()
+                raise subprocess.CalledProcessError(
+                    exit_code,
+                    f"AzCopy failed with return code {exit_code}\n\n" + "\n".join(output_lines),
+                )
 
             return
 

thds/adls/azcopy/upload.py

@@ -86,10 +86,15 @@ def run(
         env=system_resources.restrict_usage(),
     )
     assert process.stdout
+    output_lines = list()
     with progress.azcopy_tracker(uri.to_blob_windows_url(dest), size_bytes) as track:
         for line in process.stdout:
             track(line)
+            output_lines.append(line.strip())
 
     process.wait()
     if process.returncode != 0:
-        raise subprocess.SubprocessError(f"AzCopy failed with return code {process.returncode}")
+        raise subprocess.CalledProcessError(
+            process.returncode,
+            f"AzCopy failed with return code {process.returncode}\n\n" + "\n".join(output_lines),
+        )

thds/adls/download.py

@@ -7,6 +7,7 @@ import typing as ty
 from pathlib import Path
 
 import aiohttp.http_exceptions
+import requests.exceptions
 from azure.core.exceptions import AzureError, HttpResponseError, ResourceModifiedError
 from azure.storage.filedatalake import DataLakeFileClient, FileProperties, FileSystemClient, aio
 
@@ -307,6 +308,7 @@ def _excs_to_retry() -> ty.Callable[[Exception], bool]:
             filter(
                 None,
                 (
+                    requests.exceptions.ConnectionError,
                     aiohttp.http_exceptions.ContentLengthError,
                     aiohttp.client_exceptions.ClientPayloadError,
                     getattr(
@@ -318,6 +320,18 @@ def _excs_to_retry() -> ty.Callable[[Exception], bool]:
     )
 
 
+def _log_nonfatal_hash_error_exc(exc: Exception, url: str) -> None:
+    """Azure exceptions are very noisy."""
+    msg = "Unable to set hash for %s: %s"
+    exception_txt = str(exc)
+    log, extra_txt = (
+        (logger.debug, type(exc).__name__)
+        if ("AuthorizationPermissionMismatch" in exception_txt or "ConditionNotMet" in exception_txt)
+        else (logger.warning, exception_txt)
+    )
+    log(msg, url, extra_txt)
+
+
 @_dl_scope.bound
 def download_or_use_verified(
     fs_client: FileSystemClient,
@@ -360,7 +374,7 @@ def download_or_use_verified(
                 assert file_properties
                 dl_file_client.set_metadata(meta, **etag.match_etag(file_properties))
             except (HttpResponseError, ResourceModifiedError) as ex:
-                logger.info(f"Unable to set Hash for {remote_key}: {ex}")
+                _log_nonfatal_hash_error_exc(ex, dl_file_client.url)
         return si.value.hit
     except AzureError as err:
         errors.translate_azure_error(fs_client, remote_key, err)
@@ -418,7 +432,7 @@ async def async_download_or_use_verified(
                 await dl_file_client.set_metadata(meta, **etag.match_etag(file_properties))  # type: ignore[misc]
                 # TODO - check above type ignore
             except (HttpResponseError, ResourceModifiedError) as ex:
-                logger.info(f"Unable to set Hash for {remote_key}: {ex}")
+                _log_nonfatal_hash_error_exc(ex, dl_file_client.url)
         return si.value.hit
     except AzureError as err:
         errors.translate_azure_error(fs_client, remote_key, err)

{thds_adls-4.1.20250702194306.dist-info → thds_adls-4.1.20250709174808.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.adls
-Version: 4.1.20250702194306
+Version: 4.1.20250709174808
 Summary: ADLS tools
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

{thds_adls-4.1.20250702194306.dist-info → thds_adls-4.1.20250709174808.dist-info}/RECORD

@@ -7,7 +7,7 @@ thds/adls/conf.py,sha256=nTw3X1ilC3A_905jZH-rWXFsESeHAKQn5IghvfX2VIo,1991
 thds/adls/copy.py,sha256=jUWbGvTpb4B3yRGS0nhGSbDzqRPzUqYgH0z1lFRJB3k,6365
 thds/adls/dbfs.py,sha256=pPAjbIZRKJsaXKQljDMUgqS_zy1yKeEZHGMueXbuv3g,2219
 thds/adls/defaults.py,sha256=GGq5Pn4r-8cX4bZItp4nnwWAAz7S07pzPoOegw0y5Fw,676
-thds/adls/download.py,sha256=HzmhHM0FAmxtCRkK9M7NajsIzIuHD74GuxP3dyLoP1Q,18266
+thds/adls/download.py,sha256=WOpMXGUbWImBdkM4tSW7qnCbu7G_cRXKF5pFQVLPPxs,18772
 thds/adls/download_lock.py,sha256=tgT48l4C5_qmArGeq05gl7VlxT22dZBH2Xwxx0itE9o,3176
 thds/adls/errors.py,sha256=6cLg2E4SB8ic46PBzA3ynRH4b1oR8qRb07RBgKGJRxY,1783
 thds/adls/etag.py,sha256=ct7jpHhNFcKzbekn5rZ3m6DhjK48A7qOZGwDiHkc-pc,242
@@ -27,16 +27,16 @@ thds/adls/source_tree.py,sha256=yP_v2XrKxXqUOdZ-x8kqHhBFAuur3AlAq3zi4hHj4AE,2235
 thds/adls/upload.py,sha256=MRHK9Am-x5FKBPh1SXLTbPC1r0Xk0bGWNU8CcNuUMLo,6602
 thds/adls/uri.py,sha256=9MXuW_KfpPvzBc4ERxuTJ3vvi_6yr7e1kMAW9mx2zXM,1414
 thds/adls/azcopy/__init__.py,sha256=qn2dmT92EHcrtaQ8uwRoUgvtF6Fu3NQbhZItOBdIBmY,45
-thds/adls/azcopy/download.py,sha256=8shLbizgKr5WLmOitQ8TY28EVj2IdT7iSRmRgqFNLAg,6008
+thds/adls/azcopy/download.py,sha256=MwkUaQTrrXRX9yip_hiLZXzSgGqyTDUxjr0MeXitWuo,6450
 thds/adls/azcopy/login.py,sha256=923UaewVMPFzkDSgCQsbl-_g7qdFhpXpF0MGNIy3T_A,1538
 thds/adls/azcopy/progress.py,sha256=K7TVmSiWfu561orL3GuOnlQX9VtVxWVECAq9NiweYNo,1387
 thds/adls/azcopy/system_resources.py,sha256=okgDEKAp0oWGQF7OKikbgJ9buBeiOgNaDYy-36j6dHo,761
-thds/adls/azcopy/upload.py,sha256=bvtYdbaFsZkOHFLDpeBlTKqw63P3_kbImInI04ZlekM,2601
+thds/adls/azcopy/upload.py,sha256=0l5FzV9IgZ2iQhm4eKZjTdw4SO17bHd8VnwcTev1lUs,2761
 thds/adls/tools/download.py,sha256=CW2cWbCRdUqisVVVoqqvqk5Ved7pPGTkwnZj3uV0jy4,1587
 thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
 thds/adls/tools/upload.py,sha256=5WyWkpuVp2PETZ3O3ODlq8LXszSHU73ZMnIDZXPJdC8,442
-thds_adls-4.1.20250702194306.dist-info/METADATA,sha256=zgZubxCu37Sqrjn7b5NTJNlPxZbRgbcx-omuFBiVfMg,587
-thds_adls-4.1.20250702194306.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_adls-4.1.20250702194306.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
-thds_adls-4.1.20250702194306.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_adls-4.1.20250702194306.dist-info/RECORD,,
+thds_adls-4.1.20250709174808.dist-info/METADATA,sha256=-mRVDeTBKoaoIACUBVh0BokOp2jrVOM11eQLfSOP0E4,587
+thds_adls-4.1.20250709174808.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_adls-4.1.20250709174808.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
+thds_adls-4.1.20250709174808.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_adls-4.1.20250709174808.dist-info/RECORD,,