thds.adls 4.1.20250701190349__py3-none-any.whl → 4.1.20250703020842__py3-none-any.whl

thds/adls/azcopy/download.py

@@ -29,7 +29,7 @@ logger = log.getLogger(__name__)
 @dataclass
 class DownloadRequest:
     temp_path: Path
-    size_bytes: int
+    size_bytes: ty.Optional[int]
 
 
 @dataclass
@@ -73,19 +73,24 @@ def sync_fastpath(
                 env=system_resources.restrict_usage(),
             )
             assert process.stdout
-            with progress.azcopy_tracker(dl_file_client.url, download_request.size_bytes) as track:
+            output_lines = list()
+            with progress.azcopy_tracker(dl_file_client.url, download_request.size_bytes or 0) as track:
                 for line in process.stdout:
                     track(line)
+                    output_lines.append(line.strip())
 
             process.wait()
             if process.returncode != 0:
-                raise subprocess.SubprocessError(f"AzCopy failed with return code {process.returncode}")
+                raise subprocess.CalledProcessError(
+                    process.returncode,
+                    f"AzCopy failed with return code {process.returncode}\n\n" + "\n".join(output_lines),
+                )
             assert (
                 download_request.temp_path.exists()
             ), f"AzCopy did not create the file at {download_request.temp_path}"
-            return  # success
+            return
 
-        except (subprocess.CalledProcessError, FileNotFoundError):
+        except (subprocess.SubprocessError, FileNotFoundError):
             logger.warning("Falling back to Python SDK for download")
 
     logger.debug("Downloading %s using Python SDK", dl_file_client.url)
@@ -121,19 +126,24 @@ async def async_fastpath(
             assert copy_proc.stdout
 
             # Feed lines to the tracker asynchronously
-            with progress.azcopy_tracker(dl_file_client.url, download_request.size_bytes) as track:
+            output_lines = list()
+            with progress.azcopy_tracker(dl_file_client.url, download_request.size_bytes or 0) as track:
                 while True:
                     line = await copy_proc.stdout.readline()
                     if not line:  # EOF
                         break
                     track(line.decode().strip())
+                    output_lines.append(line.decode().strip())
 
             # Wait for process completion
             exit_code = await copy_proc.wait()
             if exit_code != 0:
-                raise subprocess.SubprocessError()
+                raise subprocess.CalledProcessError(
+                    exit_code,
+                    f"AzCopy failed with return code {exit_code}\n\n" + "\n".join(output_lines),
+                )
 
-            return  # success
+            return
 
         except (subprocess.SubprocessError, FileNotFoundError):
             logger.warning("Falling back to Python SDK for download")

thds/adls/azcopy/upload.py

@@ -86,10 +86,15 @@ def run(
         env=system_resources.restrict_usage(),
     )
     assert process.stdout
+    output_lines = list()
     with progress.azcopy_tracker(uri.to_blob_windows_url(dest), size_bytes) as track:
         for line in process.stdout:
             track(line)
+            output_lines.append(line.strip())
 
     process.wait()
     if process.returncode != 0:
-        raise subprocess.SubprocessError(f"AzCopy failed with return code {process.returncode}")
+        raise subprocess.CalledProcessError(
+            process.returncode,
+            f"AzCopy failed with return code {process.returncode}\n\n" + "\n".join(output_lines),
+        )

thds/adls/download.py

@@ -7,6 +7,7 @@ import typing as ty
 from pathlib import Path
 
 import aiohttp.http_exceptions
+import requests.exceptions
 from azure.core.exceptions import AzureError, HttpResponseError, ResourceModifiedError
 from azure.storage.filedatalake import DataLakeFileClient, FileProperties, FileSystemClient, aio
 
@@ -22,27 +23,31 @@ from .ro_cache import Cache, from_cache_path_to_local, from_local_path_to_cache
 logger = log.getLogger(__name__)
 
 
+def _check_size(dpath: Path, expected_size: ty.Optional[int]) -> None:
+    actual_size = os.path.getsize(dpath)
+    if expected_size is not None and actual_size != expected_size:
+        raise errors.ContentLengthMismatchError(
+            f"Downloaded file {dpath} has size {actual_size} but expected {expected_size}"
+        )
+
+
 @contextlib.contextmanager
 def _atomic_download_and_move(
     fqn: AdlsFqn,
     dest: StrOrPath,
     properties: ty.Optional[FileProperties] = None,
 ) -> ty.Iterator[azcopy.download.DownloadRequest]:
-    known_size = (properties.size or 0) if properties else 0
+    known_size = properties.size if properties else None
     with tmp.temppath_same_fs(dest) as dpath:
         logger.debug("Downloading %s", fqn)
-        if azcopy.download.should_use_azcopy(known_size):
+        if azcopy.download.should_use_azcopy(known_size or -1):
             yield azcopy.download.DownloadRequest(dpath, known_size)
         else:
             with open(dpath, "wb") as down_f:
                 yield azcopy.download.SdkDownloadRequest(
-                    dpath, known_size, report_download_progress(down_f, str(fqn), known_size)
+                    dpath, known_size, report_download_progress(down_f, str(fqn), known_size or 0)
                 )
-        if known_size and os.path.getsize(dpath) != known_size:
-            raise errors.ContentLengthMismatchError(
-                f"Downloaded file {dpath} has size {os.path.getsize(dpath)}"
-                f" but expected {known_size}."
-            )
+        _check_size(dpath, known_size)
         try:
             os.rename(dpath, dest)  # will succeed even if dest is read-only
         except OSError as oserr:
@@ -303,7 +308,7 @@ def _excs_to_retry() -> ty.Callable[[Exception], bool]:
             filter(
                 None,
                 (
-                    errors.ContentLengthMismatchError,
+                    requests.exceptions.ConnectionError,
                     aiohttp.http_exceptions.ContentLengthError,
                     aiohttp.client_exceptions.ClientPayloadError,
                     getattr(
@@ -368,6 +373,9 @@ _async_dl_scope = scope.AsyncScope("adls.download.async")
 
 @_dl_scope.bound
 @_async_dl_scope.async_bound
+@fretry.retry_regular_async(
+    fretry.is_exc(errors.ContentLengthMismatchError), fretry.iter_to_async(fretry.n_times(2))
+)
 async def async_download_or_use_verified(
     fs_client: aio.FileSystemClient,
     remote_key: str,
@@ -381,9 +389,8 @@ async def async_download_or_use_verified(
         co, co_request, file_properties, dl_file_client = _prep_download_coroutine(
             fs_client, remote_key, local_path, expected_hash, cache
         )
-        await _async_dl_scope.async_enter(
-            dl_file_client  # type: ignore[arg-type]
-        )  # on __aexit__, will release the connection to the pool
+        await _async_dl_scope.async_enter(dl_file_client)  # type: ignore[arg-type]
+        # on __aexit__, will release the connection to the pool
         while True:
             if co_request == _IoRequest.FILE_PROPERTIES:
                 if not file_properties:
@@ -393,7 +400,6 @@ async def async_download_or_use_verified(
                 co_request = co.send(file_properties)
             elif isinstance(co_request, azcopy.download.DownloadRequest):
                 # coroutine is requesting download
-
                 await fretry.retry_regular_async(
                     _excs_to_retry(), fretry.iter_to_async(fretry.n_times(2))
                 )(

thds/adls/tools/download.py

@@ -16,9 +16,10 @@ def main():
         help="A fully qualified path to an ADLS location. Accepts adls://, https:// and abfss:// URIs.",
     )
     parser.add_argument(
-        "--copy-to",
-        "-c",
+        "copy_to",
+        nargs="?",
         type=Path,
+        default=None,
         help="This will create a link to the cached download at the specified location",
     )
     parser.add_argument(

thds/adls/upload.py

@@ -32,8 +32,11 @@ def _write_through_local_cache(local_cache_path: Path, data: UploadSrc) -> ty.Op
     @scope.bound
     def _try_write_through() -> bool:
         if isinstance(data, Path) and data.exists():
+            # we don't do hard or soft links because they share file permissions,
+            # and it's not up to us to change permissions on the src file.
             link.link_or_copy(data, local_cache_path, "ref")
             return True
+
         out = scope.enter(tmp.temppath_same_fs(local_cache_path))
         if hasattr(data, "read") and hasattr(data, "seek"):
             with open(out, "wb") as f:
@@ -41,11 +44,13 @@ def _write_through_local_cache(local_cache_path: Path, data: UploadSrc) -> ty.Op
             data.seek(0)  # type: ignore
             link.link_or_copy(out, local_cache_path)
             return True
+
         if isinstance(data, bytes):
             with open(out, "wb") as f:
                 f.write(data)
             link.link_or_copy(out, local_cache_path)
             return True
+
         return False
 
     if _try_write_through():
@@ -54,6 +59,7 @@ def _write_through_local_cache(local_cache_path: Path, data: UploadSrc) -> ty.Op
             # and we don't want to allow anyone to write to its copy.
             files.set_read_only(local_cache_path)
             return local_cache_path
+
         except FileNotFoundError:
             # may have hit a race condition.
             # don't fail upload just because we couldn't write through the cache.

{thds_adls-4.1.20250701190349.dist-info → thds_adls-4.1.20250703020842.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.adls
-Version: 4.1.20250701190349
+Version: 4.1.20250703020842
 Summary: ADLS tools
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

{thds_adls-4.1.20250701190349.dist-info → thds_adls-4.1.20250703020842.dist-info}/RECORD

@@ -7,7 +7,7 @@ thds/adls/conf.py,sha256=nTw3X1ilC3A_905jZH-rWXFsESeHAKQn5IghvfX2VIo,1991
 thds/adls/copy.py,sha256=jUWbGvTpb4B3yRGS0nhGSbDzqRPzUqYgH0z1lFRJB3k,6365
 thds/adls/dbfs.py,sha256=pPAjbIZRKJsaXKQljDMUgqS_zy1yKeEZHGMueXbuv3g,2219
 thds/adls/defaults.py,sha256=GGq5Pn4r-8cX4bZItp4nnwWAAz7S07pzPoOegw0y5Fw,676
-thds/adls/download.py,sha256=N8JqNqD5ioHsEHcTl2bNJt3Bb187yyvZAXn4xW3flfU,18090
+thds/adls/download.py,sha256=u-ckaExnTMAVUlhMbnxM_urBggbZkgXMQSSfp1czPeQ,18350
 thds/adls/download_lock.py,sha256=tgT48l4C5_qmArGeq05gl7VlxT22dZBH2Xwxx0itE9o,3176
 thds/adls/errors.py,sha256=6cLg2E4SB8ic46PBzA3ynRH4b1oR8qRb07RBgKGJRxY,1783
 thds/adls/etag.py,sha256=ct7jpHhNFcKzbekn5rZ3m6DhjK48A7qOZGwDiHkc-pc,242
@@ -24,19 +24,19 @@ thds/adls/sas_tokens.py,sha256=mArbB_GYohevOmArw_1gKqVUWpv6kG8Hsbvdrhbtnbg,1957
 thds/adls/shared_credential.py,sha256=-x42aXoIM001KW59oS8PpuXQd4-F2vg-1gB6OMHlpk4,4602
 thds/adls/source.py,sha256=8HVMYuxDn1XYGwFFSBowMlvQ6r2Jm2CQlpu4h85JvsE,2559
 thds/adls/source_tree.py,sha256=yP_v2XrKxXqUOdZ-x8kqHhBFAuur3AlAq3zi4hHj4AE,2235
-thds/adls/upload.py,sha256=gS_S66gorzdW83eavPUVJ3UYrv5u3HnftDXjdwEZOo8,6441
+thds/adls/upload.py,sha256=MRHK9Am-x5FKBPh1SXLTbPC1r0Xk0bGWNU8CcNuUMLo,6602
 thds/adls/uri.py,sha256=9MXuW_KfpPvzBc4ERxuTJ3vvi_6yr7e1kMAW9mx2zXM,1414
 thds/adls/azcopy/__init__.py,sha256=qn2dmT92EHcrtaQ8uwRoUgvtF6Fu3NQbhZItOBdIBmY,45
-thds/adls/azcopy/download.py,sha256=J7QAoBehpxsY58ofgGQur-MtIwM0NEnV9_Cw4i_X3y8,6007
+thds/adls/azcopy/download.py,sha256=MwkUaQTrrXRX9yip_hiLZXzSgGqyTDUxjr0MeXitWuo,6450
 thds/adls/azcopy/login.py,sha256=923UaewVMPFzkDSgCQsbl-_g7qdFhpXpF0MGNIy3T_A,1538
 thds/adls/azcopy/progress.py,sha256=K7TVmSiWfu561orL3GuOnlQX9VtVxWVECAq9NiweYNo,1387
 thds/adls/azcopy/system_resources.py,sha256=okgDEKAp0oWGQF7OKikbgJ9buBeiOgNaDYy-36j6dHo,761
-thds/adls/azcopy/upload.py,sha256=bvtYdbaFsZkOHFLDpeBlTKqw63P3_kbImInI04ZlekM,2601
-thds/adls/tools/download.py,sha256=Dmt-EBZUEF-gVfUcwjAD8VRKR5rhw-oozxl40lZHmdw,1562
+thds/adls/azcopy/upload.py,sha256=0l5FzV9IgZ2iQhm4eKZjTdw4SO17bHd8VnwcTev1lUs,2761
+thds/adls/tools/download.py,sha256=CW2cWbCRdUqisVVVoqqvqk5Ved7pPGTkwnZj3uV0jy4,1587
 thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
 thds/adls/tools/upload.py,sha256=5WyWkpuVp2PETZ3O3ODlq8LXszSHU73ZMnIDZXPJdC8,442
-thds_adls-4.1.20250701190349.dist-info/METADATA,sha256=gJNup1vZrpFp-0nor96kwmz__Ij_Zc5pkytWoEslYMU,587
-thds_adls-4.1.20250701190349.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_adls-4.1.20250701190349.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
-thds_adls-4.1.20250701190349.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_adls-4.1.20250701190349.dist-info/RECORD,,
+thds_adls-4.1.20250703020842.dist-info/METADATA,sha256=zWALY04Mrs1zZNpje2AyaiU-B6tN7La5ldtAbVEXxac,587
+thds_adls-4.1.20250703020842.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_adls-4.1.20250703020842.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
+thds_adls-4.1.20250703020842.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_adls-4.1.20250703020842.dist-info/RECORD,,