thds.adls 4.3.20251008013223__py3-none-any.whl → 4.3.20251009011718__py3-none-any.whl

thds/adls/_upload.py

@@ -38,17 +38,22 @@ def _try_default_hash(data: hashes.AnyStrSrc) -> ty.Optional[hashing.Hash]:
     return None
 
 
+UploadSrc = ty.Union[Path, bytes, ty.IO[bytes], ty.Iterable[bytes]]
+
+
+def upload_src_len(upload_src: UploadSrc, default: int = 0) -> int:
+    if isinstance(upload_src, Path) and upload_src.exists():
+        return upload_src.stat().st_size
+    try:
+        return len(upload_src)  # type: ignore
+    except TypeError as te:
+        logger.debug(f"failed to get length? {repr(te)} for {upload_src!r}")
+        return default
+
+
 def _too_small_to_skip_upload(data: hashes.AnyStrSrc, min_size_for_remote_check: int) -> bool:
-    def _len() -> int:
-        if isinstance(data, Path) and data.exists():
-            return data.stat().st_size
-        try:
-            return len(data)  # type: ignore
-        except TypeError as te:
-            logger.debug(f"failed to get length? {repr(te)} for {data}")
-            return min_size_for_remote_check + 1
-
-    return _len() < min_size_for_remote_check
+    len_ = upload_src_len(data) or min_size_for_remote_check + 1
+    return len_ < min_size_for_remote_check
 
 
 class UploadDecision(ty.NamedTuple):

thds/adls/source.py

@@ -30,7 +30,9 @@ def _adls_uri_source_download_handler(uri: str) -> ty.Optional[source.Downloader
 source.register_download_handler("thds.adls", _adls_uri_source_download_handler)
 
 
-def from_adls(uri_or_fqn: ty.Union[str, AdlsFqn], hash: ty.Optional[Hash] = None) -> source.Source:
+def from_adls(
+    uri_or_fqn: ty.Union[str, AdlsFqn], hash: ty.Optional[Hash] = None, size: int = 0
+) -> source.Source:
     """Flexible, public interface to creating Sources from any ADLS-like reference.
 
     Does NOT automatically fetch a checksumming hash from the ADLS URI if it's not
@@ -40,7 +42,7 @@ def from_adls(uri_or_fqn: ty.Union[str, AdlsFqn], hash: ty.Optional[Hash] = None
     r_fqn = resolve_any(uri_or_fqn)
     if not r_fqn:
         raise ValueError(f"Could not resolve {uri_or_fqn} to an ADLS FQN")
-    return source.Source(str(r_fqn), hash)
+    return source.Source(str(r_fqn), hash, size)
 
 
 source.register_from_uri_handler(
@@ -55,13 +57,15 @@ def get_with_hash(fqn_or_uri: ty.Union[AdlsFqn, str]) -> source.Source:
     """
     fqn = AdlsFqn.parse(fqn_or_uri) if isinstance(fqn_or_uri, str) else fqn_or_uri
     with blob_not_found_translation(fqn):
-        uri_hashes = hashes.extract_hashes_from_props(get_file_properties(fqn))
+        props = get_file_properties(fqn)
+        uri_hashes = hashes.extract_hashes_from_props(props)
         if not uri_hashes:
             raise ValueError(
                 f"ADLS file {fqn} must have a hash to use this function. "
                 "If you know the hash, use `from_adls` with the hash parameter."
             )
-        return from_adls(fqn, next(iter(uri_hashes.values())))
+        size = int(props.get("size")) or 0
+        return from_adls(fqn, next(iter(uri_hashes.values())), size)
 
 
 def with_md5b64(uri_or_fqn: ty.Union[str, AdlsFqn], *, md5b64: str = "") -> source.Source:

thds/adls/source_tree.py

@@ -60,7 +60,7 @@ def from_path(adls_path: uri.UriIsh, match_suffix: str = "") -> SourceTree:
     container_root = root_fqn.root()
     return SourceTree(
         sources=[
-            source.from_adls(container_root / blob_meta.path, hash=blob_meta.hash)
+            source.from_adls(container_root / blob_meta.path, hash=blob_meta.hash, size=blob_meta.size)
             for blob_meta in list_blob_meta(container_client, root_fqn.path, match_suffix=match_suffix)
         ],
         higher_logical_root=fqn.split(root_fqn)[-1],

thds/adls/upload.py

@@ -15,7 +15,7 @@ from thds.core import files, fretry, link, log, scope, source, tmp
 
 from . import azcopy, hashes
 from ._progress import report_upload_progress
-from ._upload import upload_decision_and_metadata
+from ._upload import UploadSrc, upload_decision_and_metadata, upload_src_len
 from .conf import UPLOAD_FILE_MAX_CONCURRENCY
 from .file_lock import file_lock
 from .fqn import AdlsFqn
@@ -26,9 +26,6 @@ logger = log.getLogger(__name__)
 _SLOW_CONNECTION_WORKAROUND = 14400  # seconds
 
 
-UploadSrc = ty.Union[Path, bytes, ty.IO[ty.AnyStr], ty.Iterable[bytes]]
-
-
 def _write_through_local_cache(local_cache_path: Path, data: UploadSrc) -> ty.Optional[Path]:
     @scope.bound
     def _try_write_through() -> bool:
@@ -41,8 +38,8 @@ def _write_through_local_cache(local_cache_path: Path, data: UploadSrc) -> ty.Op
         out = scope.enter(tmp.temppath_same_fs(local_cache_path))
         if hasattr(data, "read") and hasattr(data, "seek"):
             with open(out, "wb") as f:
-                f.write(data.read())  # type: ignore
-            data.seek(0)  # type: ignore
+                f.write(data.read())
+            data.seek(0)
             link.link_or_copy(out, local_cache_path)
             return True
 
@@ -107,6 +104,7 @@ def upload(
     blob_container_client = get_global_blob_container_client(dest_.sa, dest_.container)
     blob_client = blob_container_client.get_blob_client(dest_.path)
     decision = upload_decision_and_metadata(blob_client.get_blob_properties, src)  # type: ignore [arg-type]
+    n_bytes = upload_src_len(src, default=0)
 
     def source_from_meta() -> source.Source:
         best_hash = next(iter(hashes.extract_hashes_from_metadata(decision.metadata)), None)
@@ -114,17 +112,14 @@ def upload(
             assert best_hash, "A hash should always be calculable for a local path."
             return source.from_file(src, hash=best_hash, uri=str(dest_))
 
-        return source.from_uri(str(dest_), hash=best_hash)
+        return source.from_uri(str(dest_), hash=best_hash, size=n_bytes)
 
     if decision.upload_required:
         # set up some bookkeeping
-        n_bytes = None  # if we pass 0 to upload_blob, it truncates the write now
         bytes_src: ty.Union[bytes, ty.IO, ty.Iterable[bytes]]
         if isinstance(src, Path):
-            n_bytes = src.stat().st_size
             bytes_src = scope.enter(open(src, "rb"))
         elif isinstance(src, bytes):
-            n_bytes = len(src)
             bytes_src = src
         else:
             bytes_src = src
@@ -132,7 +127,7 @@ def upload(
         if "metadata" in upload_data_kwargs:
             decision.metadata.update(upload_data_kwargs.pop("metadata"))
 
-        if azcopy.upload.should_use_azcopy(n_bytes or 0) and isinstance(src, Path):
+        if azcopy.upload.should_use_azcopy(n_bytes) and isinstance(src, Path):
             logger.info("Using azcopy to upload %s to %s", src, dest_)
             try:
                 azcopy.upload.run(
@@ -140,7 +135,7 @@ def upload(
                         src, dest_, content_type=content_type, metadata=decision.metadata, overwrite=True
                     ),
                     dest_,
-                    n_bytes or 0,
+                    n_bytes,
                 )
                 return source_from_meta()
 
@@ -158,9 +153,11 @@ def upload(
         # This is both faster, as well as simpler to reason about, and
         # in fact was the behavior I had been assuming all along...
         blob_client.upload_blob(
-            report_upload_progress(ty.cast(ty.IO, bytes_src), str(dest_), n_bytes or 0),
+            report_upload_progress(ty.cast(ty.IO, bytes_src), str(dest_), n_bytes),
             overwrite=True,
-            length=n_bytes,
+            length=(
+                n_bytes if n_bytes > 0 else None
+            ),  # if we pass 0 to upload_blob, it truncates the write now
             content_settings=upload_content_settings,
             connection_timeout=_SLOW_CONNECTION_WORKAROUND,
             max_concurrency=UPLOAD_FILE_MAX_CONCURRENCY(),

{thds_adls-4.3.20251008013223.dist-info → thds_adls-4.3.20251009011718.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.adls
-Version: 4.3.20251008013223
+Version: 4.3.20251009011718
 Summary: ADLS tools
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

{thds_adls-4.3.20251008013223.dist-info → thds_adls-4.3.20251009011718.dist-info}/RECORD

@@ -1,7 +1,7 @@
 thds/adls/__init__.py,sha256=Z450uhM5BIxgve9qkz_Lw7biVWFKp3Uz_iKaSPYEf1Y,1073
 thds/adls/_fork_protector.py,sha256=WML1Ul-G_MCziBOyRp6GoNq1IGktn8-OQVjny1IZ1bs,950
 thds/adls/_progress.py,sha256=0cmFZ2sOEogQY25AkMAhcunXO69FdlbkD1rU_zmLAp0,6648
-thds/adls/_upload.py,sha256=mhTdWiQroaugYuwQg7R8CEgdfCYF4xvJthlsqO0jlnE,4692
+thds/adls/_upload.py,sha256=rN-nuRZur9VvSUywaQezUi_qedj-xU30fTBG6rasuIA,4853
 thds/adls/abfss.py,sha256=nCFfcdwLiwtz_MdbrpLJ9WOhoz0zHIVxsf-tarorEwM,727
 thds/adls/cached.py,sha256=up1F5JOVXdmwdZ8RAB2UDgiy6ooLg8IMULohBh75VpQ,3034
 thds/adls/conf.py,sha256=nTw3X1ilC3A_905jZH-rWXFsESeHAKQn5IghvfX2VIo,1991
@@ -24,9 +24,9 @@ thds/adls/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/adls/ro_cache.py,sha256=P-UVFZhqnE5wojqYmRVWZcqjl-pM1xVMm9VAm3nXlnA,4769
 thds/adls/sas_tokens.py,sha256=mArbB_GYohevOmArw_1gKqVUWpv6kG8Hsbvdrhbtnbg,1957
 thds/adls/shared_credential.py,sha256=-x42aXoIM001KW59oS8PpuXQd4-F2vg-1gB6OMHlpk4,4602
-thds/adls/source.py,sha256=8HVMYuxDn1XYGwFFSBowMlvQ6r2Jm2CQlpu4h85JvsE,2559
-thds/adls/source_tree.py,sha256=gl2JLjxAduo4cGQBb8LqBnmRHHk2wqIC5yt-sqkXOEo,2589
-thds/adls/upload.py,sha256=7eWPpPuIgCBb3Svk05K1UNv0376q8wDTQkpTaKLtg-w,6694
+thds/adls/source.py,sha256=5t9bc3tUxdh4QdbRvvVd9xF9_Rv4v7CDAqE09M3rE1g,2657
+thds/adls/source_tree.py,sha256=FqVXgvfYPiowrWhRsXBItjvB7t41JRI3sCFVAHxjwgI,2610
+thds/adls/upload.py,sha256=XJvygYzn1r6pZAl460s5j-kY9dgfQ6qeoddID7R6OQ0,6621
 thds/adls/uri.py,sha256=9MXuW_KfpPvzBc4ERxuTJ3vvi_6yr7e1kMAW9mx2zXM,1414
 thds/adls/azcopy/__init__.py,sha256=qn2dmT92EHcrtaQ8uwRoUgvtF6Fu3NQbhZItOBdIBmY,45
 thds/adls/azcopy/download.py,sha256=FOtYyYh7ZXNWNdkj04yTV26lxcKOVj-YhS2p_EclYxA,6526
@@ -38,8 +38,8 @@ thds/adls/tools/download.py,sha256=CW2cWbCRdUqisVVVoqqvqk5Ved7pPGTkwnZj3uV0jy4,1
 thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
 thds/adls/tools/ls_fast.py,sha256=Nowc-efAL_Y4ybPwZzKIeh7KGIjfecRzdWvJZcBzq_8,585
 thds/adls/tools/upload.py,sha256=5WyWkpuVp2PETZ3O3ODlq8LXszSHU73ZMnIDZXPJdC8,442
-thds_adls-4.3.20251008013223.dist-info/METADATA,sha256=YNBtKknKx9Hv62k3anREQ5VBI6_KY5LoZ6h68aNLMt8,587
-thds_adls-4.3.20251008013223.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_adls-4.3.20251008013223.dist-info/entry_points.txt,sha256=rtVF0A2MMTYUsBScF6b3AlOuk2Vm02QK7Tc2bDcDpk0,200
-thds_adls-4.3.20251008013223.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_adls-4.3.20251008013223.dist-info/RECORD,,
+thds_adls-4.3.20251009011718.dist-info/METADATA,sha256=H8nvvzUdcM2NU_NQVAn7zco92Buk26EqGkqj2O4NVjk,587
+thds_adls-4.3.20251009011718.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_adls-4.3.20251009011718.dist-info/entry_points.txt,sha256=rtVF0A2MMTYUsBScF6b3AlOuk2Vm02QK7Tc2bDcDpk0,200
+thds_adls-4.3.20251009011718.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_adls-4.3.20251009011718.dist-info/RECORD,,