thds.adls 4.1.20250722213940__py3-none-any.whl → 4.1.20250724233711__py3-none-any.whl

thds/adls/__init__.py

@@ -5,7 +5,7 @@ from .cached import download_directory, download_to_cache, upload_through_cache
 from .copy import copy_file, copy_files, wait_for_copy  # noqa: F401
 from .errors import BlobNotFoundError  # noqa: F401
 from .fqn import *  # noqa: F401,F403
-from .global_client import get_global_client, get_global_fs_client  # noqa: F401
+from .global_client import get_global_fs_client  # noqa: F401
 from .impl import *  # noqa: F401,F403
 from .ro_cache import Cache, global_cache  # noqa: F401
 from .upload import upload  # noqa: F401

thds/adls/_fork_protector.py

@@ -0,0 +1,38 @@
+import os
+import typing as ty
+from functools import partial
+
+from typing_extensions import Concatenate, ParamSpec
+
+P = ParamSpec("P")
+T = ty.TypeVar("T")
+F = ty.TypeVar("F", bound=ty.Callable)
+
+
+def _pid_swallower(
+    func: ty.Callable[P, T],
+    pid: int,
+    *args: P.args,
+    **kwargs: P.kwargs,
+) -> T:
+    return func(*args, **kwargs)
+
+
+def _pid_sending_wrapper(
+    caching_pid_swallower: ty.Callable[Concatenate[int, P], T],
+    *args: P.args,
+    **kwargs: P.kwargs,
+) -> T:
+    return caching_pid_swallower(os.getpid(), *args, **kwargs)
+
+
+def fork_safe_cached(
+    cache_deco: ty.Callable[[F], F],
+    func: ty.Callable[P, T],
+) -> ty.Callable[P, T]:
+    """Decorator to make a fork-safe cached.locking function by wrapping it in a function that
+    always calls os.getpid() to invalidate the cache on new processes."""
+    return partial(
+        _pid_sending_wrapper,
+        cache_deco(ty.cast(F, partial(_pid_swallower, func))),
+    )

thds/adls/download.py

@@ -158,6 +158,12 @@ IoRequest = ty.Union[_IoRequest, azcopy.download.DownloadRequest]
 IoResponse = ty.Union[FileProperties, None]
 
 
+def _assert_fp(fp: ty.Optional[FileProperties], fqn: AdlsFqn) -> None:
+    assert fp, f"FileProperties for {fqn} should not be None."
+    assert fp.name, f"FileProperties for {fqn} should have a name."
+    assert fp.name == fqn.path, (fp, fqn)
+
+
 _dl_scope = scope.Scope("adls.download")
 
 
@@ -218,6 +224,7 @@ def _download_or_use_verified_cached_coroutine(  # noqa: C901
         # expectations from ADLS itself.
         file_properties = yield _IoRequest.FILE_PROPERTIES
         if file_properties:
+            _assert_fp(file_properties, fqn)
             # critically, we expect the _first_ one in this list to be the fastest to verify.
             expected_hash = next(iter(hashes.extract_hashes_from_props(file_properties).values()), None)
 
@@ -247,6 +254,7 @@ def _download_or_use_verified_cached_coroutine(  # noqa: C901
 
     logger.debug("Unable to find a cached version anywhere that we looked...")
     file_properties = yield _IoRequest.FILE_PROPERTIES
+    _assert_fp(file_properties, fqn)
 
     # if any of the remote hashes match the expected hash, verify that one.
     # otherwise, verify the first remote hash in the list, since that's the fastest one.
@@ -286,19 +294,14 @@ def _prep_download_coroutine(
     local_path: StrOrPath,
     expected_hash: ty.Optional[hashing.Hash] = None,
     cache: ty.Optional[Cache] = None,
-) -> ty.Tuple[
-    ty.Generator[IoRequest, IoResponse, _FileResult],
-    IoRequest,
-    ty.Optional[FileProperties],
-    DataLakeFileClient,
-]:
+) -> ty.Tuple[ty.Generator[IoRequest, IoResponse, _FileResult], IoRequest, DataLakeFileClient]:
     co = _download_or_use_verified_cached_coroutine(
         AdlsFqn(ty.cast(str, fs_client.account_name), fs_client.file_system_name, remote_key),
         local_path,
         expected_hash=expected_hash,
         cache=cache,
     )
-    return co, co.send(None), None, fs_client.get_file_client(remote_key)
+    return co, co.send(None), fs_client.get_file_client(remote_key)
 
 
 def _excs_to_retry() -> ty.Callable[[Exception], bool]:
@@ -333,6 +336,7 @@ def _log_nonfatal_hash_error_exc(exc: Exception, url: str) -> None:
 
 
 @_dl_scope.bound
+@fretry.retry_regular(fretry.is_exc(errors.ContentLengthMismatchError), fretry.n_times(2))
 def download_or_use_verified(
     fs_client: FileSystemClient,
     remote_key: str,
@@ -348,14 +352,16 @@ def download_or_use_verified(
     """
     file_properties = None
     try:
-        co, co_request, file_properties, dl_file_client = _prep_download_coroutine(
+        co, co_request, dl_file_client = _prep_download_coroutine(
             fs_client, remote_key, local_path, expected_hash, cache
         )
+        assert dl_file_client.path_name == remote_key
         _dl_scope.enter(dl_file_client)  # on __exit__, will release the connection to the pool
         while True:
             if co_request == _IoRequest.FILE_PROPERTIES:
                 if not file_properties:
                     # only fetch these if they haven't already been requested
+                    assert dl_file_client.path_name == remote_key
                     file_properties = dl_file_client.get_file_properties()
                 co_request = co.send(file_properties)
             elif isinstance(co_request, azcopy.download.DownloadRequest):
@@ -398,7 +404,7 @@ async def async_download_or_use_verified(
 ) -> ty.Optional[Path]:
     file_properties = None
     try:
-        co, co_request, file_properties, dl_file_client = _prep_download_coroutine(
+        co, co_request, dl_file_client = _prep_download_coroutine(
             fs_client, remote_key, local_path, expected_hash, cache
         )
         await _async_dl_scope.async_enter(dl_file_client)  # type: ignore[arg-type]

thds/adls/global_client.py

@@ -4,7 +4,7 @@ from azure.storage.filedatalake import DataLakeServiceClient, FileSystemClient
 
 from thds.core import cache, config
 
-from . import conf
+from . import _fork_protector, conf
 from .shared_credential import SharedCredential
 
 DEFAULT_CONNECTION_POOL_SIZE = config.item("default_connection_pool_size", default=100, parse=int)
@@ -53,9 +53,7 @@ best approach for all applications.
 This avoids creating a client at a module level and is
 thread-safe.
 """
-get_global_client = cache.locking(adls_fs_client)
-# deprecated name - prefer get_global_fs_client
-get_global_fs_client = get_global_client
+get_global_fs_client = _fork_protector.fork_safe_cached(cache.locking, adls_fs_client)
 
 
 def adls_blob_service_client(
@@ -84,4 +82,6 @@ def adls_blob_container_client(
     return get_global_blob_service_client(storage_account, connpool_size).get_container_client(container)
 
 
-get_global_blob_container_client = cache.locking(adls_blob_container_client)
+get_global_blob_container_client = _fork_protector.fork_safe_cached(
+    cache.locking, adls_blob_container_client
+)

{thds_adls-4.1.20250722213940.dist-info → thds_adls-4.1.20250724233711.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.adls
-Version: 4.1.20250722213940
+Version: 4.1.20250724233711
 Summary: ADLS tools
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

{thds_adls-4.1.20250722213940.dist-info → thds_adls-4.1.20250724233711.dist-info}/RECORD

@@ -1,4 +1,5 @@
-thds/adls/__init__.py,sha256=g2Zb0EAAH-JzPMYHAub9liU4qa5pfqQDnILfEhmObGo,1036
+thds/adls/__init__.py,sha256=PL0BRhiLhW_xY_2hhBgd8v3_NS1zpR4Kdd28zjNHBgo,1017
+thds/adls/_fork_protector.py,sha256=WML1Ul-G_MCziBOyRp6GoNq1IGktn8-OQVjny1IZ1bs,950
 thds/adls/_progress.py,sha256=D6XIipzG_xwmxs_08LuiYFfThGqHTU2KiIyjNduiOFY,6656
 thds/adls/_upload.py,sha256=mhTdWiQroaugYuwQg7R8CEgdfCYF4xvJthlsqO0jlnE,4692
 thds/adls/abfss.py,sha256=ZRJOLjDuXmS4bIbQAQpQxWWWeu74N9NKEKCNfXQek80,726
@@ -7,13 +8,13 @@ thds/adls/conf.py,sha256=nTw3X1ilC3A_905jZH-rWXFsESeHAKQn5IghvfX2VIo,1991
 thds/adls/copy.py,sha256=jUWbGvTpb4B3yRGS0nhGSbDzqRPzUqYgH0z1lFRJB3k,6365
 thds/adls/dbfs.py,sha256=pPAjbIZRKJsaXKQljDMUgqS_zy1yKeEZHGMueXbuv3g,2219
 thds/adls/defaults.py,sha256=GGq5Pn4r-8cX4bZItp4nnwWAAz7S07pzPoOegw0y5Fw,676
-thds/adls/download.py,sha256=WOpMXGUbWImBdkM4tSW7qnCbu7G_cRXKF5pFQVLPPxs,18772
+thds/adls/download.py,sha256=VmvkI3c0bAxVF2B7dBqHpSL18a701ddFNFCwpJSfdF4,19223
 thds/adls/download_lock.py,sha256=tgT48l4C5_qmArGeq05gl7VlxT22dZBH2Xwxx0itE9o,3176
 thds/adls/errors.py,sha256=6NMLHtVNsWBRDXaes9yzHj9cwKOD9t1dwL4BltdtjhU,1895
 thds/adls/etag.py,sha256=ct7jpHhNFcKzbekn5rZ3m6DhjK48A7qOZGwDiHkc-pc,242
 thds/adls/file_properties.py,sha256=dhRtbsMNOYfExkEiy76wrLfrJ6IMQeN1Z3LIxgKceqY,2042
 thds/adls/fqn.py,sha256=0zHmHhBWN7GEfKRB3fBC1NVhaiIHHifBdCRanyT01X8,5822
-thds/adls/global_client.py,sha256=f4VJw5y_Yh__8gQUcdSYTh1aU6iEPlauMchVirSAwDQ,3716
+thds/adls/global_client.py,sha256=q6oG1OOsfl4fbti81u8TE9d4Jx9-phlYgsGSc4032w8,3721
 thds/adls/hashes.py,sha256=-wRRATGmww7k2RD5Zmhq_Fq7Z2JihLy1njeHFekU15c,5316
 thds/adls/impl.py,sha256=cNf1vmeS46X_wvyVdDJ8qFfowHn2QwtU5C80BmDtu5Y,43247
 thds/adls/md5.py,sha256=hGT8AIX32VUsnRCbm8cel9OlxAiRrgjwNWQTqRDHM_k,374
@@ -35,8 +36,8 @@ thds/adls/azcopy/upload.py,sha256=RQLDJzS6qsMM12t5bykWJWBXs0UrmImrEFnPMxX2UlM,27
 thds/adls/tools/download.py,sha256=CW2cWbCRdUqisVVVoqqvqk5Ved7pPGTkwnZj3uV0jy4,1587
 thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
 thds/adls/tools/upload.py,sha256=5WyWkpuVp2PETZ3O3ODlq8LXszSHU73ZMnIDZXPJdC8,442
-thds_adls-4.1.20250722213940.dist-info/METADATA,sha256=4HRrxzhTRvxmMQS3S6cNKEa2PHvvfi1LJyrvF9ISyHI,587
-thds_adls-4.1.20250722213940.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_adls-4.1.20250722213940.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
-thds_adls-4.1.20250722213940.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_adls-4.1.20250722213940.dist-info/RECORD,,
+thds_adls-4.1.20250724233711.dist-info/METADATA,sha256=__N8wqRc6f1Ib1mvztYicgsh5F7OISGYwNZcpgtsjZo,587
+thds_adls-4.1.20250724233711.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_adls-4.1.20250724233711.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
+thds_adls-4.1.20250724233711.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_adls-4.1.20250724233711.dist-info/RECORD,,