thds.adls 4.1.20250722200042__py3-none-any.whl → 4.1.20250722213940__py3-none-any.whl

thds/adls/__init__.py

@@ -5,7 +5,7 @@ from .cached import download_directory, download_to_cache, upload_through_cache
 from .copy import copy_file, copy_files, wait_for_copy  # noqa: F401
 from .errors import BlobNotFoundError  # noqa: F401
 from .fqn import *  # noqa: F401,F403
-from .global_client import get_global_fs_client  # noqa: F401
+from .global_client import get_global_client, get_global_fs_client  # noqa: F401
 from .impl import *  # noqa: F401,F403
 from .ro_cache import Cache, global_cache  # noqa: F401
 from .upload import upload  # noqa: F401

thds/adls/download.py

@@ -158,12 +158,6 @@ IoRequest = ty.Union[_IoRequest, azcopy.download.DownloadRequest]
 IoResponse = ty.Union[FileProperties, None]
 
 
-def _assert_fp(fp: ty.Optional[FileProperties], fqn: AdlsFqn) -> None:
-    assert fp, f"FileProperties for {fqn} should not be None."
-    assert fp.name, f"FileProperties for {fqn} should have a name."
-    assert fp.name == fqn.path, (fp, fqn)
-
-
 _dl_scope = scope.Scope("adls.download")
 
 
@@ -224,7 +218,6 @@ def _download_or_use_verified_cached_coroutine(  # noqa: C901
         # expectations from ADLS itself.
         file_properties = yield _IoRequest.FILE_PROPERTIES
         if file_properties:
-            _assert_fp(file_properties, fqn)
             # critically, we expect the _first_ one in this list to be the fastest to verify.
             expected_hash = next(iter(hashes.extract_hashes_from_props(file_properties).values()), None)
 
@@ -254,7 +247,6 @@ def _download_or_use_verified_cached_coroutine(  # noqa: C901
 
     logger.debug("Unable to find a cached version anywhere that we looked...")
     file_properties = yield _IoRequest.FILE_PROPERTIES
-    _assert_fp(file_properties, fqn)
 
     # if any of the remote hashes match the expected hash, verify that one.
     # otherwise, verify the first remote hash in the list, since that's the fastest one.
@@ -294,14 +286,19 @@ def _prep_download_coroutine(
     local_path: StrOrPath,
     expected_hash: ty.Optional[hashing.Hash] = None,
     cache: ty.Optional[Cache] = None,
-) -> ty.Tuple[ty.Generator[IoRequest, IoResponse, _FileResult], IoRequest, DataLakeFileClient]:
+) -> ty.Tuple[
+    ty.Generator[IoRequest, IoResponse, _FileResult],
+    IoRequest,
+    ty.Optional[FileProperties],
+    DataLakeFileClient,
+]:
     co = _download_or_use_verified_cached_coroutine(
         AdlsFqn(ty.cast(str, fs_client.account_name), fs_client.file_system_name, remote_key),
         local_path,
         expected_hash=expected_hash,
         cache=cache,
     )
-    return co, co.send(None), fs_client.get_file_client(remote_key)
+    return co, co.send(None), None, fs_client.get_file_client(remote_key)
 
 
 def _excs_to_retry() -> ty.Callable[[Exception], bool]:
@@ -336,7 +333,6 @@ def _log_nonfatal_hash_error_exc(exc: Exception, url: str) -> None:
 
 
 @_dl_scope.bound
-@fretry.retry_regular(fretry.is_exc(errors.ContentLengthMismatchError), fretry.n_times(2))
 def download_or_use_verified(
     fs_client: FileSystemClient,
     remote_key: str,
@@ -352,16 +348,14 @@ def download_or_use_verified(
     """
     file_properties = None
     try:
-        co, co_request, dl_file_client = _prep_download_coroutine(
+        co, co_request, file_properties, dl_file_client = _prep_download_coroutine(
             fs_client, remote_key, local_path, expected_hash, cache
         )
-        assert dl_file_client.path_name == remote_key
         _dl_scope.enter(dl_file_client)  # on __exit__, will release the connection to the pool
         while True:
             if co_request == _IoRequest.FILE_PROPERTIES:
                 if not file_properties:
                     # only fetch these if they haven't already been requested
-                    assert dl_file_client.path_name == remote_key
                     file_properties = dl_file_client.get_file_properties()
                 co_request = co.send(file_properties)
             elif isinstance(co_request, azcopy.download.DownloadRequest):
@@ -404,7 +398,7 @@ async def async_download_or_use_verified(
 ) -> ty.Optional[Path]:
     file_properties = None
     try:
-        co, co_request, dl_file_client = _prep_download_coroutine(
+        co, co_request, file_properties, dl_file_client = _prep_download_coroutine(
             fs_client, remote_key, local_path, expected_hash, cache
         )
         await _async_dl_scope.async_enter(dl_file_client)  # type: ignore[arg-type]

thds/adls/global_client.py

@@ -4,7 +4,7 @@ from azure.storage.filedatalake import DataLakeServiceClient, FileSystemClient
 
 from thds.core import cache, config
 
-from . import _fork_protector, conf
+from . import conf
 from .shared_credential import SharedCredential
 
 DEFAULT_CONNECTION_POOL_SIZE = config.item("default_connection_pool_size", default=100, parse=int)
@@ -53,7 +53,9 @@ best approach for all applications.
 This avoids creating a client at a module level and is
 thread-safe.
 """
-get_global_fs_client = _fork_protector.fork_safe_cached(cache.locking, adls_fs_client)
+get_global_client = cache.locking(adls_fs_client)
+# deprecated name - prefer get_global_fs_client
+get_global_fs_client = get_global_client
 
 
 def adls_blob_service_client(
@@ -82,6 +84,4 @@ def adls_blob_container_client(
     return get_global_blob_service_client(storage_account, connpool_size).get_container_client(container)
 
 
-get_global_blob_container_client = _fork_protector.fork_safe_cached(
-    cache.locking, adls_blob_container_client
-)
+get_global_blob_container_client = cache.locking(adls_blob_container_client)

{thds_adls-4.1.20250722200042.dist-info → thds_adls-4.1.20250722213940.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.adls
-Version: 4.1.20250722200042
+Version: 4.1.20250722213940
 Summary: ADLS tools
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

{thds_adls-4.1.20250722200042.dist-info → thds_adls-4.1.20250722213940.dist-info}/RECORD

@@ -1,5 +1,4 @@
-thds/adls/__init__.py,sha256=PL0BRhiLhW_xY_2hhBgd8v3_NS1zpR4Kdd28zjNHBgo,1017
-thds/adls/_fork_protector.py,sha256=WML1Ul-G_MCziBOyRp6GoNq1IGktn8-OQVjny1IZ1bs,950
+thds/adls/__init__.py,sha256=g2Zb0EAAH-JzPMYHAub9liU4qa5pfqQDnILfEhmObGo,1036
 thds/adls/_progress.py,sha256=D6XIipzG_xwmxs_08LuiYFfThGqHTU2KiIyjNduiOFY,6656
 thds/adls/_upload.py,sha256=mhTdWiQroaugYuwQg7R8CEgdfCYF4xvJthlsqO0jlnE,4692
 thds/adls/abfss.py,sha256=ZRJOLjDuXmS4bIbQAQpQxWWWeu74N9NKEKCNfXQek80,726
@@ -8,13 +7,13 @@ thds/adls/conf.py,sha256=nTw3X1ilC3A_905jZH-rWXFsESeHAKQn5IghvfX2VIo,1991
 thds/adls/copy.py,sha256=jUWbGvTpb4B3yRGS0nhGSbDzqRPzUqYgH0z1lFRJB3k,6365
 thds/adls/dbfs.py,sha256=pPAjbIZRKJsaXKQljDMUgqS_zy1yKeEZHGMueXbuv3g,2219
 thds/adls/defaults.py,sha256=GGq5Pn4r-8cX4bZItp4nnwWAAz7S07pzPoOegw0y5Fw,676
-thds/adls/download.py,sha256=VmvkI3c0bAxVF2B7dBqHpSL18a701ddFNFCwpJSfdF4,19223
+thds/adls/download.py,sha256=WOpMXGUbWImBdkM4tSW7qnCbu7G_cRXKF5pFQVLPPxs,18772
 thds/adls/download_lock.py,sha256=tgT48l4C5_qmArGeq05gl7VlxT22dZBH2Xwxx0itE9o,3176
 thds/adls/errors.py,sha256=6NMLHtVNsWBRDXaes9yzHj9cwKOD9t1dwL4BltdtjhU,1895
 thds/adls/etag.py,sha256=ct7jpHhNFcKzbekn5rZ3m6DhjK48A7qOZGwDiHkc-pc,242
 thds/adls/file_properties.py,sha256=dhRtbsMNOYfExkEiy76wrLfrJ6IMQeN1Z3LIxgKceqY,2042
 thds/adls/fqn.py,sha256=0zHmHhBWN7GEfKRB3fBC1NVhaiIHHifBdCRanyT01X8,5822
-thds/adls/global_client.py,sha256=q6oG1OOsfl4fbti81u8TE9d4Jx9-phlYgsGSc4032w8,3721
+thds/adls/global_client.py,sha256=f4VJw5y_Yh__8gQUcdSYTh1aU6iEPlauMchVirSAwDQ,3716
 thds/adls/hashes.py,sha256=-wRRATGmww7k2RD5Zmhq_Fq7Z2JihLy1njeHFekU15c,5316
 thds/adls/impl.py,sha256=cNf1vmeS46X_wvyVdDJ8qFfowHn2QwtU5C80BmDtu5Y,43247
 thds/adls/md5.py,sha256=hGT8AIX32VUsnRCbm8cel9OlxAiRrgjwNWQTqRDHM_k,374
@@ -36,8 +35,8 @@ thds/adls/azcopy/upload.py,sha256=RQLDJzS6qsMM12t5bykWJWBXs0UrmImrEFnPMxX2UlM,27
 thds/adls/tools/download.py,sha256=CW2cWbCRdUqisVVVoqqvqk5Ved7pPGTkwnZj3uV0jy4,1587
 thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
 thds/adls/tools/upload.py,sha256=5WyWkpuVp2PETZ3O3ODlq8LXszSHU73ZMnIDZXPJdC8,442
-thds_adls-4.1.20250722200042.dist-info/METADATA,sha256=zExlhcfTWnL3I8pX3K_AAWuw9q61OucuXRpBHYA_3OM,587
-thds_adls-4.1.20250722200042.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_adls-4.1.20250722200042.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
-thds_adls-4.1.20250722200042.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_adls-4.1.20250722200042.dist-info/RECORD,,
+thds_adls-4.1.20250722213940.dist-info/METADATA,sha256=4HRrxzhTRvxmMQS3S6cNKEa2PHvvfi1LJyrvF9ISyHI,587
+thds_adls-4.1.20250722213940.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_adls-4.1.20250722213940.dist-info/entry_points.txt,sha256=uTqreT1AIwqJboMfLv5w6sviM8mNbAkln765gIjzoA4,152
+thds_adls-4.1.20250722213940.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_adls-4.1.20250722213940.dist-info/RECORD,,

thds/adls/_fork_protector.py

@@ -1,38 +0,0 @@
-import os
-import typing as ty
-from functools import partial
-
-from typing_extensions import Concatenate, ParamSpec
-
-P = ParamSpec("P")
-T = ty.TypeVar("T")
-F = ty.TypeVar("F", bound=ty.Callable)
-
-
-def _pid_swallower(
-    func: ty.Callable[P, T],
-    pid: int,
-    *args: P.args,
-    **kwargs: P.kwargs,
-) -> T:
-    return func(*args, **kwargs)
-
-
-def _pid_sending_wrapper(
-    caching_pid_swallower: ty.Callable[Concatenate[int, P], T],
-    *args: P.args,
-    **kwargs: P.kwargs,
-) -> T:
-    return caching_pid_swallower(os.getpid(), *args, **kwargs)
-
-
-def fork_safe_cached(
-    cache_deco: ty.Callable[[F], F],
-    func: ty.Callable[P, T],
-) -> ty.Callable[P, T]:
-    """Decorator to make a fork-safe cached.locking function by wrapping it in a function that
-    always calls os.getpid() to invalidate the cache on new processes."""
-    return partial(
-        _pid_sending_wrapper,
-        cache_deco(ty.cast(F, partial(_pid_swallower, func))),
-    )