thds.adls 4.2.20251002171438__py3-none-any.whl → 4.3.20251008013223__py3-none-any.whl

thds/adls/download.py

@@ -16,7 +16,7 @@ from thds.core.types import StrOrPath
 
 from . import azcopy, errors, etag, hashes
 from ._progress import report_download_progress
-from .download_lock import download_lock
+from .file_lock import file_lock
 from .fqn import AdlsFqn
 from .ro_cache import Cache, from_cache_path_to_local, from_local_path_to_cache
 
@@ -240,12 +240,12 @@ def _download_or_use_verified_cached_coroutine(  # noqa: C901
     # No cache hit, so its time to prepare to download. if a cache was provided, we will
     # _put_ the resulting file in it.
 
-    file_lock = str(cache.path(fqn) if cache else local_path)
+    file_lock_str = str(cache.path(fqn) if cache else local_path)
     # create lockfile name from the (shared) cache path if present, otherwise the final
     # destination.  Non-cache users may then still incur multiple downloads in parallel,
     # but if you wanted to coordinate then you should probably have been using the global
     # cache in the first place.
-    _dl_scope.enter(download_lock(file_lock))
+    _dl_scope.enter(file_lock(file_lock_str))
 
     # re-attempt cache hit - we may have gotten the lock after somebody else downloaded
     if file_result := attempt_cache_hit():

thds/adls/{download_lock.py → file_lock.py}

@@ -9,18 +9,18 @@ from thds.core import config, home, log
 
 from .md5 import hex_md5_str
 
-DOWNLOAD_LOCKS_DIR = config.item("dir", home.HOMEDIR() / ".thds/adls/download-locks", parse=Path)
+FILELOCKS_DIR = config.item("dir", home.HOMEDIR() / ".thds/adls/file-locks", parse=Path)
 _CLEAN_UP_LOCKFILES_AFTER_TIME = timedelta(hours=24)
 _CLEAN_UP_LOCKFILES_EVERY = timedelta(hours=1).total_seconds()
 _LAST_CLEANED_BY_THIS_PROCESS = time.monotonic() - _CLEAN_UP_LOCKFILES_EVERY
 logger = log.getLogger(__name__)
 
 
-def _clean_download_locks() -> int:
+def _clean_file_locks() -> int:
     deleted = 0
     deletion_threshold = time.time() - _CLEAN_UP_LOCKFILES_AFTER_TIME.total_seconds()
     try:
-        for f in DOWNLOAD_LOCKS_DIR().iterdir():
+        for f in FILELOCKS_DIR().rglob("*"):
             fstat = f.stat()
             if stat.S_ISREG(fstat.st_mode) and fstat.st_mtime < deletion_threshold:
                 f.unlink()
@@ -29,20 +29,20 @@ def _clean_download_locks() -> int:
         # this should be, hopefully, both very rare and completely inconsequential as to
         # program correctness.  if you see this happen multiple times, you may have some
         # read-only files or something and want to manually clean up this directory.
-        logger.exception("Failed to clean download locks directory.")
+        logger.exception("Failed to clean file locks directory.")
     return deleted
 
 
-def _occasionally_clean_download_locks():
+def _occasionally_clean_file_locks():
     global _LAST_CLEANED_BY_THIS_PROCESS
     # do this about once an hour
     if time.monotonic() > _LAST_CLEANED_BY_THIS_PROCESS + _CLEAN_UP_LOCKFILES_EVERY:
         _LAST_CLEANED_BY_THIS_PROCESS = time.monotonic()
         # minor race condition with other threads but it doesn't really matter.
-        _clean_download_locks()
+        _clean_file_locks()
 
 
-def download_lock(download_unique_str: str) -> FileLock:
+def file_lock(lock_unique_str: str, locktype: str = "download") -> FileLock:
     """Note that the lockfiles will never be deleted automatically.
     https://py-filelock.readthedocs.io/en/latest/api.html#filelock.BaseFileLock.release
 
@@ -50,7 +50,7 @@ def download_lock(download_unique_str: str) -> FileLock:
     https://stackoverflow.com/questions/58098634/why-does-the-python-filelock-library-delete-lockfiles-on-windows-but-not-unix
 
     This means local developers would have a whole bunch of zero-byte files in their
-    download locks directory. So, we take a slightly idiosyncratic approach to cleaning
+    file locks directory. So, we take a slightly idiosyncratic approach to cleaning
     this up: not wanting to run this code on every download, but also not wanting
     developers to see an infinitely-growing mess.  Since parallel downloads will
     (generally) not constitute a correctness issue, the 'safest' time to clean it up will
@@ -58,11 +58,11 @@ def download_lock(download_unique_str: str) -> FileLock:
     we can get rid of old lockfiles after they've existed for more than 24 hours, since
     it's quite rare that a download would last that long.
     """
-    DOWNLOAD_LOCKS_DIR().mkdir(parents=True, exist_ok=True)
-    _occasionally_clean_download_locks()
+    lock_type_dir = FILELOCKS_DIR() / locktype
+    lock_type_dir.mkdir(parents=True, exist_ok=True)
+    _occasionally_clean_file_locks()
     return FileLock(
-        DOWNLOAD_LOCKS_DIR()
-        / (download_unique_str.split("/")[-1][:50] + hex_md5_str(download_unique_str)),
+        lock_type_dir / (lock_unique_str.split("/")[-1][:50] + hex_md5_str(lock_unique_str)),
         # is_singleton=True,
         # critical for keeping this reentrant without passing the lock around.
         # see https://github.com/tox-dev/filelock/issues/315#issuecomment-2016797681

thds/adls/list_fast.py

@@ -31,7 +31,7 @@ def multilayer_yield_blob_meta(fqn: AdlsFqn, layers: int = 1) -> ty.Iterator[Blo
         # directly yield the blobs
         yield from yield_blob_meta(
             global_client.get_global_blob_container_client(fqn.sa, fqn.container),
-            fqn.path,
+            fqn.path.rstrip("/") + "/",
         )
         return
 

thds/adls/upload.py

@@ -17,6 +17,7 @@ from . import azcopy, hashes
 from ._progress import report_upload_progress
 from ._upload import upload_decision_and_metadata
 from .conf import UPLOAD_FILE_MAX_CONCURRENCY
+from .file_lock import file_lock
 from .fqn import AdlsFqn
 from .global_client import get_global_blob_container_client
 from .ro_cache import Cache
@@ -101,6 +102,8 @@ def upload(
         # we always use the original source file to upload, not the cached path,
         # because uploading from a shared location risks race conditions.
 
+    scope.enter(file_lock(str(dest_), locktype="upload"))
+
     blob_container_client = get_global_blob_container_client(dest_.sa, dest_.container)
     blob_client = blob_container_client.get_blob_client(dest_.path)
     decision = upload_decision_and_metadata(blob_client.get_blob_properties, src)  # type: ignore [arg-type]

{thds_adls-4.2.20251002171438.dist-info → thds_adls-4.3.20251008013223.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.adls
-Version: 4.2.20251002171438
+Version: 4.3.20251008013223
 Summary: ADLS tools
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

{thds_adls-4.2.20251002171438.dist-info → thds_adls-4.3.20251008013223.dist-info}/RECORD

@@ -8,16 +8,16 @@ thds/adls/conf.py,sha256=nTw3X1ilC3A_905jZH-rWXFsESeHAKQn5IghvfX2VIo,1991
 thds/adls/copy.py,sha256=aD6AquUR8r5W9SXd6Nm1qPrFH_fYpLC5dZk6HjPJnSQ,6611
 thds/adls/dbfs.py,sha256=pPAjbIZRKJsaXKQljDMUgqS_zy1yKeEZHGMueXbuv3g,2219
 thds/adls/defaults.py,sha256=GGq5Pn4r-8cX4bZItp4nnwWAAz7S07pzPoOegw0y5Fw,676
-thds/adls/download.py,sha256=z31w4Yuz4CqmU0iectcXmSoM2QJb1mSp9tGs0GHEhtY,19146
-thds/adls/download_lock.py,sha256=tgT48l4C5_qmArGeq05gl7VlxT22dZBH2Xwxx0itE9o,3176
+thds/adls/download.py,sha256=IPg5nz_sGE7dX8DUQyWjG2D9z54PXLScap-pZzTUFTk,19142
 thds/adls/errors.py,sha256=6NMLHtVNsWBRDXaes9yzHj9cwKOD9t1dwL4BltdtjhU,1895
 thds/adls/etag.py,sha256=ct7jpHhNFcKzbekn5rZ3m6DhjK48A7qOZGwDiHkc-pc,242
+thds/adls/file_lock.py,sha256=yLak5XDpnIYwfUNdpGFbIGG64uEs98-yVscNpJlqMxM,3176
 thds/adls/file_properties.py,sha256=dhRtbsMNOYfExkEiy76wrLfrJ6IMQeN1Z3LIxgKceqY,2042
 thds/adls/fqn.py,sha256=pIfEw25SjZNGmtzOLwrYCblciK35VQ35ZWb_mRaZKK0,5915
 thds/adls/global_client.py,sha256=q6oG1OOsfl4fbti81u8TE9d4Jx9-phlYgsGSc4032w8,3721
 thds/adls/hashes.py,sha256=2x1zcT_87en_vqFrLFs6F_EZCGpn7hk_81dYAwcypm8,5459
 thds/adls/impl.py,sha256=cNf1vmeS46X_wvyVdDJ8qFfowHn2QwtU5C80BmDtu5Y,43247
-thds/adls/list_fast.py,sha256=7jHnln4DMWYVLHhejj-fdWMBWflBiWfynegKxcUlNDY,4189
+thds/adls/list_fast.py,sha256=yk0ydFiBa7U5JU3BCcIGCcrnS-J3yJaZbaZQ_Xj9xWU,4207
 thds/adls/md5.py,sha256=hGT8AIX32VUsnRCbm8cel9OlxAiRrgjwNWQTqRDHM_k,374
 thds/adls/named_roots.py,sha256=7SLbAoQQpV_mrFZaUPjYoS-F9dxQxN5Hg4M3YPirF_w,751
 thds/adls/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -26,7 +26,7 @@ thds/adls/sas_tokens.py,sha256=mArbB_GYohevOmArw_1gKqVUWpv6kG8Hsbvdrhbtnbg,1957
 thds/adls/shared_credential.py,sha256=-x42aXoIM001KW59oS8PpuXQd4-F2vg-1gB6OMHlpk4,4602
 thds/adls/source.py,sha256=8HVMYuxDn1XYGwFFSBowMlvQ6r2Jm2CQlpu4h85JvsE,2559
 thds/adls/source_tree.py,sha256=gl2JLjxAduo4cGQBb8LqBnmRHHk2wqIC5yt-sqkXOEo,2589
-thds/adls/upload.py,sha256=MRHK9Am-x5FKBPh1SXLTbPC1r0Xk0bGWNU8CcNuUMLo,6602
+thds/adls/upload.py,sha256=7eWPpPuIgCBb3Svk05K1UNv0376q8wDTQkpTaKLtg-w,6694
 thds/adls/uri.py,sha256=9MXuW_KfpPvzBc4ERxuTJ3vvi_6yr7e1kMAW9mx2zXM,1414
 thds/adls/azcopy/__init__.py,sha256=qn2dmT92EHcrtaQ8uwRoUgvtF6Fu3NQbhZItOBdIBmY,45
 thds/adls/azcopy/download.py,sha256=FOtYyYh7ZXNWNdkj04yTV26lxcKOVj-YhS2p_EclYxA,6526
@@ -38,8 +38,8 @@ thds/adls/tools/download.py,sha256=CW2cWbCRdUqisVVVoqqvqk5Ved7pPGTkwnZj3uV0jy4,1
 thds/adls/tools/ls.py,sha256=OgEaIfTK359twlZIj-A0AW_nv81Z6zi0b9Tw6OJJfWA,1083
 thds/adls/tools/ls_fast.py,sha256=Nowc-efAL_Y4ybPwZzKIeh7KGIjfecRzdWvJZcBzq_8,585
 thds/adls/tools/upload.py,sha256=5WyWkpuVp2PETZ3O3ODlq8LXszSHU73ZMnIDZXPJdC8,442
-thds_adls-4.2.20251002171438.dist-info/METADATA,sha256=V7J4XH60NB7l3QCwQnIrlZls6UEJhFYBKoi8DLtGSbY,587
-thds_adls-4.2.20251002171438.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_adls-4.2.20251002171438.dist-info/entry_points.txt,sha256=rtVF0A2MMTYUsBScF6b3AlOuk2Vm02QK7Tc2bDcDpk0,200
-thds_adls-4.2.20251002171438.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_adls-4.2.20251002171438.dist-info/RECORD,,
+thds_adls-4.3.20251008013223.dist-info/METADATA,sha256=YNBtKknKx9Hv62k3anREQ5VBI6_KY5LoZ6h68aNLMt8,587
+thds_adls-4.3.20251008013223.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_adls-4.3.20251008013223.dist-info/entry_points.txt,sha256=rtVF0A2MMTYUsBScF6b3AlOuk2Vm02QK7Tc2bDcDpk0,200
+thds_adls-4.3.20251008013223.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_adls-4.3.20251008013223.dist-info/RECORD,,