thds.mops 3.8.20250715194424__py3-none-any.whl → 3.8.20250718170958__py3-none-any.whl

thds/mops/pure/core/deferred_work.py

@@ -3,15 +3,23 @@
 # this could be _any_ kind of work, but is only uploads as of initial abstraction.
 # this basic idea was stolen from `pure.core.source` as a form of optimization for
 # uploading Sources and their hashrefs.
+import concurrent.futures
 import typing as ty
 from contextlib import contextmanager
 
 from thds import core
+from thds.core import config, refcount
 from thds.core.stack_context import StackContext
 
 _DEFERRED_INVOCATION_WORK: StackContext[
     ty.Optional[ty.Dict[ty.Hashable, ty.Callable[[], ty.Any]]]
 ] = StackContext("DEFERRED_INVOCATION_WORK", None)
+_MAX_DEFERRED_WORK_THREADS = config.item("max_deferred_work_threads", default=50, parse=int)
+_DEFERRED_WORK_THREADPOOL = refcount.Resource[concurrent.futures.ThreadPoolExecutor](
+    lambda: concurrent.futures.ThreadPoolExecutor(
+        max_workers=_MAX_DEFERRED_WORK_THREADS(), **core.concurrency.initcontext()
+    )
+)
 logger = core.log.getLogger(__name__)
 
 
@@ -74,10 +82,13 @@ def perform_all() -> None:
     work_items = _DEFERRED_INVOCATION_WORK()
     if work_items:
         logger.info("Performing %s items of deferred work", len(work_items))
-        for key, _result in core.parallel.yield_all(dict(work_items).items()):
-            # consume iterator but don't keep results in memory.
-            logger.debug("Popping deferred work %s from %s", key, id(work_items))
-            work_items.pop(key)
+        with _DEFERRED_WORK_THREADPOOL.get() as thread_pool_executor:
+            for key, _ in core.parallel.failfast(
+                core.parallel.yield_all(dict(work_items).items(), executor_cm=thread_pool_executor)
+            ):
+                # consume iterator but don't keep results in memory.
+                logger.debug("Popping deferred work %s from %s", key, id(work_items))
+                work_items.pop(key)
 
         logger.debug("Done performing deferred work on %s", id(work_items))
         assert not work_items, f"Some deferred work was not performed! {work_items}"

{thds_mops-3.8.20250715194424.dist-info → thds_mops-3.8.20250718170958.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.mops
-Version: 3.8.20250715194424
+Version: 3.8.20250718170958
 Summary: ML Ops tools for Trilliant Health
 Author-email: Trilliant Health <info@trillianthealth.com>
 Project-URL: Repository, https://github.com/TrilliantHealth/ds-monorepo

{thds_mops-3.8.20250715194424.dist-info → thds_mops-3.8.20250718170958.dist-info}/RECORD

@@ -44,7 +44,7 @@ thds/mops/pure/adls/blob_store.py,sha256=ZWr7CKKcI-jz1sWZq4Jwq6LYkhFNxp-EFnNh83E
 thds/mops/pure/adls/output_fqn.py,sha256=qnwdubjVwKShzZ5RruD0_85x86DtPwZNSgwADrdhrTs,748
 thds/mops/pure/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/pure/core/content_addressed.py,sha256=RaCPvtM7bf0NnY5lNR5jPcNn2Moh-bmLtC4zOvdWjCU,1202
-thds/mops/pure/core/deferred_work.py,sha256=3vjfqWFlqLLMcmX4nHVaaiidrG5N5KyAYhw6R0hoMzI,3716
+thds/mops/pure/core/deferred_work.py,sha256=OPnEZGQsBYY2FMSmYzm6c0cLBGgZwAs7UDmnDPUlCMM,4270
 thds/mops/pure/core/file_blob_store.py,sha256=N4m4LLrBZaqTJFR4D_eYl03a-n6yQBRsv0ID1bOS9TA,4298
 thds/mops/pure/core/metadata.py,sha256=xAL2iz0pXrcKapmYnNrqSZ8nH2GVakA167NSpAfwiCI,8276
 thds/mops/pure/core/output_naming.py,sha256=ntufOVNJiVPiUM-Azl9mFpDFhIxiB-V2je9dv9AUQhg,2283
@@ -104,8 +104,8 @@ thds/mops/pure/tools/summarize/cli.py,sha256=7kDtn24ok8oBO3jFjlMmOK3jnZYpMoE_5Y8
 thds/mops/pure/tools/summarize/run_summary.py,sha256=LUtvbankAYbss2NCF_XbNl05jkNgxYz_SLyERJlp4sk,5773
 thds/mops/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/testing/deferred_imports.py,sha256=f0ezCgQAtzTqW1yAOb0OWgsB9ZrlztLB894LtpWDaVw,3780
-thds_mops-3.8.20250715194424.dist-info/METADATA,sha256=pa81tiENkdmQSS5-aRTXNmT839y-RTVxh8Ktcz2nGWE,2225
-thds_mops-3.8.20250715194424.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_mops-3.8.20250715194424.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
-thds_mops-3.8.20250715194424.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_mops-3.8.20250715194424.dist-info/RECORD,,
+thds_mops-3.8.20250718170958.dist-info/METADATA,sha256=WowkWj9XmZ6yxtFiEl8GTzuA8V8ZeYPjDU0q-lhbeZg,2225
+thds_mops-3.8.20250718170958.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_mops-3.8.20250718170958.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
+thds_mops-3.8.20250718170958.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_mops-3.8.20250718170958.dist-info/RECORD,,