thds.mops 3.9.20250813225451__py3-none-any.whl → 3.9.20250814171126__py3-none-any.whl

thds/mops/pure/core/deferred_work.py

@@ -23,6 +23,10 @@ _DEFERRED_WORK_THREADPOOL = refcount.Resource[concurrent.futures.ThreadPoolExecu
 logger = core.log.getLogger(__name__)
 
 
+class NoDeferredWorkContext(Exception):
+    """Raised when work is pushed with no open context"""
+
+
 @contextmanager
 def open_context() -> ty.Iterator[None]:
     """Enter this context before you begin serializing your invocation. When perform_all()
@@ -44,12 +48,6 @@ def open_context() -> ty.Iterator[None]:
             )
 
 
-@contextmanager
-def push_non_context() -> ty.Iterator[None]:
-    with _DEFERRED_INVOCATION_WORK.set(None):
-        yield
-
-
 def add(work_owner: str, work_id: ty.Hashable, work: ty.Callable[[], ty.Any]) -> None:
     """Add some work to an open context. The work will be performed when perform_all() is
     called. If there is no open context, perform the work immediately.
@@ -62,8 +60,7 @@ def add(work_owner: str, work_id: ty.Hashable, work: ty.Callable[[], ty.Any]) ->
     """
     deferred_work = _DEFERRED_INVOCATION_WORK()
     if deferred_work is None:
-        logger.debug("No open context - performing work %s immediately", work)
-        work()
+        raise NoDeferredWorkContext("Deferred work can only be added when there is an open context.")
     else:
         logger.debug("Adding work %s to deferred work %s", (work_owner, work_id), id(deferred_work))
         deferred_work[(work_owner, work_id)] = work

thds/mops/pure/core/entry/route_result.py

@@ -38,15 +38,7 @@ def route_return_value_or_exception(
     pipeline_function_and_arguments_unique_key: ty.Optional[ty.Tuple[str, str]] = None,
 ) -> None:
     """The remote side of your runner implementation doesn't have to use this, but it's a reasonable approach."""
-    _routing_scope.enter(deferred_work.push_non_context())
-    # deferred work can be requested during result serialization, but because we don't want
-    # to leave a 'broken' result payload (one that refers to unperformed deferred work,
-    # maybe because of network or other failure), we simply don't open a deferred work
-    # context on the remote side, which forces all the work to be performed as it is
-    # added for deferral instead of actually being deferred.
-    #
-    # pushing this non-context is only necessary in the case of a thread-local
-    # 'remote' invocation - in all true remote invocations, there will be no context open.
+    _routing_scope.enter(deferred_work.open_context())
 
     _routing_scope.enter(log.logger_context(remote=pipeline_id))
     if pipeline_function_and_arguments_unique_key:

thds/mops/pure/core/source.py

@@ -178,10 +178,6 @@ def prepare_source_argument(source_: Source) -> ty.Union[str, hashing.Hash]:
     return hashing.Hash(algo=sys.intern(source_.hash.algo), bytes=source_.hash.bytes)
 
 
-def perform_source_uploads() -> None:  # has been replaced by a general work-deferring mechanism.
-    deferred_work.perform_all()
-
-
 # RETURNING FROM REMOTE
 #
 # when returning a Source from a remote, we cannot avoid the upload.  this is because the
@@ -222,6 +218,11 @@ class DuplicateSourceBasenameError(ValueError):
     """
 
 
+def _put_file_to_blob_store(local_path: Path, remote_uri: str) -> None:
+    logger.info("Uploading Source to remote URI %s", remote_uri)
+    lookup_blob_store(remote_uri).putfile(local_path, remote_uri)
+
+
 def prepare_source_result(source_: Source, existing_uris: ty.Collection[str] = tuple()) -> SourceResult:
     """Call from within the remote side of an invocation, while serializing the function return value.
 
@@ -237,8 +238,12 @@ def prepare_source_result(source_: Source, existing_uris: ty.Collection[str] = t
             # it exists locally - an upload may be necessary.
             file_uri = to_uri(source_.cached_path)
             if source_.uri not in existing_uris:
-                lookup_blob_store(source_.uri).putfile(source_.cached_path, source_.uri)
-                logger.info("Uploading Source to chosen URI %s", source_.uri)
+                logger.info("Using existing remote URI on Source %s", source_.uri)
+                deferred_work.add(
+                    __name__ + "-chosen-source-result",
+                    source_.uri,
+                    partial(_put_file_to_blob_store, source_.cached_path, source_.uri),
+                )
         else:
             file_uri = ""
             logger.debug("Creating a SourceResult for a URI that is presumed to already be uploaded.")
@@ -249,7 +254,7 @@ def prepare_source_result(source_: Source, existing_uris: ty.Collection[str] = t
     # future caller on a different machine could try to use this memoized result.
     local_path = source.path_from_uri(source_.uri)
     assert local_path.exists(), f"{local_path} does not exist"
-    logger.debug("Automatically selecting a remote URI for a Source being returned.")
+    logger.info("Automatically selecting a remote URI for a Source being returned.")
     remote_uri = invocation_output_uri(name=local_path.name)
     # the line above is a bit of opinionated magic. it uses the 'end' of the filename
     # to automagically assign a meaningful name to the output remote URI.
@@ -266,9 +271,12 @@ def prepare_source_result(source_: Source, existing_uris: ty.Collection[str] = t
             " with unique basenames, in order to allow retention of the basename for usability and debugging."
         )
 
-    lookup_blob_store(remote_uri).putfile(local_path, remote_uri)
+    deferred_work.add(
+        __name__ + "-derived-source-result",
+        remote_uri,
+        partial(_put_file_to_blob_store, local_path, remote_uri),
+    )
     # upload must _always_ happen on remotely-returned Sources, as detailed above.
-    # There is no advantage to waiting to upload past this point.
     return SourceResult(remote_uri, source_.hash, source_.uri)
 
 

thds/mops/pure/pickling/remote.py

@@ -7,7 +7,7 @@ from functools import cached_property
 from thds.core import log, scope
 
 from ..._utils.once import Once
-from ..core import lock, metadata, pipeline_id, uris
+from ..core import deferred_work, lock, metadata, pipeline_id, uris
 from ..core.entry import route_return_value_or_exception
 from ..core.memo import results
 from ..core.serialize_big_objs import ByIdRegistry, ByIdSerializer
@@ -67,6 +67,14 @@ class _ResultExcWithMetadataChannel:
             self._write_metadata_only("lost-race-after-serialization")
             return
 
+        # It's important that all deferred work is performed before the return
+        # value is written to the blob store so that result consumers don't read
+        # inconsistent data. For example, one type of deferred work is uploading
+        # result sources. If the invocation result is written with the source
+        # uri before the source is uploaded, then result consumers might try to
+        # download a non-existent file in the meantime.
+        deferred_work.perform_all()
+
         # BUG: there remains a race condition between fs.exists and putbytes.
         # multiple callers could get a False from fs.exists and then proceed to write.
         # the biggest issue here is for functions that are not truly pure, because

{thds_mops-3.9.20250813225451.dist-info → thds_mops-3.9.20250814171126.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.mops
-Version: 3.9.20250813225451
+Version: 3.9.20250814171126
 Summary: ML Ops tools for Trilliant Health
 Author-email: Trilliant Health <info@trillianthealth.com>
 Project-URL: Repository, https://github.com/TrilliantHealth/ds-monorepo

{thds_mops-3.9.20250813225451.dist-info → thds_mops-3.9.20250814171126.dist-info}/RECORD

@@ -48,7 +48,7 @@ thds/mops/pure/adls/blob_store.py,sha256=ZWr7CKKcI-jz1sWZq4Jwq6LYkhFNxp-EFnNh83E
 thds/mops/pure/adls/output_fqn.py,sha256=qnwdubjVwKShzZ5RruD0_85x86DtPwZNSgwADrdhrTs,748
 thds/mops/pure/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/pure/core/content_addressed.py,sha256=RaCPvtM7bf0NnY5lNR5jPcNn2Moh-bmLtC4zOvdWjCU,1202
-thds/mops/pure/core/deferred_work.py,sha256=zN-iRJj2lUB83SrxOC0JLhfkSWA6qIc1cvSScfkWKOM,3756
+thds/mops/pure/core/deferred_work.py,sha256=lCgjFERyVKhfcy8R7MNA8NsEZAkZgmCQ71c8Bo7Nq8Q,3741
 thds/mops/pure/core/file_blob_store.py,sha256=N4m4LLrBZaqTJFR4D_eYl03a-n6yQBRsv0ID1bOS9TA,4298
 thds/mops/pure/core/metadata.py,sha256=xAL2iz0pXrcKapmYnNrqSZ8nH2GVakA167NSpAfwiCI,8276
 thds/mops/pure/core/output_naming.py,sha256=ntufOVNJiVPiUM-Azl9mFpDFhIxiB-V2je9dv9AUQhg,2283
@@ -58,13 +58,13 @@ thds/mops/pure/core/pipeline_id_mask.py,sha256=Ll2yyQM5nSgzihx8i7fCrrSNlUUnIsbAO
 thds/mops/pure/core/script_support.py,sha256=3j9Z1O5ynSSPmWSghtJgAj-Lt4GwYcA8cWcpUIRM7q0,952
 thds/mops/pure/core/serialize_big_objs.py,sha256=YcOS1ccs82ZWO7nTbeumErMzYVe4hgXCTsfvMggYmd8,2332
 thds/mops/pure/core/serialize_paths.py,sha256=bWI-AKNP_Tf29JGO7DKqshOh7b7gu51lfGryDXo3aMI,5787
-thds/mops/pure/core/source.py,sha256=mubbxj3s2tsYHfG-m4R1DDJT63zDaw9COrRzQqbwDRs,14859
+thds/mops/pure/core/source.py,sha256=R36ajrCU1JdWF-8iD8YqAiP-q39ypZqf2DeBsqC9lYo,15105
 thds/mops/pure/core/types.py,sha256=w2g83miGhnjaWr2_4TW2Fc3BdIgoIHFbIr_wX1HC7A0,5452
 thds/mops/pure/core/uris.py,sha256=qO9_f-ro7kax6haNOPTPe81-_aUSRFELeeZH4PMTTU4,2694
 thds/mops/pure/core/use_runner.py,sha256=m1Mu1XDr3xRf_u_VSiHfTG4TH6fnSg0IqwmtbLKG_oc,2103
 thds/mops/pure/core/entry/__init__.py,sha256=kiDcsj16CwjRSexOZW-4h4b4tDCYIS_eLS5wgu2yIlk,151
 thds/mops/pure/core/entry/main.py,sha256=b1F5lFDK_hnpvW3bqzt5MWDcpKvCXZpWdEHI8zroC4k,2061
-thds/mops/pure/core/entry/route_result.py,sha256=2LcS9M2mYtu56kso0YcMEZbR1mbTWZm0hFlbE2yaf4k,2741
+thds/mops/pure/core/entry/route_result.py,sha256=sifgDWUn0LRUScJur4KfjVJjkinbjFJ5KQdhq3wJwdU,2148
 thds/mops/pure/core/entry/runner_registry.py,sha256=aPDCML7gM_zP6NfPnqx0_Q1oRHzgdaCa_XzYc5VIw7U,601
 thds/mops/pure/core/lock/__init__.py,sha256=4x9NdborLPGktDNs8LDapW17LeuAHLCbO9v-8IWdT2I,268
 thds/mops/pure/core/lock/_acquire.py,sha256=lVxHzDA30VB95Cfb4Fl2m0eatdLXCDv6rOCnERiyMNw,9468
@@ -89,7 +89,7 @@ thds/mops/pure/pickling/_pickle.py,sha256=YB8xbqDiwdk8ccnVZ2_4kQn98V2JSrFqw2E3J-
 thds/mops/pure/pickling/memoize_only.py,sha256=oI5CMy6IEJc46Gb_BGWNUuAe3fysS7HxRSTajN0WssI,837
 thds/mops/pure/pickling/mprunner.py,sha256=vabdHIVteddkU5ncOq73wWC7-naChW_3_vvAQArvjqU,8814
 thds/mops/pure/pickling/pickles.py,sha256=CSlnjLssE0Ad8YzqyaKqWCSNyW5LiMFKiXO6hWAZmvU,5097
-thds/mops/pure/pickling/remote.py,sha256=eax4y0FgoH6id-ceptYbs9_bSLA9TIBhw2EEHqBkvHE,7971
+thds/mops/pure/pickling/remote.py,sha256=7JXZRGnLI5y5dqElIDrhIlaRv6Q_zQ_78aqNhO7O4KY,8478
 thds/mops/pure/pickling/sha256_b64.py,sha256=HL0cPixHPZYuZDVDBscxsnI-3a2amWEfw-LseOX-PyY,2916
 thds/mops/pure/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/pure/runner/get_results.py,sha256=1K6qf_Vg2YfUPfUuu103WyYsfS3e_ju6W7Z_PV01-pU,4053
@@ -109,8 +109,8 @@ thds/mops/pure/tools/summarize/cli.py,sha256=7kDtn24ok8oBO3jFjlMmOK3jnZYpMoE_5Y8
 thds/mops/pure/tools/summarize/run_summary.py,sha256=w45qiQr7elrHDiK9Hgs85gtU3gwLuXa447ih1Y23BBY,5776
 thds/mops/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/testing/deferred_imports.py,sha256=f0ezCgQAtzTqW1yAOb0OWgsB9ZrlztLB894LtpWDaVw,3780
-thds_mops-3.9.20250813225451.dist-info/METADATA,sha256=90fMUw8RjqqtPCFfuMDMecVULq7DGyOOaOcSqP55muA,2225
-thds_mops-3.9.20250813225451.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_mops-3.9.20250813225451.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
-thds_mops-3.9.20250813225451.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_mops-3.9.20250813225451.dist-info/RECORD,,
+thds_mops-3.9.20250814171126.dist-info/METADATA,sha256=_uAlVicbPSNmKbIiwRID23fmuWdWulmrFhXcdzZdpBI,2225
+thds_mops-3.9.20250814171126.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_mops-3.9.20250814171126.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
+thds_mops-3.9.20250814171126.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_mops-3.9.20250814171126.dist-info/RECORD,,