thds.mops 3.9.20250730010431__py3-none-any.whl → 3.9.20250730172122__py3-none-any.whl

thds/mops/pure/core/lock/__init__.py

@@ -1,6 +1,7 @@
 from ._acquire import acquire  # noqa: F401
 from .maintain import (  # noqa: F401
     CannotMaintainLock,
+    LockWasStolenError,
     launch_daemon_lock_maintainer,
     maintain_to_release,
     remote_lock_maintain,

thds/mops/pure/core/lock/_acquire.py

@@ -101,7 +101,12 @@ def acquire(  # noqa: C901
 
     start = _funcs.utc_now()
 
-    my_writer_id = humenc.encode(uuid4().bytes)
+    my_writer_id = humenc.encode(uuid4().bytes, num_bytes=2)
+    # we do not expect there to be many writers, so we make the humenc part of the writer
+    # id relatively short so it doesn't 'look' like other uses of humenc.  Making the rest
+    # of the string identical to the base64 encoding (by choosing a multiple of 3) is not
+    # useful to us because we are only using this as a big UUID, not as a hash of an
+    # actual input.
 
     lockfile_writer = LockfileWriter(
         my_writer_id,

thds/mops/pure/pickling/remote.py

@@ -52,9 +52,30 @@ class _ResultExcWithMetadataChannel:
         )
 
     def return_value(self, r: T) -> None:
+        result_uri = self.fs.join(self.call_id, results.RESULT)
+        if self.fs.exists(result_uri):
+            logger.warning("Not overwriting existing result at %s prior to serialization", result_uri)
+            self._write_metadata_only("lost-race")
+            return
+
+        # when we pickle the return value, we also end up potentially uploading
+        # various Sources and Paths and other special-cased things inside the result.
         return_value_bytes = _pickle.gimme_bytes(self.dumper, r)
+        if self.fs.exists(result_uri):
+            logger.warning("Not overwriting existing result at %s after serialization", result_uri)
+            self._write_metadata_only("lost-race-after-serialization")
+            return
+
+        # BUG: there remains a race condition between fs.exists and putbytes.
+        # multiple callers could get a False from fs.exists and then proceed to write.
+        # the biggest issue here is for functions that are not truly pure, because
+        # they will be writing different results, and theoretically different callers
+        # could end up seeing the different results.
+        #
+        # In the future, if a Blob Store provided a put_unless_exists method, we could use
+        # that to avoid the race condition.
         self.fs.putbytes(
-            self.fs.join(self.call_id, results.RESULT),
+            result_uri,
             self._metadata_header + return_value_bytes,
             type_hint="application/mops-return-value",
         )
@@ -92,6 +113,8 @@ def run_pickled_invocation(memo_uri: str, *metadata_args: str) -> None:
 
     # any recursively-called functions that use metadata will retain the original invoker.
 
+    failure_to_lock = None
+    stop_lock: ty.Callable = lambda: None  # noqa: E731
     try:
         stop_lock = lock.launch_daemon_lock_maintainer(
             lock.remote_lock_maintain(
@@ -100,7 +123,9 @@ def run_pickled_invocation(memo_uri: str, *metadata_args: str) -> None:
         )
     except lock.CannotMaintainLock as e:
         logger.info(f"Cannot maintain lock: {e}. Continuing without the lock.")
-        stop_lock = lambda: None  # noqa: E731
+    except lock.LockWasStolenError as e:
+        logger.error(f"Lock was stolen: {e}. Exiting without running the function.")
+        failure_to_lock = e
 
     def _extract_invocation_unique_key(memo_uri: str) -> ty.Tuple[str, str]:
         parts = fs.split(memo_uri)
@@ -124,6 +149,14 @@ def run_pickled_invocation(memo_uri: str, *metadata_args: str) -> None:
     def do_work_return_result() -> object:
         # ONLY failures in this code should transmit an EXCEPTION
         # back to the orchestrator side.
+
+        # if the lock was stolen, we should write an exception
+        # so that the orchestrator knows that it failed.
+        # in theory, it could resume waiting for a result, though
+        # currently it does not do this.
+        if failure_to_lock:
+            raise failure_to_lock
+
         with unwrap_use_runner(func):
             return func(*args, **kwargs)
 

{thds_mops-3.9.20250730010431.dist-info → thds_mops-3.9.20250730172122.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.mops
-Version: 3.9.20250730010431
+Version: 3.9.20250730172122
 Summary: ML Ops tools for Trilliant Health
 Author-email: Trilliant Health <info@trillianthealth.com>
 Project-URL: Repository, https://github.com/TrilliantHealth/ds-monorepo

{thds_mops-3.9.20250730010431.dist-info → thds_mops-3.9.20250730172122.dist-info}/RECORD

@@ -66,8 +66,8 @@ thds/mops/pure/core/entry/__init__.py,sha256=kiDcsj16CwjRSexOZW-4h4b4tDCYIS_eLS5
 thds/mops/pure/core/entry/main.py,sha256=b1F5lFDK_hnpvW3bqzt5MWDcpKvCXZpWdEHI8zroC4k,2061
 thds/mops/pure/core/entry/route_result.py,sha256=2LcS9M2mYtu56kso0YcMEZbR1mbTWZm0hFlbE2yaf4k,2741
 thds/mops/pure/core/entry/runner_registry.py,sha256=aPDCML7gM_zP6NfPnqx0_Q1oRHzgdaCa_XzYc5VIw7U,601
-thds/mops/pure/core/lock/__init__.py,sha256=Fq5Fa9DGFcADFxfyFckSl1mYX7p8DAemjMKH9PMpb-s,240
-thds/mops/pure/core/lock/_acquire.py,sha256=O__mQCUQy0mx-JGNgQjbaSPbfCOXkaiJealYt2Av7Cg,9079
+thds/mops/pure/core/lock/__init__.py,sha256=MVwbqK-tQX-Y20lhmofLHFoyEgkQGsaKa9a9JgCP4jc,264
+thds/mops/pure/core/lock/_acquire.py,sha256=lVxHzDA30VB95Cfb4Fl2m0eatdLXCDv6rOCnERiyMNw,9468
 thds/mops/pure/core/lock/_funcs.py,sha256=j4g8yVWnrAMPDKqLlq8nTnccM1KHSJ3g71L1iWNbV2Q,969
 thds/mops/pure/core/lock/cli.py,sha256=uidtmgHB2y5LDkj7SQTncy_cNe1EfIseuiJPV9kcxBU,2488
 thds/mops/pure/core/lock/maintain.py,sha256=5IUQFAU96p46nNt6SMwTAlB2e0HGHJj8n7kqeRxb26M,5767
@@ -89,7 +89,7 @@ thds/mops/pure/pickling/_pickle.py,sha256=YB8xbqDiwdk8ccnVZ2_4kQn98V2JSrFqw2E3J-
 thds/mops/pure/pickling/memoize_only.py,sha256=oI5CMy6IEJc46Gb_BGWNUuAe3fysS7HxRSTajN0WssI,837
 thds/mops/pure/pickling/mprunner.py,sha256=vabdHIVteddkU5ncOq73wWC7-naChW_3_vvAQArvjqU,8814
 thds/mops/pure/pickling/pickles.py,sha256=CSlnjLssE0Ad8YzqyaKqWCSNyW5LiMFKiXO6hWAZmvU,5097
-thds/mops/pure/pickling/remote.py,sha256=l4bIDc7jg7923IFKBeJE3oJYCN4OeT5tmtAg1eS_k7c,6011
+thds/mops/pure/pickling/remote.py,sha256=i2fD9WMuk3rqNxqnQWvkSw9lW6eXVdQ1laxXQWEB3hQ,7665
 thds/mops/pure/pickling/sha256_b64.py,sha256=HL0cPixHPZYuZDVDBscxsnI-3a2amWEfw-LseOX-PyY,2916
 thds/mops/pure/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/pure/runner/get_results.py,sha256=1K6qf_Vg2YfUPfUuu103WyYsfS3e_ju6W7Z_PV01-pU,4053
@@ -109,8 +109,8 @@ thds/mops/pure/tools/summarize/cli.py,sha256=7kDtn24ok8oBO3jFjlMmOK3jnZYpMoE_5Y8
 thds/mops/pure/tools/summarize/run_summary.py,sha256=w45qiQr7elrHDiK9Hgs85gtU3gwLuXa447ih1Y23BBY,5776
 thds/mops/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/testing/deferred_imports.py,sha256=f0ezCgQAtzTqW1yAOb0OWgsB9ZrlztLB894LtpWDaVw,3780
-thds_mops-3.9.20250730010431.dist-info/METADATA,sha256=SDHHsq-QNwEmuB66uQ50V_Iehj3fgcblyjOxxuglv4I,2225
-thds_mops-3.9.20250730010431.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_mops-3.9.20250730010431.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
-thds_mops-3.9.20250730010431.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_mops-3.9.20250730010431.dist-info/RECORD,,
+thds_mops-3.9.20250730172122.dist-info/METADATA,sha256=DEv6JovXRJRIU7BlhFCiR0eQqH8nv2oBtq1P_dyOHvs,2225
+thds_mops-3.9.20250730172122.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_mops-3.9.20250730172122.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
+thds_mops-3.9.20250730172122.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_mops-3.9.20250730172122.dist-info/RECORD,,