thds.mops 3.9.20250929214325__py3-none-any.whl → 3.9.20250930235847__py3-none-any.whl

thds/mops/k8s/uncertain_future.py

@@ -44,20 +44,20 @@ class _FutureInterpretationShim(ty.Generic[R_0, R]):
     def __hash__(self) -> int:
         return hash(self._id)
 
-    def __call__(self, r_0: ty.Optional[R_0], last_seen_at: float) -> ty.Optional[Self]:
+    def interpret(self, r_0: ty.Optional[R_0], last_seen_at: float) -> ty.Optional[Self]:
         """First and foremost - this _must_ be treated as an object that the creator
         is ultimately responsible for calling on a semi-regular basis. It represents a
         likely deadlock for the holder of the Future if it is never called.
 
-        Return False if the Future is still in progress and should not be unregistered.
-        Return True if the Future is done and should be unregistered.
+        Return None if the Future is still in progress and should not be unregistered.
+        Return self if the Future is done and should be unregistered.
         """
         try:
             interpretation = self._interpreter(r_0, last_seen_at)
             if isinstance(interpretation, NotYetDone):
                 return None  # do nothing and do not unregister - the status is still in progress.
 
-            self.future.set_result(interpretation)
+            self.future.set_result(interpretation)  # resolved successfully!
         except Exception as e:
             self.future.set_exception(e)
 
@@ -71,7 +71,7 @@ K = ty.TypeVar("K")  # Key type for the UncertainFuturesTracker
 class _FuturesState(ty.Generic[R_0]):
     """Represents a single 'observable' that may have multiple Futures (and therefore interpretations) associated with it."""
 
-    futshims: list[_FutureInterpretationShim[R_0, ty.Any]]
+    futshims: set[_FutureInterpretationShim[R_0, ty.Any]]
     last_seen_at: float
 
 
@@ -94,10 +94,16 @@ class UncertainFuturesTracker(ty.Generic[K, R_0]):
     never resolve the Future, then a caller may be waiting for it forever. Therefore, we
     ask the original requestor of the Future to specify how long they are willing to wait
     to get a result, after which point we will resolve the Future as an exception.
+
+    Notably, once we have seen an object, we will not ever remove it from our tracking list.
+    This implies a certain amount of memory growth over time, but it avoids race conditions
+    between producers and consumers of the Futures.
     """
 
     def __init__(self, allowed_stale_seconds: float) -> None:
         self._keyed_futures_state = collections.OrderedDict[K, _FuturesState[R_0]]()
+        # ordered from least-recently-seen to most-recently-seen, so that we can easily garbage collect
+        # potentially stale Futures, which will be at the front of the OrderedDict.
         self._lock = threading.Lock()  # i don't trust ordered dict operations to be thread-safe.
         self._check_stale_seconds = allowed_stale_seconds
 
@@ -105,16 +111,11 @@ class UncertainFuturesTracker(ty.Generic[K, R_0]):
         futshim = _FutureInterpretationShim(interpreter)
         with self._lock:
             if key not in self._keyed_futures_state:
-                self._keyed_futures_state[key] = _FuturesState(
-                    [futshim],
-                    last_seen_at=official_timer() + self._check_stale_seconds,
-                    # we provide a double margin for objects that we have never seen before.
-                )
-                self._keyed_futures_state.move_to_end(key, last=False)
-                # never seen and therefore should be at the beginning (most stale)
+                self._keyed_futures_state[key] = _FuturesState({futshim}, last_seen_at=official_timer())
+                # never seen and therefore should be at the end (least stale)
             else:
                 # maintain our ordered dict so we can handle garbage collection of stale Futures.
-                self._keyed_futures_state[key].futshims.append(futshim)
+                self._keyed_futures_state[key].futshims.add(futshim)
 
         return futshim.future
 
@@ -126,10 +127,10 @@ class UncertainFuturesTracker(ty.Generic[K, R_0]):
         If `key` is None, we will update all Futures that have been created so far.
         """
 
-        def check_resolution(fut_state: _FuturesState[R_0], inner_r_0: ty.Optional[R_0]) -> None:
+        def interpret_event(fut_state: _FuturesState[R_0], inner_r_0: ty.Optional[R_0]) -> None:
             for future_shim_that_is_done in core.parallel.yield_results(
                 [
-                    core.thunks.thunking(futshim)(inner_r_0, fut_state.last_seen_at)
+                    core.thunks.thunking(futshim.interpret)(inner_r_0, fut_state.last_seen_at)
                     for futshim in fut_state.futshims
                 ],
                 progress_logger=core.log.getLogger(__name__).debug,
@@ -137,24 +138,33 @@ class UncertainFuturesTracker(ty.Generic[K, R_0]):
             ):
                 if future_shim_that_is_done is not None:
                     # the Future is done, so we can remove it from the list of Futures.
-                    fut_state.futshims.remove(future_shim_that_is_done)
+                    fut_state.futshims.discard(future_shim_that_is_done)
 
         if key is not None:
             with self._lock:
                 if key not in self._keyed_futures_state:
-                    self._keyed_futures_state[key] = _FuturesState(list(), last_seen_at=official_timer())
+                    self._keyed_futures_state[key] = _FuturesState(set(), last_seen_at=official_timer())
                 else:
                     # maintain our ordered dict so we can handle garbage collection of stale Futures.
                     self._keyed_futures_state.move_to_end(key)
                     self._keyed_futures_state[key].last_seen_at = official_timer()
 
-            fut_state = self._keyed_futures_state[key]
-            check_resolution(fut_state, r_0)
+            if fut_state := self._keyed_futures_state.get(key):
+                interpret_event(fut_state, r_0)
 
+        #
         # 'garbage collect' any Futures that haven't been updated in a while.
-        for futs_state in self._keyed_futures_state.values():
-            if futs_state.last_seen_at + self._check_stale_seconds < official_timer():
-                check_resolution(futs_state, None)
+        #
+        with self._lock:
+            safe_futures = list(self._keyed_futures_state.values())
+            # this avoids holding the lock, but also avoids RuntimeError: OrderedDict mutated during iteration
+        now = official_timer()
+        for futs_state in safe_futures:
+            if now > futs_state.last_seen_at + self._check_stale_seconds:
+                interpret_event(futs_state, None)
+                # None means we have no new information about the object.
+                # the interpreter must decide what to do with that, plus the last seen time.
+
             else:  # these are ordered, so once we see one that's not stale, we can stop checking.
                 # this prevents us from having to do O(N) checks for every update.
                 break

thds/mops/k8s/watch.py

@@ -112,9 +112,14 @@ def callback_events(
 ) -> None:
     """Suitable for use with a daemon thread."""
     for namespace, obj, event in event_yielder:
-        should_exit = on_event(namespace, obj, event)
-        if should_exit:
-            break
+        try:
+            should_exit = on_event(namespace, obj, event)
+            if should_exit:
+                break
+        except Exception:
+            logger.exception(
+                "Exception in k8s watch event callback [probably a bug in mops] - continuing..."
+            )
 
 
 def _default_get_name(obj: ty.Any) -> str:

{thds_mops-3.9.20250929214325.dist-info → thds_mops-3.9.20250930235847.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.mops
-Version: 3.9.20250929214325
+Version: 3.9.20250930235847
 Summary: ML Ops tools for Trilliant Health
 Author-email: Trilliant Health <info@trillianthealth.com>
 Project-URL: Repository, https://github.com/TrilliantHealth/ds-monorepo

{thds_mops-3.9.20250929214325.dist-info → thds_mops-3.9.20250930235847.dist-info}/RECORD

@@ -31,10 +31,10 @@ thds/mops/k8s/namespace.py,sha256=Z6trVTU9WFashto4PqIhTcxu-foOF93W0TpgqCU7WIA,38
 thds/mops/k8s/node_selection.py,sha256=Gy2Jz8IxZblg2LmtGg8-MtKI4RmXz2AMXqFPP8OQyu0,2065
 thds/mops/k8s/retry.py,sha256=JVfP304kItpLs5nrONHE5UWkVWlrFGlV_oFQqhq3zHg,2846
 thds/mops/k8s/too_old_resource_version.py,sha256=S7ltVA-LrxUpQ8Q__AB0nQmezN8Mmnx5oKK62_baAKI,1500
-thds/mops/k8s/uncertain_future.py,sha256=60v9yVlhnCDN_yUv8l4Z4KafR4TsTGxN7dprkGI8pQQ,7152
+thds/mops/k8s/uncertain_future.py,sha256=_ix-4EqZE_MY5sbLjT-lQ9GIa7woQ3iunPVtDapIhi8,7752
 thds/mops/k8s/wait_job.py,sha256=_X5lSn-3CE4V-_ra0kF1WtxkAiOgqSom8mU1-0hhMio,2445
 thds/mops/k8s/warn_image_backoff.py,sha256=ls_zLSnRbJjO4ICjq1Rk21EXh190l2dT6nKg-PT8Das,1934
-thds/mops/k8s/watch.py,sha256=4LBLZ9s9hp4jSOr3OTBDMWWCEDedgxsUoYt89zCLLsw,14020
+thds/mops/k8s/watch.py,sha256=nH9HBoRCvt8FRhUav9at71QnTCh0plWga3rp7aUYb1E,14212
 thds/mops/k8s/tools/krsync.py,sha256=us7pXX0-bRMwD2oAno7Z6BJcPs6FgaUabHW0STyQJYg,1773
 thds/mops/k8s/tools/krsync.sh,sha256=fWgwkdzWnJeTbzEA_uBiIIi-bNU4nXAYj3dNovyRluU,747
 thds/mops/pure/__init__.py,sha256=3xLimQ2JWdeq1YgPs7bPwlwOspzPRwaR2w2KX7vfJU0,1624
@@ -109,8 +109,8 @@ thds/mops/pure/tools/summarize/cli.py,sha256=7kDtn24ok8oBO3jFjlMmOK3jnZYpMoE_5Y8
 thds/mops/pure/tools/summarize/run_summary.py,sha256=w45qiQr7elrHDiK9Hgs85gtU3gwLuXa447ih1Y23BBY,5776
 thds/mops/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/testing/deferred_imports.py,sha256=f0ezCgQAtzTqW1yAOb0OWgsB9ZrlztLB894LtpWDaVw,3780
-thds_mops-3.9.20250929214325.dist-info/METADATA,sha256=eldgvZ4clfti48u5YIosxuUN1mcZZVz-UyQ55hlWezg,2225
-thds_mops-3.9.20250929214325.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_mops-3.9.20250929214325.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
-thds_mops-3.9.20250929214325.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_mops-3.9.20250929214325.dist-info/RECORD,,
+thds_mops-3.9.20250930235847.dist-info/METADATA,sha256=lZx15ZxrydRhLaSQMO7KnUyxnUoVzOAqBtgQSN1bgyY,2225
+thds_mops-3.9.20250930235847.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_mops-3.9.20250930235847.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
+thds_mops-3.9.20250930235847.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_mops-3.9.20250930235847.dist-info/RECORD,,