thds.mops 3.6.20250407135813__py3-none-any.whl → 3.6.20250407153122__py3-none-any.whl

thds/mops/k8s/config.py

@@ -16,9 +16,22 @@ k8s_job_retry_count = config.item("mops.k8s.job.retry_count", 6, parse=int)
 k8s_job_cleanup_ttl_seconds_after_completion = config.item(
     "mops.k8s.job.cleanup_ttl_seconds", int(timedelta(minutes=60).total_seconds()), parse=int
 )
-k8s_job_timeout_seconds = config.item(
-    "mops.k8s.job.timeout_seconds", int(timedelta(minutes=3).total_seconds()), parse=int
+
+# https://github.com/kubernetes-client/python/blob/master/examples/watch/timeout-settings.md
+k8s_watch_server_timeout_seconds = config.item(
+    "mops.k8s.watch.server_timeout", int(timedelta(hours=1).total_seconds()), parse=int
+)
+k8s_watch_connection_timeout_seconds = config.item(
+    "mops.k8s.watch.connection_timeout", int(timedelta(seconds=5).total_seconds()), parse=int
 )
+k8s_watch_read_timeout_seconds = config.item(
+    "mops.k8s.watch.read_timeout", int(timedelta(seconds=20).total_seconds()), parse=int
+)
+# the above values are designed to retry aggressively if we're not receiving events; from
+# what I've read online, k9s exhibits similar behavior (though I don't know what their
+# values are, and it's not 100% clear to me that they have different types of timeouts
+# from trying to grep their codebase.).
+
 k8s_monitor_delay = config.item("mops.k8s.monitor.delay_seconds", 5, parse=int)
 k8s_monitor_max_attempts = config.item("mops.k8s.monitor.max_attempts", 100, parse=int)
 

thds/mops/k8s/jobs.py

@@ -1,10 +1,11 @@
 import typing as ty
+from datetime import datetime, timezone
 
 from kubernetes import client
 
 from ._shared import logger
 from .retry import k8s_sdk_retry
-from .watch import WatchingObjectSource
+from .watch import EventType, WatchingObjectSource, watch_forever
 
 
 @k8s_sdk_retry()
@@ -32,18 +33,26 @@ def get_job(job_name: str, namespace: str = "") -> ty.Optional[client.models.V1J
 # https://kubernetes.io/docs/concepts/workloads/controllers/job/#terminal-job-conditions
 
 
-def is_job_succeeded(job: client.models.V1Job) -> bool:
+def job_completion_time(job: client.models.V1Job) -> ty.Optional[datetime]:
     if not job.status:
-        return False
+        return None
 
-    if not job.status.completion_time:
-        return False
+    if job.status.completion_time:
+        return job.status.completion_time
 
     for condition in job.status.conditions or tuple():
         if condition.type == "Complete" and condition.status == "True":
-            return True
+            return (
+                condition.last_transition_time
+                if condition.last_transition_time
+                else datetime.now(tz=timezone.utc)
+            )
 
-    return False
+    return None
+
+
+def is_job_succeeded(job: client.models.V1Job) -> bool:
+    return bool(job_completion_time(job))
 
 
 def is_job_failed(job: client.models.V1Job) -> bool:
@@ -55,3 +64,49 @@ def is_job_failed(job: client.models.V1Job) -> bool:
             return True
 
     return False
+
+
+def watch_jobs(
+    namespace: str, timeout: ty.Optional[int] = None
+) -> ty.Iterator[ty.Tuple[client.models.V1Job, EventType]]:
+    yield from watch_forever(
+        lambda _, __: client.BatchV1Api().list_namespaced_job,
+        namespace,
+        typename="Job",
+        timeout=timeout,
+    )
+
+
+def watch_jobs_cli() -> None:
+    import argparse
+    from datetime import datetime
+
+    parser = argparse.ArgumentParser(description="Watch Kubernetes jobs")
+    parser.add_argument("namespace", help="Kubernetes namespace to watch")
+    parser.add_argument("--timeout", type=int, help="Timeout in seconds", default=None)
+    args = parser.parse_args()
+
+    for job, event_type in watch_jobs(args.namespace, timeout=args.timeout):
+        completion_time = job_completion_time(job)
+        creation_time = job.metadata.creation_timestamp
+
+        status = ""
+        if is_job_failed(job):
+            status = "FAILED"
+        elif completion_time := job_completion_time(job):
+            job_duration = completion_time - creation_time
+            status = f"completed_after={job_duration}"
+        else:
+            status = "incomplete" + " " * 20
+
+        print(
+            datetime.now().isoformat(),
+            f"{event_type:<10}",
+            f"{job.metadata.name:<64}",
+            job.metadata.creation_timestamp,
+            status,
+        )
+
+
+if __name__ == "__main__":
+    watch_jobs_cli()

thds/mops/k8s/launch.py

@@ -41,6 +41,19 @@ class Counter:
             return self.value
 
 
+def construct_job_name(user_prefix: str, job_num: str) -> str:
+    # we want some consistency here, but also some randomness in case the prefixes don't exist or aren't unique.
+    mops_name_part = "-".join([str(os.getpid()), job_num, str(uuid.uuid4())[:8]]).lstrip("-")
+    if len(mops_name_part) > 63:
+        # this should be _impossible_, because having a job num longer than even 20 digits would be an impossibly large
+        # number of jobs. but just in case, we'll truncate it to the last 63 characters.
+        mops_name_part = mops_name_part[-63:]  # keep the most random part, to avoid collisions
+    name = f"{user_prefix[:63 - (len(mops_name_part) + 1)]}-{mops_name_part}"
+    name = "".join([c if c.isalnum() or c == "-" else "-" for c in name.lower()])
+    assert len(name) <= 63, f"Job name `{name}` is too long; max length is 63 characters."
+    return name
+
+
 _LAUNCH_COUNT = Counter()
 _FINISH_COUNT = Counter()
 _SIMULTANEOUS_LAUNCHES = threading.BoundedSemaphore(20)
@@ -80,7 +93,7 @@ def launch(
     if not container_image:
         raise ValueError("container_image (the fully qualified Docker tag) must not be empty.")
     job_num = f"{_LAUNCH_COUNT.inc():0>3}"
-    name = "-".join([name_prefix, str(os.getpid()), job_num, str(uuid.uuid4())[:8]]).lstrip("-")
+    name = construct_job_name(name_prefix, job_num)
     scope.enter(logger_context(job=name))
     node_narrowing = node_narrowing or dict()
 

thds/mops/k8s/logging.py

@@ -161,7 +161,10 @@ def _get_pod_phase(pod_name: str) -> str:
         .read_namespaced_pod(
             namespace=config.k8s_namespace(),
             name=pod_name,
-            _request_timeout=(10, config.k8s_job_timeout_seconds()),
+            _request_timeout=(
+                config.k8s_watch_connection_timeout_seconds(),
+                config.k8s_watch_read_timeout_seconds(),
+            ),
         )
         .status.phase
     )
@@ -188,7 +191,10 @@ def _scrape_pod_logs(
     base_kwargs = dict(
         name=pod_name,
         namespace=config.k8s_namespace(),
-        _request_timeout=(10, config.k8s_job_timeout_seconds()),
+        _request_timeout=(
+            config.k8s_watch_connection_timeout_seconds(),
+            config.k8s_watch_read_timeout_seconds(),
+        ),
         # i'm occasionally seeing the `stream()` call below hang
         # indefinitely if logs don't come back from the pod for a
         # while. Which is ironic, since most of this code is here to

thds/mops/k8s/warn_image_backoff.py

@@ -24,7 +24,7 @@ def _emit_basic(event: client.CoreV1Event) -> None:
 def _warn_image_pull_backoff(namespace: str, on_backoff: OnCoreEvent = _emit_basic) -> None:
     """Log scary errors when ImagePullBackoff is observed."""
     start_dt = datetime.now(tz=timezone.utc)
-    for _ns, obj in yield_objects_from_list(
+    for _ns, obj, _event_type in yield_objects_from_list(
         namespace,
         lambda _, __: ty.cast(
             # do NOT use client.EventsV1Api here - for some reason

thds/mops/k8s/watch.py

@@ -3,6 +3,7 @@
 This is a general-purpose fix for using watchers in a thread reliably.
 """
 
+import queue
 import threading
 import time
 import typing as ty
@@ -39,17 +40,23 @@ class K8sList(ty.Protocol[T]):
 # If this does not return a K8sList API method, the loop will exit
 GetListMethod = ty.Callable[[str, ty.Optional[Exception]], ty.Optional[K8sList[T]]]
 # if this returns True, the loop will exit.
-OnEvent = ty.Callable[[str, T], ty.Optional[bool]]
+EventType = ty.Literal["FETCH", "ADDED", "MODIFIED", "DELETED"]
+OnEvent = ty.Callable[[str, T, EventType], ty.Optional[bool]]
 
 
 def yield_objects_from_list(
     namespace: str,
     get_list_method: GetListMethod[T],
-    server_timeout: int = 10,
+    *,
+    server_timeout: int = config.k8s_watch_server_timeout_seconds(),
+    connection_timeout: int = config.k8s_watch_connection_timeout_seconds(),
+    read_timeout: int = config.k8s_watch_read_timeout_seconds(),
+    # connection and read timeout should generally be fairly aggressive so that we retry
+    # quickly if we don't hear anything for a while, and the config defaults are.
     object_type_hint: str = "items",
     init: ty.Optional[ty.Callable[[], None]] = None,
     **kwargs: ty.Any,
-) -> ty.Iterator[ty.Tuple[str, T]]:
+) -> ty.Iterator[ty.Tuple[str, T, EventType]]:
     ex = None
     if init:
         init()
@@ -58,14 +65,15 @@ def yield_objects_from_list(
             load_config()
             list_method = get_list_method(namespace, ex)
             if not list_method:
-                logger.debug(f"No longer watching {object_type_hint} events in namespace: {namespace}")
-                break
+                logger.debug(f"Stopped watching {object_type_hint} events in namespace: {namespace}")
+                return
+
             initial_list = list_method(namespace=namespace)
             logger.debug(
                 f"Listed {len(initial_list.items)} {object_type_hint} in namespace: {namespace}"
             )
             for object in initial_list.items:
-                yield namespace, object
+                yield namespace, object, "FETCH"
 
             if initial_list.metadata._continue:
                 logger.warning(
@@ -76,11 +84,12 @@ def yield_objects_from_list(
                 namespace=namespace,
                 resource_version=initial_list.metadata.resource_version,
                 **kwargs,
-                _request_timeout=(server_timeout, config.k8s_job_timeout_seconds()),
+                timeout_seconds=server_timeout,
+                _request_timeout=(connection_timeout, read_timeout),
             ):
                 object = evt.get("object")
                 if object:
-                    yield namespace, object
+                    yield namespace, object, evt["type"]
                 # once we've received events, let the resource version
                 # be managed automatically if possible.
         except urllib3.exceptions.ProtocolError:
@@ -96,10 +105,12 @@ def yield_objects_from_list(
             ex = e
 
 
-def callback_events(on_event: OnEvent[T], event_yielder: ty.Iterable[ty.Tuple[str, T]]) -> None:
+def callback_events(
+    on_event: OnEvent[T], event_yielder: ty.Iterable[ty.Tuple[str, T, EventType]]
+) -> None:
     """Suitable for use with a daemon thread."""
-    for namespace, event in event_yielder:
-        should_exit = on_event(namespace, event)
+    for namespace, obj, event in event_yielder:
+        should_exit = on_event(namespace, obj, event)
         if should_exit:
             break
 
@@ -116,7 +127,7 @@ def _default_get_namespace(obj: ty.Any) -> str:
     return obj.metadata.namespace
 
 
-STARTING = colorized(fg="white", bg="orange")
+STARTING = colorized(fg="black", bg="orange")
 
 
 class OneShotLimiter:
@@ -151,6 +162,67 @@ def is_stale(api_last_update_time: float, obj_last_seen_time: float) -> bool:
     return (time_since_obj_update := now - obj_last_seen_time) > allowed_stale_seconds  # noqa: F841
 
 
+def _wrap_get_list_method_with_too_old_check(
+    typename: str,
+    get_list_method: GetListMethod[T],
+) -> GetListMethod[T]:
+    def wrapped_get_list_method(namespace: str, exc: ty.Optional[Exception]) -> ty.Optional[K8sList[T]]:
+        suffix = ""
+        if exc:
+            too_old = parse_too_old_resource_version(exc)
+            if not too_old:
+                logger.exception(f"Not fatal, but sleeping before we retry {typename} scraping...")
+                time.sleep(config.k8s_monitor_delay())
+                suffix = f" after {type(exc).__name__}: {exc}"
+                logger.info(f"Watching {typename}s in namespace: {namespace}{suffix}")
+        return get_list_method(namespace, exc)
+
+    return wrapped_get_list_method
+
+
+def create_watch_thread(
+    get_list_method: GetListMethod[T],
+    callback: ty.Callable[[str, T, EventType], None],
+    namespace: str,
+    *,
+    typename: str = "object",
+) -> threading.Thread:
+    return threading.Thread(
+        target=callback_events,
+        args=(
+            callback,
+            yield_objects_from_list(
+                namespace,
+                _wrap_get_list_method_with_too_old_check(typename, get_list_method),
+                # arguably this wrapper could be composed externally, but i see no use cases so far where we'd want that.
+                object_type_hint=typename + "s",
+                init=lambda: logger.info(STARTING(f"Watching {typename}s in {namespace}")),
+            ),
+        ),
+        daemon=True,
+    )
+
+
+def watch_forever(
+    get_list_method: GetListMethod[T],
+    namespace: str,
+    *,
+    typename: str = "object",
+    timeout: ty.Optional[int] = None,
+) -> ty.Iterator[ty.Tuple[T, EventType]]:
+    q: queue.Queue[ty.Tuple[T, EventType]] = queue.Queue()
+
+    def put_queue(namespace: str, obj: T, event_type: EventType) -> None:
+        q.put((obj, event_type))
+
+    create_watch_thread(get_list_method, put_queue, namespace, typename=typename).start()
+    while True:
+        try:
+            yield q.get(timeout=timeout)
+        except queue.Empty:
+            break
+
+
 class WatchingObjectSource(ty.Generic[T]):
     """Efficiently 'get' objects by reliably watching for changes to all such objects in a given namespace.
 
@@ -184,21 +256,11 @@ class WatchingObjectSource(ty.Generic[T]):
         self._limiter = OneShotLimiter()
 
     def _start_thread(self, namespace: str) -> None:
-        threading.Thread(
-            target=callback_events,
-            args=(
-                self._add_object,
-                yield_objects_from_list(
-                    namespace,
-                    self._get_list_method_on_restart,
-                    object_type_hint=self.typename + "s",
-                    init=lambda: logger.info(STARTING(f"Watching {self.typename}s in {namespace}")),
-                ),
-            ),
-            daemon=True,
+        create_watch_thread(
+            self.get_list_method, self._add_object, namespace, typename=self.typename
         ).start()
 
-    def _add_object(self, namespace: str, obj: T) -> None:
+    def _add_object(self, namespace: str, obj: T, _event_type: EventType) -> None:
         """This is where we receive updates from the k8s API."""
         self._last_api_update_time = time.monotonic()
 
@@ -211,19 +273,6 @@ class WatchingObjectSource(ty.Generic[T]):
         self._last_seen_time_by_name[name] = time.monotonic()
         self._objs_by_name[name] = obj
 
-    def _get_list_method_on_restart(
-        self, namespace: str, exc: ty.Optional[Exception]
-    ) -> ty.Optional[K8sList[T]]:
-        suffix = ""
-        if exc:
-            too_old = parse_too_old_resource_version(exc)
-            if not too_old:
-                logger.exception(f"Not fatal, but sleeping before we retry {self.typename} scraping...")
-                time.sleep(config.k8s_monitor_delay())
-                suffix = f" after {type(exc).__name__}: {exc}"
-                logger.info(f"Watching {self.typename}s in namespace: {namespace}{suffix}")
-        return self.get_list_method(namespace, exc)
-
     def _is_stale(self, name: str) -> bool:
         return is_stale(self._last_api_update_time, self._last_seen_time_by_name.get(name) or 0)
 
@@ -250,7 +299,7 @@ class WatchingObjectSource(ty.Generic[T]):
             # doing a lot of manual fetches may indicate that the k8s API is having trouble keeping up...
             try:
                 if obj := self.backup_fetch(namespace, obj_name):
-                    self._add_object(namespace, obj)  # updates last seen, too
+                    self._add_object(namespace, obj, "FETCH")  # updates last seen, too
                     return obj
 
             except Exception:

{thds_mops-3.6.20250407135813.dist-info → thds_mops-3.6.20250407153122.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: thds.mops
-Version: 3.6.20250407135813
+Version: 3.6.20250407153122
 Summary: ML Ops tools for Trilliant Health
 Author-email: Trilliant Health <info@trillianthealth.com>
 Project-URL: Repository, https://github.com/TrilliantHealth/ds-monorepo

{thds_mops-3.6.20250407135813.dist-info → thds_mops-3.6.20250407153122.dist-info}/RECORD

@@ -20,18 +20,18 @@ thds/mops/k8s/__init__.py,sha256=y4bvR5DQLiriBV0I8CZoQdNlNrXWQ2VpcxY5lOrdnzU,669
 thds/mops/k8s/_shared.py,sha256=MR-s6ijWUHZGjxK_fsOpHuRDB6kuofjo5xiIb7ul2VM,86
 thds/mops/k8s/apply_yaml.py,sha256=hVW6dIVbNdzHdbGlc2VAPGkdByv_rH2oPybyIm7tKIM,820
 thds/mops/k8s/auth.py,sha256=mXFPZvyJYEPASsBatv1r8syB9AoayuHGptHHnNUg8LE,1517
-thds/mops/k8s/config.py,sha256=AeOhstn2OhMvq53Zt0pSP0EF4DPmuCSLYKkTXHhk4ZQ,1876
+thds/mops/k8s/config.py,sha256=FyzuiUaNq8mQR01GqrezQWSc21QVqjLY9iaJUGu5WAo,2568
 thds/mops/k8s/container_registry.py,sha256=qOiGCE4t_tLYgJDGrhKV9KNv48lF_AlwCDHyFgucd2s,539
-thds/mops/k8s/jobs.py,sha256=4jVoj-mo__VYVdzfLwueHr_n192qBM3kKosWikkmcsU,1506
-thds/mops/k8s/launch.py,sha256=JF2de7iQFu_-n2vtXJQaQtwqV2qwCMiF329l-V6jdgY,9255
-thds/mops/k8s/logging.py,sha256=K7wVam88rgi09EczomtY0YyD2uJUNjlr2Pi8ViwHCls,9409
+thds/mops/k8s/jobs.py,sha256=3u0jc5Fnll2ncnmcdTUHlcxJ_KYNK9s66W7r6ez49As,3271
+thds/mops/k8s/launch.py,sha256=PXAcqFSLp839bKhu13RPFVH-UzOdS6u7hm_J-N5Kkeg,10087
+thds/mops/k8s/logging.py,sha256=yJ5Gt_luyo6LXbUJSXbhA4f1z7il1WuHQjMcP8eB00U,9591
 thds/mops/k8s/namespace.py,sha256=Z6trVTU9WFashto4PqIhTcxu-foOF93W0TpgqCU7WIA,383
 thds/mops/k8s/node_selection.py,sha256=Gy2Jz8IxZblg2LmtGg8-MtKI4RmXz2AMXqFPP8OQyu0,2065
 thds/mops/k8s/retry.py,sha256=JVfP304kItpLs5nrONHE5UWkVWlrFGlV_oFQqhq3zHg,2846
 thds/mops/k8s/too_old_resource_version.py,sha256=S7ltVA-LrxUpQ8Q__AB0nQmezN8Mmnx5oKK62_baAKI,1500
 thds/mops/k8s/wait_job.py,sha256=aCzjuhkiSnhW5SkOgRM1ufuMavAuOztkvn2mM2Rx1Ok,2440
-thds/mops/k8s/warn_image_backoff.py,sha256=a0b2OQf0VOG0n0W1nVOIOaazLSUATPFIDWYpNoHfEx0,1916
-thds/mops/k8s/watch.py,sha256=M2SSPm_IER7s45OO_f7gIAGeFpzmADgjRus1uEZtA7Y,9733
+thds/mops/k8s/warn_image_backoff.py,sha256=sUoKHaT7eh5C26veb8X7FH1CnBNBE9kX3ZMEDaH9kK4,1929
+thds/mops/k8s/watch.py,sha256=zeexTpXMoWKuhDOfdawmzWxMWkX__XutecueYXSADqY,11420
 thds/mops/k8s/tools/krsync.py,sha256=us7pXX0-bRMwD2oAno7Z6BJcPs6FgaUabHW0STyQJYg,1773
 thds/mops/k8s/tools/krsync.sh,sha256=lskw4COt51Bv1yy2IAYUc8u8uQV-coSyUiOT8rADKkQ,546
 thds/mops/pure/__init__.py,sha256=kbG0lMvXRBS3LGbb2gPPE9-qjYMXrypyb2tJX2__aZc,1533
@@ -103,8 +103,8 @@ thds/mops/pure/tools/summarize/cli.py,sha256=gaechsJhRZsOxGJGG1dQsW5dMBlgSv2sUmE
 thds/mops/pure/tools/summarize/run_summary.py,sha256=HCfAJhgAa9u6kXbzJlHa2n-9vVTaTHYSxrN_DP-Sjo4,4892
 thds/mops/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/testing/deferred_imports.py,sha256=f0ezCgQAtzTqW1yAOb0OWgsB9ZrlztLB894LtpWDaVw,3780
-thds_mops-3.6.20250407135813.dist-info/METADATA,sha256=gRS0ByEEpaFHqCO1I7oA50_D_MZG4AikVQMkdCvbHws,2161
-thds_mops-3.6.20250407135813.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-thds_mops-3.6.20250407135813.dist-info/entry_points.txt,sha256=GShNqjcjbq0TAJuwpyeCI5XCltiwdZxnNHkBpmYbNkU,329
-thds_mops-3.6.20250407135813.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_mops-3.6.20250407135813.dist-info/RECORD,,
+thds_mops-3.6.20250407153122.dist-info/METADATA,sha256=Mxupz7gO3eUSDt4fmHtAmmb4CyTmxZ9SY5ayxJuio5A,2161
+thds_mops-3.6.20250407153122.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+thds_mops-3.6.20250407153122.dist-info/entry_points.txt,sha256=GShNqjcjbq0TAJuwpyeCI5XCltiwdZxnNHkBpmYbNkU,329
+thds_mops-3.6.20250407153122.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_mops-3.6.20250407153122.dist-info/RECORD,,