thds.mops 3.9.20250721231027__py3-none-any.whl → 3.9.20250722163657__py3-none-any.whl

thds/mops/impure/runner.py

@@ -67,7 +67,7 @@ class KeyedLocalRunner(MemoizingPicklingRunner):
             redirect=lambda _f, _args, _kwargs: _perform_original_invocation,
         )
 
-    def __call__(self, raw_func: ty.Callable[P, R], raw_args: P.args, raw_kwargs: P.kwargs) -> R:  # type: ignore[valid-type]
+    def __call__(self, raw_func: ty.Callable[P, R], raw_args: P.args, raw_kwargs: P.kwargs) -> R:
         actual_function_to_call = self._pre_pickle_redirect(raw_func, raw_args, raw_kwargs)
         with _ORIGINAL_F_ARGS_KWARGS.set((actual_function_to_call, raw_args, raw_kwargs)):
             return super().__call__(*self._impure_keyfunc(raw_func, raw_args, raw_kwargs))

thds/mops/k8s/__init__.py

@@ -7,10 +7,8 @@ except ModuleNotFoundError as mnf:
         "Please install mops with the `k8s` extra to use `thds.mops.k8s`."
     ) from mnf
 
-from . import batching, counts, job_future  # noqa: F401
-from ._launch import launch, shim  # noqa
 from .container_registry import autocr  # noqa: F401
-from .job_future import K8sJobFailedError  # noqa: F401
+from .launch import K8sJobFailedError, launch, shim  # noqa
 from .node_selection import (  # noqa
     NodeNarrowing,
     ResourceDefinition,

thds/mops/k8s/config.py

@@ -10,7 +10,7 @@ k8s_namespace_env_var_key = config.item("mops.k8s.namespace_env_var_key", "MOPS_
 # environment variable.  it will not affect how your namespace is selected in the first
 # place.
 
-k8s_watch_object_stale_seconds = config.item("mops.k8s.watch.object_stale_seconds", 5 * 60, parse=int)
+k8s_watch_object_stale_seconds = config.item("mops.k8s.watch.object_stale_seconds", 30 * 60, parse=int)
 k8s_acr_url = config.item("mops.k8s.acr.url", "")
 k8s_job_retry_count = config.item("mops.k8s.job.retry_count", 6, parse=int)
 k8s_job_cleanup_ttl_seconds_after_completion = config.item(

thds/mops/k8s/jobs.py

@@ -25,10 +25,6 @@ _JOB_SOURCE = WatchingObjectSource(
 )
 
 
-def job_source() -> WatchingObjectSource[client.models.V1Job]:
-    return _JOB_SOURCE
-
-
 def get_job(job_name: str, namespace: str = "") -> ty.Optional[client.models.V1Job]:
     return _JOB_SOURCE.get(job_name, namespace=namespace)
 

thds/mops/k8s/{_launch.py → launch.py}

@@ -4,22 +4,41 @@ import os
 import threading
 import typing as ty
 import uuid
-from functools import partial
 
 from kubernetes import client
 
-from thds import core
+from thds.core import scope
+from thds.core.log import logger_context
 from thds.mops.pure.runner.simple_shims import samethread_shim
 from thds.termtool.colorize import colorized
 
-from . import config, counts, job_future, logging
+from . import config
 from ._shared import logger
 from .auth import load_config, upsert_namespace
+from .logging import JobLogWatcher
 from .node_selection import NodeNarrowing, ResourceDefinition
 from .retry import k8s_sdk_retry
 from .thds_std import embed_thds_auth
+from .wait_job import wait_for_job
 
 LAUNCHED = colorized(fg="white", bg="green")
+COMPLETE = colorized(fg="white", bg="blue")
+FAILED = colorized(fg="white", bg="red")
+
+
+class K8sJobFailedError(Exception):
+    """Raised by `launch` when a Job is seen to terminate in a Failed state."""
+
+
+class Counter:
+    def __init__(self) -> None:
+        self.value = 0
+        self._lock = threading.Lock()
+
+    def inc(self) -> int:
+        with self._lock:
+            self.value += 1
+            return self.value
 
 
 def sanitize_str(name: str) -> str:
@@ -30,7 +49,7 @@ def sanitize_str(name: str) -> str:
 
 def construct_job_name(user_prefix: str, job_num: str) -> str:
     # we want some consistency here, but also some randomness in case the prefixes don't exist or aren't unique.
-    mops_name_part = "-".join([sanitize_str(job_num), str(os.getpid()), str(uuid.uuid4())[:8]])
+    mops_name_part = "-".join([str(os.getpid()), sanitize_str(job_num), str(uuid.uuid4())[:8]])
     if len(mops_name_part) > 63:
         # this should be _impossible_, because having a job num longer than even 20 digits would be an impossibly large
         # number of jobs. but just in case, we'll truncate it to the last 63 characters.
@@ -46,11 +65,12 @@ def construct_job_name(user_prefix: str, job_num: str) -> str:
     return name
 
 
+_LAUNCH_COUNT = Counter()
+_FINISH_COUNT = Counter()
 _SIMULTANEOUS_LAUNCHES = threading.BoundedSemaphore(20)
-JOB_NAME = core.stack_context.StackContext("job_name", "")
 
 
-@core.scope.bound
+@scope.bound
 def launch(
     container_image: str,
     args: ty.Sequence[str],
@@ -61,46 +81,38 @@ def launch(
     # arguments below are for launching; arguments above are for
     # building.  these should get separated in a future change.
     name_prefix: str = "",
-    full_name: str = "",
     dry_run: bool = False,
+    fire_and_forget: bool = False,
     suppress_logs: bool = False,
     transform_job: ty.Callable[[client.models.V1Job], client.models.V1Job] = embed_thds_auth,
     # this is a default for now. later if we share this code we'll need to have a wrapper interface
     service_account_name: str = "",
-) -> core.futures.LazyFuture[bool]:
+) -> None:
     """Launch a Kubernetes job.
 
     Required parameters are the container_image and the arguments to
     that image, just as if you were running this directly with Docker.
 
-    Returns a Future that will resolve to True when the Job completes successfully, or
-    raise K8sJobFailedError if the Job fails.
+    Unless fire_and_forget=True, will poll until Job completes and
+    will raise K8sJobFailedError if the Job fails. None is returned
+    if the Job succeeds.
 
     `name_prefix` is an optional parameter for debugging/developer
     convenience. A generated suffix will be added to it.
+
     """
     if not container_image:
         raise ValueError("container_image (the fully qualified Docker tag) must not be empty.")
-
-    full_name = full_name or JOB_NAME()
-    # in certain cases, it may be necessary to set the job name
-    # via a StackContext, so we check that here, and prefer it over name_prefix.
-
-    if full_name and name_prefix:
-        raise ValueError("You cannot specify both full_name and name_prefix; use one or the other.")
-
-    if not full_name:
-        name = construct_job_name(name_prefix, counts.to_name(counts.inc(counts.LAUNCH_COUNT)))
-    else:
-        name = full_name
-
-    core.scope.enter(core.log.logger_context(job=name))
+    job_num = f"{_LAUNCH_COUNT.inc():0>3}"
+    name = construct_job_name(name_prefix, job_num)
+    scope.enter(logger_context(job=name))
     node_narrowing = node_narrowing or dict()
 
     # TODO move this entire function out to be separately callable
     @k8s_sdk_retry()
     def assemble_base_job() -> client.models.V1Job:
         logger.debug(f"Assembling job named `{name}` on image `{container_image}`")
+        logger.debug("Fire and forget: %s", fire_and_forget)
         logger.debug("Loading kube configs ...")
         load_config()
         logger.debug("Populating job object ...")
@@ -173,7 +185,7 @@ def launch(
     if dry_run:
         job_with_all_transforms()
         logger.info("Dry run assembly successful; not launching...")
-        return core.futures.LazyFuture(partial(core.futures.ResolvedFuture, True))
+        return
 
     @k8s_sdk_retry()
     def launch_job() -> client.models.V1Job:
@@ -186,41 +198,32 @@ def launch(
             )
 
     job = launch_job()
-    logger.info(LAUNCHED(f"Job {name} launched!") + f" on {container_image}")
-    return core.futures.make_lazy(_launch_logs_and_create_future)(  # see below for implementation
-        job.metadata.name,
-        num_pods_expected=len(job.spec.template.spec.containers),
-        namespace=config.k8s_namespace(),
-        suppress_logs=suppress_logs,
-    )
-
-
-# this function has to be a top level def because it will sometimes be transferred across process boundaries,
-# and Python/pickle in its infinite wisdom does not allow nested functions to be pickled.
-def _launch_logs_and_create_future(
-    job_name: str, *, num_pods_expected: int, namespace: str, suppress_logs: bool
-) -> core.futures.PFuture[bool]:
+    logger.info(LAUNCHED(f"Job {job_num} launched!") + f" on {container_image}")
     if not suppress_logs:
-        logging.maybe_start_job_thread(job_name, num_pods_expected)
-    return job_future.make_job_completion_future(job_name, namespace=namespace)
+        threading.Thread(  # fire and forget a log watching thread
+            target=JobLogWatcher(job.metadata.name, len(job.spec.template.spec.containers)).start,
+            daemon=True,
+        ).start()
+
+    if not fire_and_forget:
 
+        def counts() -> str:
+            launched = _LAUNCH_COUNT.value
+            return f"- ({launched - _FINISH_COUNT.inc()} unfinished of {launched})"
 
-def create_lazy_job_logging_future(
-    job_name: str, *, namespace: str = "", num_pods_expected: int = 1
-) -> core.futures.LazyFuture[bool]:
-    return core.futures.make_lazy(_launch_logs_and_create_future)(
-        job_name,
-        num_pods_expected=num_pods_expected,
-        namespace=namespace or config.k8s_namespace(),
-        suppress_logs=False,
-    )
+        job_name = job.metadata.name
+        del job  # trying to save memory here while we wait...
+        if not wait_for_job(job_name, short_name=job_num):
+            logger.error(FAILED(f"Job {job_num} Failed! {counts()}"))
+            raise K8sJobFailedError(f"Job {job_name} failed.")
+        logger.info(COMPLETE(f"Job {job_num} Complete! {counts()}"))
 
 
 def shim(
     container_image: ty.Union[str, ty.Callable[[], str]],
     disable_remote: ty.Callable[[], bool] = lambda: False,
     **outer_kwargs: ty.Any,
-) -> ty.Callable[[ty.Sequence[str]], core.futures.LazyFuture[bool]]:
+) -> ty.Callable[[ty.Sequence[str]], None]:
     """Return a closure that can launch the given configuration and run a mops pure function.
 
     Now supports callables that return a container image name; the
@@ -237,18 +240,16 @@ def shim(
     ), "Passing 'args' as a keyword argument will cause conflicts with the closure."
 
     if disable_remote():
-        return samethread_shim  # type: ignore[return-value]
+        return samethread_shim
 
     if isinstance(container_image, str):
         get_container_image: ty.Callable[[], str] = lambda: container_image  # noqa: E731
     else:
         get_container_image = container_image
 
-    def launch_container_on_k8s_with_args(
-        args: ty.Sequence[str], **inner_kwargs: ty.Any
-    ) -> core.futures.LazyFuture[bool]:
+    def launch_container_on_k8s_with_args(args: ty.Sequence[str], **inner_kwargs: ty.Any) -> None:
         assert "args" not in inner_kwargs
-        return launch(
+        launch(
             get_container_image(),
             ["python", "-m", "thds.mops.pure.core.entry.main", *args],
             **{**outer_kwargs, **inner_kwargs},

thds/mops/k8s/logging.py

@@ -32,17 +32,6 @@ BOINK = colorized(fg="white", bg="magenta")
 # string in this and it'll stand out.
 
 
-def should_log(job_name: str) -> bool:
-    if NO_K8S_LOGS():
-        return False
-
-    if random.random() > K8S_LOG_POD_FRACTION():
-        logger.info(f"Skipping log watcher for {job_name} due to fraction.")
-        return False
-
-    return True
-
-
 class JobLogWatcher:
     """Will spawn one or more daemon threads.
 
@@ -70,7 +59,11 @@ class JobLogWatcher:
     @core.scope.bound
     def start(self, failed_pod_name: str = "") -> None:
         """Call this one time - it will spawn threads as needed."""
-        if not should_log(self.job_name):
+        if NO_K8S_LOGS():
+            return
+
+        if random.random() > K8S_LOG_POD_FRACTION():
+            logger.info(f"Skipping log watcher for {self.job_name} due to fraction.")
             return
 
         core.scope.enter(self.job_pods_discovery_lock)
@@ -252,28 +245,3 @@ def _scrape_pod_logs(
         logger.exception(BOINK("Pod log scraping failed utterly. Pod may have died?"))
         # at least let the caller know something went horribly wrong
         failure_callback(pod_name)
-
-
-_JOB_LOG_THREADS: set[str] = set()
-_JOB_LOG_THREAD_COUNT: int = 0
-_JOB_LOG_THREADS_LOCK = threading.Lock()
-
-
-def maybe_start_job_thread(job_name: str, num_pods_expected: int = 1) -> bool:
-    """Starts a thread to watch the logs of a job. Makes sure we only start one thread per
-    job even if there are multiple calls to this function.
-    """
-    if job_name not in _JOB_LOG_THREADS:
-        with _JOB_LOG_THREADS_LOCK:
-            if job_name not in _JOB_LOG_THREADS:
-                # double-checked locking to avoid creating multiple threads for the same job
-                _JOB_LOG_THREADS.add(job_name)
-                if should_log(job_name):
-                    global _JOB_LOG_THREAD_COUNT
-                    _JOB_LOG_THREAD_COUNT += 1
-                    logger.info(f"Starting log watcher {_JOB_LOG_THREAD_COUNT} for job {job_name}")
-                    threading.Thread(
-                        target=JobLogWatcher(job_name, num_pods_expected).start, daemon=True
-                    ).start()
-                    return True
-    return False

thds/mops/k8s/watch.py

@@ -12,20 +12,17 @@ import urllib3
 from kubernetes import client
 from kubernetes import watch as k8s_watch
 
-from thds.core import futures, scope
+from thds.core import scope
 from thds.core.log import getLogger, logger_context
 from thds.termtool.colorize import colorized
 
 from . import config
 from .auth import load_config
 from .too_old_resource_version import parse_too_old_resource_version
-from .uncertain_future import FutureInterpreter, UncertainFuturesTracker
 
 logger = getLogger(__name__)
 
 T = ty.TypeVar("T")
-K = ty.TypeVar("K")
-R = ty.TypeVar("R")
 
 
 class V1List(ty.Protocol[T]):
@@ -118,6 +115,10 @@ def callback_events(
             break
 
 
+def _make_name(namespace: str, name: str) -> str:
+    return f"{namespace}/{name}"
+
+
 def _default_get_name(obj: ty.Any) -> str:
     return obj.metadata.name
 
@@ -147,15 +148,9 @@ class OneShotLimiter:
             self._names.add(name)
 
 
-def _watch_timer() -> float:
-    # in this context, monotonicity (actual timing) is most useful because we don't need sentinels.
-    return time.monotonic()
-
-
 def is_stale(api_last_update_time: float, obj_last_seen_time: float) -> bool:
-    now = _watch_timer()
+    now = time.monotonic()
     allowed_stale_seconds = config.k8s_watch_object_stale_seconds()
-    # about 5 minutes by default as of 2025-07-15.
     if (time_since_api_update := now - api_last_update_time) > allowed_stale_seconds:  # noqa: F841
         # we haven't heard anything from the API in a while; probably
         # the API is down. Ignore object staleness to avoid false positives.
@@ -228,146 +223,93 @@ def watch_forever(
             break
 
 
-class _SeenObjectContainer(ty.Generic[K, T]):
-    """Splits some of the logic for 'get' out of WatchingObjectSource
-    so that we can have it be a simpler container for both this and the UncertainFuturesTracker.
+class WatchingObjectSource(ty.Generic[T]):
+    """Efficiently 'get' objects by reliably watching for changes to all such objects in a given namespace.
+
+    This is network-efficient for observing many different objects,
+    but not memory efficient if you really only need to fetch details
+    for a few objects.
     """
 
     def __init__(
         self,
-        backup_fetch: ty.Optional[ty.Callable[[K], ty.Optional[T]]] = None,
+        get_list_method: GetListMethod[T],
+        get_name: ty.Callable[[T], str] = ty.cast(  # noqa: B008
+            ty.Callable[[T], str], _default_get_name
+        ),
+        backup_fetch: ty.Optional[ty.Callable[[str, str], T]] = None,
+        typename: str = "object",
+        starting: ty.Callable[[str], str] = STARTING,
     ) -> None:
-        self._objs: ty.Dict[K, T] = dict()
+        self.get_list_method = get_list_method
+        self.get_name = get_name
+        self.backup_fetch = backup_fetch
+        self.typename = typename
+        self._objs_by_name: ty.Dict[str, T] = dict()
         # ^ is a possibly big/expensive local cache of the most recent
         # state for all of the event type in the namespace.  Don't use
         # this class if you can't afford the memory overhead of
         # observing everything in your namespace and keeping the last
         # known copy of everything forever.
-        self._last_seen_times: ty.Dict[K, float] = dict()
+        self._last_seen_time_by_name: ty.Dict[str, float] = dict()
         self._last_api_update_time = 0.0
-        self.backup_fetch = backup_fetch
+        self._limiter = OneShotLimiter()
 
-    def set_object(self, key: K, obj: T) -> None:
-        """Set an object in the cache, updating the last seen time."""
-        now = _watch_timer()
-        self._last_api_update_time = now
-        self._last_seen_times[key] = now
-        self._objs[key] = obj
+    def _start_thread(self, namespace: str) -> None:
+        create_watch_thread(
+            self.get_list_method, self._add_object, namespace, typename=self.typename
+        ).start()
+
+    def _add_object(self, namespace: str, obj: T, _event_type: EventType) -> None:
+        """This is where we receive updates from the k8s API."""
+        self._last_api_update_time = time.monotonic()
+
+        if not obj:
+            logger.warning(f"Received null/empty {self.typename}")
+            return
 
-    def _is_stale(self, key: K) -> bool:
-        return is_stale(self._last_api_update_time, self._last_seen_times.get(key) or 0)
+        name = _make_name(namespace, self.get_name(obj))
+        logger.debug(f"{self.typename} {name} updated")
+        self._last_seen_time_by_name[name] = time.monotonic()
+        self._objs_by_name[name] = obj
+
+    def _is_stale(self, name: str) -> bool:
+        return is_stale(self._last_api_update_time, self._last_seen_time_by_name.get(name) or 0)
+
+    @scope.bound
+    def get(self, obj_name: str, namespace: str = "") -> ty.Optional[T]:
+        namespace = namespace or config.k8s_namespace()
+        name = _make_name(namespace, obj_name)
+        scope.enter(logger_context(name=obj_name, namespace=namespace))
 
-    def get(self, key: K) -> ty.Optional[T]:
         # first try is looking in our local cache
-        if (obj := self._objs.get(key)) and not self._is_stale(key):
+        if (obj := self._objs_by_name.get(name)) and not self._is_stale(name):
             return obj
 
         # second try is making sure the namespace watcher is running, sleeping, and then looking in the cache again.
         # This is much more efficient than a manual fetch.
+        self._limiter(namespace, self._start_thread)
         time.sleep(config.k8s_monitor_delay())
-        if (obj := self._objs.get(key)) and not self._is_stale(key):
+        if (obj := self._objs_by_name.get(name)) and not self._is_stale(name):
             return obj
 
         # if that doesn't work, try a manual fetch.
         if self.backup_fetch:
-            logger.warning(f"Manually fetching {key}...")
+            logger.warning(f"Manually fetching {self.typename}...")
             # doing a lot of manual fetches may indicate that the k8s API is having trouble keeping up...
             try:
-                if obj := self.backup_fetch(key):
-                    self.set_object(key, obj)  # updates last seen, too
+                if obj := self.backup_fetch(namespace, obj_name):
+                    self._add_object(namespace, obj, "FETCH")  # updates last seen, too
                     return obj
 
             except Exception:
-                logger.exception(f"Unexpected error during manual fetch of {key}.")
+                logger.exception(f"Unexpected error during manual fetch of {self.typename}.")
 
-        if self._is_stale(key):
+        if self._is_stale(name):
             logger.warning(
-                f"Could not refresh {key}, and our record of it is stale - dropping stale object!"
+                f"Could not refresh {name}, and our record of it is stale - dropping stale object!"
             )
-            self._objs.pop(key, None)
-            self._last_seen_times.pop(key, None)
+            self._objs_by_name.pop(name, None)
+            self._last_seen_time_by_name.pop(name, None)
 
         return None
-
-
-class WatchingObjectSource(ty.Generic[T]):
-    """Efficiently 'get' objects by launching a single thread to
-    watch for changes to all such objects in a given namespace.
-
-    Also provide a way to create a future that will be resolved according to the logic
-    provided by the caller whenever an object is updated, or if the object has not been
-    updated in a while.
-
-    Importantly, the Futures are only prevented from deadlocking (never awakening their
-    condition variable) by the fact that we very occasionally will go through the list
-    of seen objects and raise Exceptions for objects that have not been updated in a while.
-    This is vaguely akin to garbage collection, in that it will occasionally
-    cause a 'pause' in the watcher thread as it tries to collect stale objects.
-
-    This is network-efficient for observing many different objects,
-    but not memory efficient if you really only need to fetch details
-    for a few objects, because we retain the last known state for every observed object indefinitely.
-    """
-
-    def __init__(
-        self,
-        get_list_method: GetListMethod[T],
-        get_name: ty.Callable[[T], str] = ty.cast(  # noqa: B008
-            ty.Callable[[T], str], _default_get_name
-        ),
-        backup_fetch: ty.Optional[ty.Callable[[str, str], ty.Optional[T]]] = None,
-        typename: str = "object",
-    ) -> None:
-        self.get_list_method = get_list_method
-        self.get_name = get_name
-        self.typename = typename
-        self._limiter = OneShotLimiter()
-        self._uncertain_futures = UncertainFuturesTracker[tuple[str, str], T](
-            config.k8s_watch_object_stale_seconds()
-        )
-        self._seen_objects = _SeenObjectContainer[tuple[str, str], T](
-            lambda namespace_and_name: backup_fetch(*namespace_and_name) if backup_fetch else None
-        )
-
-    def _add_object(self, namespace: str, obj: T, _event_type: EventType) -> None:
-        """This is where we receive updates from the k8s API."""
-        if not obj:
-            logger.warning(f"Received null/empty {self.typename}")
-            return
-
-        key = (namespace, self.get_name(obj))
-        self._seen_objects.set_object(key, obj)
-        self._uncertain_futures.update(key, obj)
-        logger.debug("%s %s updated", self.typename, key)
-
-    def _start_namespace_watcher_thread(self, namespace: str) -> None:
-        create_watch_thread(
-            self.get_list_method, self._add_object, namespace, typename=self.typename
-        ).start()
-
-    @scope.bound
-    def get(self, obj_name: str, namespace: str = "") -> ty.Optional[T]:
-        """May block for a little while if a manual fetch is required."""
-        namespace = namespace or config.k8s_namespace()
-        scope.enter(logger_context(name=obj_name, namespace=namespace))
-        self._limiter(namespace, self._start_namespace_watcher_thread)
-        return self._seen_objects.get((namespace, obj_name))
-
-    def create_future(
-        self,
-        interpreter: FutureInterpreter[T, R],
-        obj_name: str,
-        *,
-        namespace: str = "",
-    ) -> futures.PFuture[R]:
-        """Create a future that will be resolved when the object is available according to
-        the interpreter.
-
-        The FutureInterpreter must:
-         - raise an exception if it wishes the future to raise.
-         - return a Done with the result if it wishes the future to resolve successfully.
-          -return None if the status is still in progress.
-        """
-        namespace = namespace or config.k8s_namespace()
-        self._limiter(namespace, self._start_namespace_watcher_thread)
-        return self._uncertain_futures.create((namespace, obj_name), interpreter)

thds/mops/pure/__init__.py

@@ -7,7 +7,6 @@
 from . import adls  # noqa
 from ._magic.api import magic  # noqa
 from .core.entry import register_entry_handler
-from .core.lock.maintain import no_maintain as no_maintain_locks  # noqa: F401
 from .core.memo import results  # noqa
 from .core.memo.function_memospace import (  # noqa
     add_pipeline_memospace_handlers,
@@ -22,7 +21,7 @@ from .core.use_runner import use_runner  # noqa
 from .pickling.memoize_only import memoize_in  # noqa
 from .pickling.mprunner import MemoizingPicklingRunner  # noqa
 from .runner.simple_shims import samethread_shim, subprocess_shim  # noqa
-from .runner.types import FutureShim, Shim, ShimBuilder  # noqa
+from .runner.types import Shim, ShimBuilder  # noqa
 
 
 def _register_things() -> None:

thds/mops/pure/_magic/sauce.py

@@ -6,7 +6,7 @@ import typing as ty
 
 from typing_extensions import ParamSpec
 
-from thds.core import futures, stack_context
+from thds.core import stack_context
 from thds.mops._utils import config_tree
 
 from ..core import file_blob_store, pipeline_id, pipeline_id_mask, uris
@@ -109,7 +109,7 @@ class Magic(ty.Generic[P, R]):
     def _is_off(self) -> bool:
         return self._shim_builder_or_off is None
 
-    def _shimbuilder(self, f: ty.Callable[P, R], args: P.args, kwargs: P.kwargs) -> Shim:  # type: ignore[valid-type]
+    def _shimbuilder(self, f: ty.Callable[P, R], args: P.args, kwargs: P.kwargs) -> Shim:
         # this can be set using a stack-local context, or set globally as specifically
         # or generally as the user needs. We prefer stack local over everything else.
         sb = self._shim_builder_or_off
@@ -123,16 +123,8 @@ class Magic(ty.Generic[P, R]):
     def _pipeline_id(self) -> str:
         return self.config.pipeline_id.getv(self._func_config_path)
 
-    def submit(self, *args: P.args, **kwargs: P.kwargs) -> futures.PFuture[R]:
-        """A futures-based interface that doesn't block on the result of the wrapped
-        function call, but returns a PFuture once either a result has been found or a a
-        new invocation has been started.
-        """
-        with pipeline_id.set_pipeline_id_for_stack(self._pipeline_id):
-            return self.runner.submit(self.__wrapped__, *args, **kwargs)
-
     def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R:
-        """This is the wrapped function - call this as though it were the function itself."""
+        """This is the wrapped function."""
         with pipeline_id.set_pipeline_id_for_stack(self._pipeline_id):
             return self._func(*args, **kwargs)
 

thds/mops/pure/_magic/shims.py

@@ -4,14 +4,14 @@ from thds import core
 
 from ..runner.shim_builder import make_builder
 from ..runner.simple_shims import samethread_shim, subprocess_shim
-from ..runner.types import FutureShim, Shim, ShimBuilder
+from ..runner.types import Shim, ShimBuilder
 
 ShimName = ty.Literal[
     "samethread",  # memoization and coordination, but run in the same thread as the caller.
     "subprocess",  # memoization and coordination, but transfer to a subprocess rather than remote.
     "off",  # equivalent to None - disables use of mops.
 ]
-ShimOrBuilder = ty.Union[ShimBuilder, Shim, FutureShim]
+ShimOrBuilder = ty.Union[ShimBuilder, Shim]
 logger = core.log.getLogger(__name__)