thds.mops 3.6.20250219172032__py3-none-any.whl

thds/mops/pure/pickling/pickles.py

@@ -0,0 +1,149 @@
+"""This module is a good place to define actual objects that need to
+pickled in a backward-compatible way - i.e., we want to remember not
+to refactor their names or the name of the module they live in so as
+to maintain backward-compatibility more easily.
+"""
+
+import importlib
+import io
+import pickle
+import typing as ty
+from dataclasses import dataclass
+from pathlib import Path
+
+from thds.core import hashing, log, source
+
+from ..core.script_support import add_main_module_function, get_main_module_function
+from ..core.source import source_from_hashref, source_from_source_result
+from ..core.uris import get_bytes, lookup_blob_store
+
+logger = log.getLogger(__name__)
+
+
+@dataclass
+class Invocation:
+    """Basically, NestedFunctionPickle was the v2. This is v3. By switching to dataclass,
+    we can more easily add new optional attributes later on.
+    """
+
+    func: ty.Callable
+    args_kwargs_pickle: bytes
+    # this is pickled separately so that we can hash it separately.
+    # the identity of the function is represented by the name part of the blob path.
+
+
+class NestedFunctionPickle(ty.NamedTuple):
+    """Not in use - retained for mops-inspect backward-compatibility"""
+
+    f: ty.Callable
+    args_kwargs_pickle: bytes
+
+
+class PicklableFunction:
+    """The main 'issue' this is working around is that decorated
+    functions aren't picklable because of something having to do with
+    the way the function gets 'replaced' at decoration time.
+
+    There may be other solutions to this, but this seems to work fine.
+    """
+
+    def __init__(self, f: ty.Callable) -> None:
+        if f.__module__ == "__main__":
+            add_main_module_function(f.__name__, f)
+        self.fmod = f.__module__
+        self.fname = f.__name__
+        self.f = None
+
+    def __str__(self) -> str:
+        return f"{self.fmod}.{self.fname}"
+
+    def __repr__(self) -> str:
+        return str(self)
+
+    @property
+    def __name__(self) -> str:
+        return self.fname
+
+    def __call__(self, *args: ty.Any, **kwargs: ty.Any) -> ty.Any:
+        logger.debug(f"Dynamically importing function {str(self)}")
+        if self.fmod == "__main__":
+            self.f = get_main_module_function(self.fname)  # type: ignore
+        else:
+            mod = importlib.import_module(self.fmod)
+            self.f = getattr(mod, self.fname)
+        assert self.f
+        return self.f(*args, **kwargs)
+
+
+class UnpickleSimplePickleFromUri:
+    def __init__(self, uri: str):
+        self.uri = uri  # serializable as a pure string for simplicity
+        self._cached = None
+
+    def __call__(self) -> object:
+        # i don't believe there's any need for thread safety here, since pickle won't use threads.
+        if self._cached is None:
+            self._cached = pickle.load(io.BytesIO(get_bytes(self.uri, type_hint="simple-uri-pickle")))
+        return self._cached
+
+
+class UnpicklePathFromUri(ty.NamedTuple):
+    uri: str
+
+    def __call__(self) -> Path:
+        return lookup_blob_store(self.uri).getfile(self.uri)
+
+
+class UnpickleSourceUriArgument(ty.NamedTuple):
+    """The URI fully specifies this type of source. Nothing fancy happens here.  We just
+    return a new Source object that represents the URI.
+    """
+
+    uri: str
+
+    def __call__(self) -> source.Source:
+        return source.from_uri(self.uri)
+
+
+class UnpickleSourceHashrefArgument(ty.NamedTuple):
+    """Represents the root for a single file hashref. May be either local or remote.
+
+    For stability, the module name and the class name must not change.
+
+    This only applies to arguments _into_ a function. Results _from_ a function should
+    have a different form.
+    """
+
+    hash: hashing.Hash
+
+    def __call__(self) -> source.Source:
+        return source_from_hashref(self.hash)
+
+
+class UnpickleSourceResult(ty.NamedTuple):
+    """Stability for this is not critical, as it will only ever exist in the result
+    payload, which does not participate in memoization.
+    """
+
+    remote_uri: str
+    hash: ty.Optional[hashing.Hash]
+    file_uri: str
+
+    def __call__(self) -> source.Source:
+        return source_from_source_result(*self)
+
+
+class UnpickleFunctionWithLogicKey(ty.NamedTuple):
+    """When a mops-memoized function receives, in standard "functional programming" style,
+    a function as an argument (whether partially-applied or not), we need to make
+    sure to represent any function-logic-key on that callable as part of what gets serialized,
+    so that memoization does not happen when unexpected/undesired.
+
+    The function itself must be picklable in the natural way.
+    """
+
+    func_bytes: bytes
+    function_logic_key: str
+
+    def __call__(self) -> ty.Callable:
+        return pickle.loads(self.func_bytes)

thds/mops/pure/pickling/remote.py

@@ -0,0 +1,145 @@
+import typing as ty
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from functools import cached_property
+
+from thds.core import log, scope
+
+from ..._utils.once import Once
+from ..core import lock, metadata, pipeline_id, uris
+from ..core.entry import route_return_value_or_exception
+from ..core.memo import results
+from ..core.pipeline_id_mask import pipeline_id_mask
+from ..core.serialize_big_objs import ByIdRegistry, ByIdSerializer
+from ..core.serialize_paths import CoordinatingPathSerializer
+from ..core.types import Args, BlobStore, Kwargs, T
+from ..runner import strings
+from . import _pickle, mprunner, pickles, sha256_b64
+
+logger = log.getLogger(__name__)
+
+
+@dataclass  # needed for cached_property
+class _ResultExcWithMetadataChannel:
+    fs: BlobStore
+    dumper: _pickle.Dumper
+    call_id: str
+    invocation_metadata: metadata.InvocationMetadata
+    started_at: datetime
+
+    @cached_property
+    def _metadata_header(self) -> bytes:
+        """This is always embedded _alongside_ the actual return value or exception.
+        This is to make sure that whatever metadata is in the result is atomically
+        part of the result, such that in the rare case of racing invocations,
+        the metadata can be trusted to be accurate.
+        """
+        result_metadata = metadata.ResultMetadata.from_invocation(
+            self.invocation_metadata, self.started_at, datetime.now(tz=timezone.utc)
+        )
+        logger.info(f"Remote code version: {result_metadata.remote_code_version}")
+        return metadata.format_result_header(result_metadata).encode("utf-8")
+
+    def _write_metadata_only(self, prefix: str) -> None:
+        """This is a mops v3 thing that is unnecessary but adds clarity when debugging.
+        If you see more than one of these files in a directory, that usually means either
+        the success was preceded by a failure, _or_ it means that there was an (unusual) race condition.
+        """
+        self.fs.putbytes(
+            self.fs.join(self.call_id, f"{prefix}-metadata-{self.invocation_metadata.invoker_uuid}.txt"),
+            self._metadata_header,
+            type_hint="text/plain",
+        )
+
+    def return_value(self, r: T) -> None:
+        return_value_bytes = _pickle.gimme_bytes(self.dumper, r)
+        self.fs.putbytes(
+            self.fs.join(self.call_id, results.RESULT),
+            self._metadata_header + return_value_bytes,
+            type_hint="application/mops-return-value",
+        )
+        self._write_metadata_only("result")
+
+    def exception(self, exc: Exception) -> None:
+        exc_bytes = _pickle.gimme_bytes(self.dumper, exc)
+        self.fs.putbytes(
+            self.fs.join(self.call_id, results.EXCEPTION),
+            self._metadata_header + exc_bytes,
+            type_hint="application/mops-exception",
+        )
+        self._write_metadata_only("exception")
+
+
+def _unpickle_invocation(memo_uri: str) -> ty.Tuple[ty.Callable, Args, Kwargs]:
+    _, invocation_raw = _pickle.make_read_header_and_object(strings.INVOCATION)(
+        uris.lookup_blob_store(memo_uri).join(memo_uri, strings.INVOCATION)
+    )
+    invocation = ty.cast(pickles.Invocation, invocation_raw)
+    args, kwargs = _pickle.unfreeze_args_kwargs(invocation.args_kwargs_pickle)
+    return invocation.func, args, kwargs
+
+
+def run_pickled_invocation(memo_uri: str, *metadata_args: str) -> None:
+    """The arguments are those supplied by MemoizingPicklingRunner.
+
+    As of v3, we now expect a number of (required) metadata arguments with every invocation.
+    """
+    started_at = datetime.now(tz=timezone.utc)  # capture this timestamp right at the outset.
+    invocation_metadata = metadata.parse_invocation_metadata_args(metadata_args)
+    metadata.INVOKED_BY.set_global(invocation_metadata.invoked_by)
+    pipeline_id.set_pipeline_id(invocation_metadata.pipeline_id)
+    fs = uris.lookup_blob_store(memo_uri)
+
+    # any recursively-called functions that use metadata will retain the original invoker.
+
+    try:
+        stop_lock = lock.launch_daemon_lock_maintainer(
+            lock.remote_lock_maintain(
+                fs.join(memo_uri, "lock"), expected_writer_id=invocation_metadata.invoker_uuid
+            )
+        )
+    except lock.CannotMaintainLock as e:
+        logger.info(f"Cannot maintain lock: {e}. Continuing without the lock.")
+        stop_lock = lambda: None  # noqa: E731
+
+    def _extract_invocation_unique_key(memo_uri: str) -> ty.Tuple[str, str]:
+        parts = fs.split(memo_uri)
+        try:
+            runner_idx = parts.index(mprunner.RUNNER_NAME)
+        except ValueError as ve:
+            raise ValueError(
+                f"Unable to find the runner name {mprunner.RUNNER_NAME} in parts {parts}"
+            ) from ve
+        invocation_parts = parts[runner_idx + 1 :]
+        return fs.join(*invocation_parts[:-1]), invocation_parts[-1]
+
+    scope.enter(uris.ACTIVE_STORAGE_ROOT.set(uris.get_root(memo_uri)))
+
+    try:
+        func, args, kwargs = _unpickle_invocation(memo_uri)
+    except Exception:
+        logger.error(f"Failed to unpickle invocation from {memo_uri} - this is a bug in mops!")
+        raise
+
+    def do_work_return_result() -> object:
+        # ONLY failures in this code should transmit an EXCEPTION
+        # back to the orchestrator side.
+        return pipeline_id_mask(invocation_metadata.pipeline_id)(func)(*args, **kwargs)
+
+    route_return_value_or_exception(
+        _ResultExcWithMetadataChannel(
+            fs,
+            _pickle.Dumper(
+                ByIdSerializer(ByIdRegistry()),
+                CoordinatingPathSerializer(sha256_b64.Sha256B64PathStream(), Once()),
+                _pickle.SourceResultPickler(),
+            ),
+            memo_uri,
+            invocation_metadata,
+            started_at,
+        ),
+        ty.cast(ty.Callable[[], T], do_work_return_result),
+        invocation_metadata.pipeline_id,
+        _extract_invocation_unique_key(memo_uri),
+    )
+    stop_lock()  # not critical since we don't _own_ the lock, but keeps things cleaner

thds/mops/pure/pickling/sha256_b64.py

@@ -0,0 +1,71 @@
+"""Context-local, content-aware ser/de from/to a known URI prefix.
+
+Basically, we take some generic pickle utilities and stitch them together into something
+that efficiently serializes object graphs to a combination of locations at some URI prefix,
+such that they are self-deserializing (via CallableUnpickler) on the other side.
+"""
+
+import hashlib
+import io
+import pickle
+import typing as ty
+from pathlib import Path
+
+from thds.core import hashing, log
+
+from ..core.content_addressed import storage_content_addressed, wordybin_content_addressed
+from ..core.serialize_paths import Downloader
+from ..core.uris import active_storage_root, lookup_blob_store
+from .pickles import UnpicklePathFromUri, UnpickleSimplePickleFromUri
+
+logger = log.getLogger(__name__)
+T = ty.TypeVar("T")
+
+
+class Sha256B64PathStream:
+    def local_to_remote(self, path: Path, sha256: str) -> None:
+        """Return fully qualified remote information after put."""
+        # lazily fetches the active storage root.
+        full_remote_sha256 = storage_content_addressed(sha256, "sha256")
+        lookup_blob_store(full_remote_sha256).putfile(path, full_remote_sha256)
+
+    def get_downloader(self, remote_sha256: str) -> Downloader:
+        return UnpicklePathFromUri(storage_content_addressed(remote_sha256, "sha256"))  # type: ignore # NamedTuple silliness
+
+
+def _pickle_obj_and_upload_to_content_addressed_path(
+    obj: object, debug_name: str = ""
+) -> UnpickleSimplePickleFromUri:
+    # active_storage_root is lazily fetched because we may want to register the pickler
+    # somewhere before settling on the final destination of objects pickled.
+    storage_root = active_storage_root()
+    with io.BytesIO() as bio:
+        pickle.dump(obj, bio)
+        bio.seek(0)
+        fs = lookup_blob_store(storage_root)
+        bytes_uri, debug_uri = wordybin_content_addressed(
+            hashing.Hash("sha256", hashing.hash_using(bio, hashlib.sha256()).digest()),
+            storage_root,
+            debug_name=f"objname_{debug_name}" if debug_name else "",
+        )
+        fs.putbytes(bytes_uri, bio, type_hint="application/octet-stream")
+        if debug_uri:
+            # this name is purely for debugging and affects no part of the runtime.
+            fs.putbytes(debug_uri, "goodbeef".encode(), type_hint="text/plain")
+
+    return UnpickleSimplePickleFromUri(bytes_uri)
+
+
+class Sha256B64Pickler:
+    """A type of CallbackPickler, intended for picklable objects that should be serialized
+    as pure bytes and stored at a content-addressed URI. Only used (currently) by the
+    ById/shared object serializer, most likely for something like a large dataframe.
+
+    Name exists solely for debugging purposes.
+    """
+
+    def __init__(self, name: str = ""):
+        self.name = name
+
+    def __call__(self, obj: ty.Any) -> UnpickleSimplePickleFromUri:
+        return _pickle_obj_and_upload_to_content_addressed_path(obj, self.name)

thds/mops/pure/runner/local.py

@@ -0,0 +1,239 @@
+"""Joins pickle functionality and Blob Store functionality to run functions remotely.
+"""
+
+import threading
+import time
+import typing as ty
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+from thds.core import config, log, scope
+
+from ..._utils.colorize import colorized, make_colorized_out
+from ...config import max_concurrent_network_ops
+from ..core import deferred_work, lock, memo, metadata, pipeline_id_mask, uris
+from ..core.partial import unwrap_partial
+from ..core.types import Args, Kwargs, NoResultAfterShimSuccess, T
+from ..tools.summarize import run_summary
+from . import strings, types
+
+MAINTAIN_LOCKS = config.item("thds.mops.pure.local.maintain_locks", default=True, parse=config.tobool)
+
+# these two semaphores allow us to prioritize getting meaningful units
+# of progress _complete_, rather than issuing many instructions to the
+# underlying client and allowing it to randomly order the operations
+# such that it takes longer to get a full unit of work complete.
+_BEFORE_INVOCATION_SEMAPHORE = threading.BoundedSemaphore(int(max_concurrent_network_ops()))
+# _OUT prioritizes uploading a single invocation and its dependencies so the Shim can start running.
+_AFTER_INVOCATION_SEMAPHORE = threading.BoundedSemaphore(int(max_concurrent_network_ops()))
+# _IN prioritizes retrieving the result of a Shim that has completed.
+
+_DarkBlue = colorized(fg="white", bg="#00008b")
+_GreenYellow = colorized(fg="black", bg="#adff2f")
+_Purple = colorized(fg="white", bg="#800080")
+logger = log.getLogger(__name__)
+_LogKnownResult = make_colorized_out(_DarkBlue, out=logger.info, fmt_str=" {} ")
+_LogNewInvocation = make_colorized_out(_GreenYellow, out=logger.info, fmt_str=" {} ")
+_LogAwaitedResult = make_colorized_out(_Purple, out=logger.info, fmt_str=" {} ")
+
+
+def invoke_via_shim_or_return_memoized(  # noqa: C901
+    serialize_args_kwargs: types.SerializeArgsKwargs,
+    serialize_invocation: types.SerializeInvocation,
+    shim_builder: types.ShimBuilder,
+    get_meta_and_result: types.GetMetaAndResult,
+    run_directory: ty.Optional[Path] = None,
+) -> ty.Callable[[bool, str, ty.Callable[..., T], Args, Kwargs], T]:
+    @scope.bound
+    def create_invocation__check_result__wait_shim(
+        rerun_exceptions: bool,
+        function_memospace: str,
+        # by allowing the caller to set the function memospace, we allow 'redirects' to look up an old result by name.
+        # while still guaranteeing that the function arguments were the same.
+        func: ty.Callable[..., T],
+        args_: Args,
+        kwargs_: Kwargs,
+    ) -> T:
+        """This is the generic local runner. Its core abstractions are:
+
+        - serializers of some sort (for the function and its arguments)
+        - a runtime shim of some sort (can start a Python process somewhere else)
+        - a result and metadata deserializer
+        - URIs that are supported by a registered BlobStore implementation.
+
+        It uses a mops-internal locking mechanism to prevent concurrent invocations for the same function+args.
+        """
+        invoked_at = datetime.now(tz=timezone.utc)
+        # capture immediately, because many things may delay actual start.
+        storage_root = uris.get_root(function_memospace)
+        scope.enter(uris.ACTIVE_STORAGE_ROOT.set(storage_root))
+        fs = uris.lookup_blob_store(function_memospace)
+        val_or_res = "value" if rerun_exceptions else "result"
+
+        # we need to unwrap any partial object and combine its wrapped
+        # args, kwargs with the provided args, kwargs, otherwise the
+        # args and kwargs will not get properly considered in the memoization key.
+        func, args, kwargs = unwrap_partial(func, args_, kwargs_)
+        pipeline_id = scope.enter(pipeline_id_mask.including_function_docstr(func))
+        # TODO pipeline_id should probably be passed in explicitly
+
+        scope.enter(deferred_work.open_context())  # optimize Source objects during serialization
+
+        args_kwargs_bytes = serialize_args_kwargs(storage_root, func, args, kwargs)
+        memo_uri = fs.join(function_memospace, memo.args_kwargs_content_address(args_kwargs_bytes))
+
+        # Define some important and reusable 'chunks of work'
+
+        class ResultAndInvocationType(ty.NamedTuple):
+            value_or_error: ty.Union[memo.results.Success, memo.results.Error]
+            invoc_type: run_summary.InvocationType
+
+        def check_result(
+            invoc_type: run_summary.InvocationType,
+        ) -> ty.Union[ResultAndInvocationType, None]:
+            result = memo.results.check_if_result_exists(
+                memo_uri, rerun_excs=rerun_exceptions, before_raise=debug_required_result_failure
+            )
+            if not result:
+                return None
+
+            _LogKnownResult(
+                f"{invoc_type} {val_or_res} for {memo_uri} already exists and is being returned without invocation!"
+            )
+            return ResultAndInvocationType(result, invoc_type)
+
+        def unwrap_value_or_error(result_and_itype: ResultAndInvocationType) -> T:
+            result = result_and_itype.value_or_error
+            metadata = None
+            value_t = None
+            try:
+                if isinstance(result, memo.results.Success):
+                    metadata, value_t = get_meta_and_result("value", result.value_uri)
+                    return ty.cast(T, value_t)
+                else:
+                    assert isinstance(result, memo.results.Error), "Must be Error or Success"
+                    metadata, exc = get_meta_and_result("EXCEPTION", result.exception_uri)
+                    raise exc
+            finally:
+                run_summary.log_function_execution(
+                    *(run_directory, func, memo_uri, result_and_itype.invoc_type),
+                    metadata=metadata,
+                    runner_prefix=function_memospace.split(pipeline_id)[0],
+                    was_error=not isinstance(result, memo.results.Success),
+                    return_value=value_t,
+                )
+
+        def acquire_lock() -> ty.Optional[lock.LockAcquired]:
+            return lock.acquire(fs.join(memo_uri, "lock"), expire=timedelta(seconds=88))
+
+        def upload_invocation_and_deps() -> None:
+            # we're just about to transfer to a remote context,
+            # so it's time to perform any deferred work
+            deferred_work.perform_all()
+
+            fs.putbytes(
+                fs.join(memo_uri, strings.INVOCATION),
+                serialize_invocation(storage_root, func, args_kwargs_bytes),
+                type_hint="application/mops-invocation",
+            )
+
+        def debug_required_result_failure() -> None:
+            # This is entirely for the purpose of making debugging easier. It serves no internal functional purpose.
+            #
+            # first, upload the invocation as an accessible marker of what was expected to exist.
+            upload_invocation_and_deps()
+            # then use mops-inspect programmatically to print the IRE in the same format as usual.
+            from thds.mops.pure.tools.inspect import inspect_and_log
+
+            inspect_and_log(memo_uri)
+
+        # the network ops being grouped by _BEFORE_INVOCATION include one or more
+        # download attempts (consider possible Paths) plus
+        # one or more uploads (embedded Paths & Sources/refs, and then invocation).
+        with _BEFORE_INVOCATION_SEMAPHORE:
+            # now actually execute the chunks of work that are required...
+
+            # it's possible that our result may already exist from a previous run of this pipeline id.
+            # we can short-circuit the entire process by looking for that result and returning it immediately.
+            result = check_result("memoized")
+            if result:
+                return unwrap_value_or_error(result)
+
+            lock_owned = acquire_lock()
+            # if no result exists, the vastly most common outcome here will be acquiring
+            # the lock on the first try.  this will lead to breaking out of
+            # the LOCK LOOP directly below and going on to the shim invocation.
+            # still, we release the semaphore b/c we can't sleep while holding a lock.
+
+        # LOCK LOOP: entering this loop (where we attempt to acquire the lock) is the common non-memoized case
+        while not result:
+            if lock_owned:
+                if MAINTAIN_LOCKS():
+                    release_lock = lock.launch_daemon_lock_maintainer(lock_owned)
+                else:
+                    release_lock = lock_owned.release
+                break  # we own the invocation - invoke the shim ourselves (below)
+
+            # getting to this point ONLY happens if we failed to acquire the lock, which
+            # is not expected to be the usual situation. We log a differently-colored
+            # message here to make that clear to users.
+            _LogAwaitedResult(
+                f"{val_or_res} for {memo_uri} does not exist, but the lock is owned by another process."
+            )
+            time.sleep(22)
+
+            with _BEFORE_INVOCATION_SEMAPHORE:
+                result = check_result("awaited")
+                if result:
+                    _LogAwaitedResult(
+                        f"{val_or_res} for {memo_uri} was found after waiting for the lock."
+                    )
+                    return unwrap_value_or_error(result)
+
+                lock_owned = acquire_lock()  # still inside the semaphore, as it's a network op
+
+        assert release_lock is not None
+        assert lock_owned is not None
+        # if/when we acquire the lock, we move forever into 'run this ourselves mode'.
+        # If something about our invocation fails,
+        # we fail just as we would have previously, without any attempt to go
+        # 'back' to waiting for someone else to compute the result.
+
+        try:
+            with _BEFORE_INVOCATION_SEMAPHORE:
+                _LogNewInvocation(f"Invoking {memo_uri}")
+                upload_invocation_and_deps()
+
+            # can't hold the semaphore while we block on the shim, though.
+            shim_ex = None
+            shim = shim_builder(func, args_, kwargs_)
+            shim(  # ACTUAL INVOCATION (handoff to remote shim) HAPPENS HERE
+                (
+                    memo_uri,
+                    *metadata.format_invocation_cli_args(
+                        metadata.InvocationMetadata.new(pipeline_id, invoked_at, lock_owned.writer_id)
+                    ),
+                )
+            )
+        except Exception as ex:
+            # network or similar errors are very common and hard to completely eliminate.
+            # We know that if a result (or error) exists, then the network failure is
+            # not important, because results in blob storage are atomically populated (either fully there or not)
+            logger.exception("Error awaiting shim. Optimistically checking for result.")
+            shim_ex = ex
+
+        finally:
+            release_lock()
+
+        # the network ops being grouped by _AFTER_INVOCATION include one or more downloads.
+        with _AFTER_INVOCATION_SEMAPHORE:
+            value_or_error = memo.results.check_if_result_exists(memo_uri)
+            if not value_or_error:
+                if shim_ex:
+                    raise shim_ex  # re-raise the underlying exception rather than making up our own.
+                raise NoResultAfterShimSuccess(
+                    f"The shim for {memo_uri} exited cleanly, but no result or exception was found."
+                )
+            return unwrap_value_or_error(ResultAndInvocationType(value_or_error, "invoked"))
+
+    return create_invocation__check_result__wait_shim

thds/mops/pure/runner/shim_builder.py

@@ -0,0 +1,25 @@
+import inspect
+import typing as ty
+
+from ..core.types import Args, F, Kwargs
+from .types import Shim, ShimBuilder
+
+
+class _static_shim_builder:
+    def __init__(self, shim: Shim) -> None:
+        self.shim = shim
+
+    def __call__(self, _f: F, _args: Args, _kwargs: Kwargs) -> Shim:
+        return self.shim
+
+    def __repr__(self) -> str:
+        return f"<static_shim_builder for {self.shim}>"
+
+
+def make_builder(shim: ty.Union[Shim, ShimBuilder]) -> ShimBuilder:
+    """If you have a Shim and you want to make it into the simplest possible ShimBuilder."""
+
+    if len(inspect.signature(shim).parameters) == 3:
+        return ty.cast(ShimBuilder, shim)
+
+    return _static_shim_builder(ty.cast(Shim, shim))

thds/mops/pure/runner/simple_shims.py

@@ -0,0 +1,21 @@
+import subprocess
+from typing import Sequence
+
+from thds.core import log
+
+from ..core.entry.runner_registry import run_named_entry_handler
+
+logger = log.getLogger(__name__)
+
+
+def samethread_shim(shim_args: Sequence[str]) -> None:
+    """Use this inside a memoizing Runner to get the memoization
+    without needing to transfer control to an external process.
+    """
+    logger.debug("Running a mops function locally in the current thread.")
+    run_named_entry_handler(*shim_args)
+
+
+def subprocess_shim(shim_args: Sequence[str]) -> None:
+    logger.debug("Running a mops function locally in a new subprocess.")
+    subprocess.check_call(["python", "-m", "thds.mops.pure.core.entry.main", *shim_args])

thds/mops/pure/runner/strings.py

@@ -0,0 +1 @@
+INVOCATION = "invocation"

thds/mops/pure/runner/types.py

@@ -0,0 +1,28 @@
+import typing as ty
+
+from ..core.metadata import ResultMetadata
+from ..core.types import Args, F, Kwargs
+
+Shim = ty.Callable[[ty.Sequence[str]], ty.Any]
+"""A runner Shim is a way of getting back into a Python process with enough
+context to download the uploaded function and its arguments from the
+location where a runner placed it, and then invoke the function. All
+arguments are strings because it is assumed that this represents some
+kind of command line invocation.
+
+The Shim must be a blocking call, and its result(s) must be available
+immediately after its return.
+"""
+
+
+class ShimBuilder(ty.Protocol):
+    def __call__(self, __f: F, __args: Args, __kwargs: Kwargs) -> Shim:
+        ...  # pragma: no cover
+
+
+StorageRootURI = str
+SerializeArgsKwargs = ty.Callable[[StorageRootURI, F, Args, Kwargs], bytes]
+SerializeInvocation = ty.Callable[[StorageRootURI, F, bytes], bytes]
+# the bytes parameter is the previously-serialized args,kwargs
+GetMetaAndResult = ty.Callable[[str, str], ty.Tuple[ty.Optional[ResultMetadata], ty.Any]]
+# the above should probably not 'hide' the fetch of the bytes, but it is what it is for now.