thds.mops 3.6.20250219172032__py3-none-any.whl

thds/mops/pure/core/lock/maintain.py

@@ -0,0 +1,150 @@
+"""Part of the design of our lock is that a remote process can take over 'maintenance' of
+the lock if (and especially) if the orchestrator process dies.
+
+This allows a killed orchestrator process to be restarted as long as all of its remote
+processes have gotten started working.
+
+The remote process lock maintainers never _acquire_ the lock; they simply read what's in
+it when they get started, and from then on keep the `written_at` timestamp up to date.
+
+"""
+
+import time
+import typing as ty
+from datetime import datetime, timedelta
+from functools import partial
+from threading import Thread
+
+from thds.core import log
+
+from ._funcs import make_lock_uri
+from .read import get_writer_id, make_read_lockfile
+from .types import LockAcquired
+from .write import LockfileWriter, make_lock_contents
+
+logger = log.getLogger(__name__)
+
+
+class _MaintainOnly(ty.NamedTuple):
+    """Matches the LockAcquired interface except that release() will do nothing."""
+
+    maintain: ty.Callable[[], None]
+    expire_s: float
+    release: ty.Callable[[], None]
+
+
+class _MaintainForever(ty.Protocol):
+    def __call__(self) -> None:
+        ...  # pragma: no cover
+
+
+def _maintain_forever(
+    maintain: ty.Callable[[], ty.Any], expire_s: float, should_exit: ty.Callable[[], bool]
+) -> None:
+    while True:
+        # maintain the lock twice as often as necessary, to be safe
+        time.sleep(expire_s / 2)
+        if should_exit():
+            return
+        maintain()
+
+
+class CannotMaintainLock(ValueError):
+    pass  # pragma: no cover
+
+
+class LockWasStolenError(ValueError):
+    pass  # pragma: no cover
+
+
+def remote_lock_maintain(lock_dir_uri: str, expected_writer_id: str = "") -> LockAcquired:
+    """Only for use by remote side - does not _acquire_ the lock,
+    but merely maintains it as unexpired. Does not allow for releasing,
+    as it is not the responsibility of the remote side to release the lock.
+
+    Will raise a CannotMaintainLock exception if the lock does not exist or has no
+    expiration time.
+
+    Will raise a LockWasStolenError if a provided expected_writer_id (which is the
+    writer_id of the lock as provided to the remote side by the original writer) does not
+    match the lock's actual current writer_id - in other words, if some other writer has
+    acquired the lock before the remote side has been able to start running.
+
+    The return value is intended to be launched as the target of a Thread or Process.
+    """
+
+    try:
+        lock_uri = make_lock_uri(lock_dir_uri)
+        read_lockfile = make_read_lockfile(lock_uri)
+        lock_contents = read_lockfile()
+    except Exception:
+        logger.exception(f"Could not read lockfile: {lock_uri}")
+
+    if not lock_contents:
+        raise CannotMaintainLock(f"Lock does not exist: {lock_uri}")
+
+    expire_s = lock_contents["expire_s"]
+    if not expire_s or expire_s < 0:
+        raise CannotMaintainLock(f"Lock is missing an expiry time: {lock_contents}")
+
+    first_acquired_at_s = lock_contents["first_acquired_at"]
+    if not first_acquired_at_s:
+        raise CannotMaintainLock(f"Lock was never acquired: {lock_contents}")
+
+    current_writer_id = lock_contents["writer_id"]
+    if expected_writer_id and expected_writer_id != current_writer_id:
+        raise LockWasStolenError(
+            "Refusing to maintain lock that was created by a different writer:"
+            f" expected `{expected_writer_id}`, got `{current_writer_id}`."
+            "This probably means you just need to kill and restart your orchestrator "
+            " and it will begin awaiting the results of the new owner of the lock."
+        )
+
+    lockfile_writer = LockfileWriter(
+        current_writer_id,
+        lock_dir_uri,
+        make_lock_contents(get_writer_id(lock_contents), timedelta(seconds=expire_s)),
+        expire_s,
+        writer_name="remote",
+    )
+    lockfile_writer.first_acquired_at = datetime.fromisoformat(first_acquired_at_s)
+    # disable releasing from remote
+    lockfile_writer.release = lambda: None  # type: ignore # noqa: E731
+    return lockfile_writer
+
+
+def launch_daemon_lock_maintainer(lock_acq: LockAcquired) -> ty.Callable[[], None]:
+    """Run lock maintenance until the process exits, or until the returned callable gets
+    returned.
+
+    Return a 'release wrapper' that stops maintenance of the lock and releases it.
+
+    A whole thread for this seems expensive, but the simplest alternative is having too
+    many lock maintainers trying to share time slices within some global lock maintainer,
+    and that runs a definite risk of overrunning the expiry time(s) for those locks.
+
+    If we were async all the way down, we could more plausibly make a bunch of async
+    network/filesystem calls here without taking into consideration how long they actually
+    take to execute.
+    """
+    should_exit = False
+
+    def should_stop_maintaining() -> bool:
+        return should_exit
+
+    Thread(
+        target=partial(
+            _maintain_forever,
+            lock_acq.maintain,
+            lock_acq.expire_s,
+            should_stop_maintaining,
+        ),
+        daemon=True,
+    ).start()
+
+    def stop_maintaining() -> None:
+        nonlocal should_exit
+        should_exit = True
+        lock_acq.release()
+
+    return stop_maintaining

thds/mops/pure/core/lock/read.py

@@ -0,0 +1,39 @@
+import io
+import json
+import typing as ty
+
+from thds.core import log
+
+from ..types import DISABLE_CONTROL_CACHE
+from ..uris import lookup_blob_store
+from .types import LockContents
+
+logger = log.getLogger(__name__)
+
+
+def get_writer_id(lock_contents: LockContents) -> str:
+    return lock_contents["writer_id"]
+
+
+def make_read_lockfile(lock_uri: str) -> ty.Callable[[], ty.Optional[LockContents]]:
+    def read_lockfile() -> ty.Optional[LockContents]:
+        with DISABLE_CONTROL_CACHE.set_local(True):
+            blob_store = lookup_blob_store(lock_uri)
+
+        while True:
+            lockfile_bio = io.BytesIO()
+            try:
+                # NO OPTIMIZE: this read must never be optimized in any way.
+                blob_store.readbytesinto(lock_uri, lockfile_bio, type_hint="lock")
+            except Exception as e:
+                if blob_store.is_blob_not_found(e):
+                    return None
+                logger.error(f"Failed on {lock_uri}: {e}")
+                raise
+
+            if lockfile_bio.tell() == 0:  # nothing was written
+                logger.debug("Lockfile %s was empty - retrying read.", lock_uri)
+                continue
+            return json.loads(lockfile_bio.getvalue().decode())
+
+    return read_lockfile

thds/mops/pure/core/lock/types.py

@@ -0,0 +1,37 @@
+import typing as ty
+
+
+class LockContents(ty.TypedDict):
+    """Only writer_id, written_at, and expire are technically required for the algorithm
+    - everything else is debugging info.
+
+    In fact, expire_s would be 'optional' as well (this can be acquirer-only state), but
+    it is advantegous to embed this explicitly, partly so that we can have remote
+    'maintainers' that do not need to have any information other than the lock uri passed
+    to them in order to maintain the lock.
+    """
+
+    writer_id: str
+    written_at: str  # ISO8601 string with timezone in UTC
+    expire_s: float  # seconds after written_at to expire
+
+    # just for debugging
+    hostname: str
+    pid: str
+    write_count: int
+    first_written_at: str
+    first_acquired_at: str
+    released_at: str
+
+
+class LockAcquired(ty.Protocol):
+
+    writer_id: str
+
+    def maintain(self) -> None:
+        ...  # pragma: no cover
+
+    def release(self) -> None:
+        ...  # pragma: no cover
+
+    expire_s: float

thds/mops/pure/core/lock/write.py

@@ -0,0 +1,136 @@
+import os
+import typing as ty
+from datetime import datetime, timedelta
+
+from thds.core import hostname, log
+
+from . import _funcs
+from .types import LockContents
+
+logger = log.getLogger(__name__)
+
+
+def make_lock_contents(
+    writer_id: str, expire: timedelta
+) -> ty.Callable[[ty.Optional[datetime]], LockContents]:
+    """Impure - Resets written_at to 'right now' to keep the lock 'live'."""
+    write_count = 0
+    first_written_at = ""
+
+    assert (
+        "/" not in writer_id
+    ), f"{writer_id} should not contain a slash - maybe you passed a URI instead?"
+
+    def lock_contents(first_acquired_at: ty.Optional[datetime]) -> LockContents:
+        nonlocal write_count, first_written_at
+        write_count += 1
+        now = _funcs.utc_now().isoformat()
+        first_written_at = first_written_at or now
+
+        return {
+            "writer_id": writer_id,
+            "written_at": now,
+            "expire_s": expire.total_seconds(),
+            # debug stuff:
+            "write_count": write_count,
+            "hostname": hostname.friendly(),
+            "pid": str(os.getpid()),
+            "first_written_at": first_written_at,
+            "first_acquired_at": first_acquired_at.isoformat() if first_acquired_at else "",
+            "released_at": "",
+        }
+
+    return lock_contents
+
+
+class LockfileWriter:
+    """The core purpose of this class is to allow setting of first_acquired_at immediately
+    after the first time that it is confirmed that we have acquired the lock.
+
+    Everything else could have been done as a (simpler) closure.
+    """
+
+    def __init__(
+        self,
+        lock_writer_id: str,
+        lock_dir_uri: str,
+        generate_lock: ty.Callable[[ty.Optional[datetime]], LockContents],
+        expire_s: float,
+        *,
+        debug: bool = True,
+        writer_name: str = "",
+    ) -> None:
+        self.writer_id = lock_writer_id
+        self.lock_dir_uri = lock_dir_uri
+        self.blob_store, self.lock_uri = _funcs.store_and_lock_uri(lock_dir_uri)
+        self.generate_lock = generate_lock
+        self.expire_s = expire_s
+        self.debug = debug
+        self.writer_name = writer_name
+        self.first_acquired_at: ty.Optional[datetime] = None
+
+    def mark_acquired(self) -> None:
+        assert not self.first_acquired_at
+        self.first_acquired_at = _funcs.utc_now()
+        logger.debug("Acquired lock %s", self.lock_uri)
+        self.write()  # record the first_acquired_at value for posterity
+
+    def write(self) -> None:
+        lock_contents = self.generate_lock(self.first_acquired_at)
+        if self.writer_name:
+            lock_contents["writer_name"] = self.writer_name  # type: ignore
+        assert "/" not in lock_contents["writer_id"], lock_contents
+        assert self.writer_id == lock_contents["writer_id"], (self.writer_id, lock_contents)
+        lock_bytes = _funcs.json_dumpb(lock_contents)
+        assert lock_bytes
+        # technically, writing these bytes may cause an overwrite of someone else's lock.
+        # the only way we get to 'decide' who acquired the lock is by waiting an
+        # appropriate period of time (agreed upon by all acquirers, and sufficient to be
+        # certain that everyone who tried is going to actually wait long enough to see the
+        # results - and then we see who wrote it last. Whoever wrote it last 'won',
+        # and should continue as though they acquired the lock. Everyone else should 'fail'
+        # to acquire the lock.
+        _funcs.write(self.blob_store, self.lock_uri, lock_bytes)
+        self._maybe_write_debug(lock_contents)
+
+    def maintain(self) -> None:
+        """It is valid to call this method multiple times as necessary once the lock has been acquired."""
+        self.write()
+
+    def release(self) -> None:
+        assert self.first_acquired_at
+        lock_contents = self.generate_lock(self.first_acquired_at)
+        lock_contents["released_at"] = lock_contents["written_at"]
+        lock_contents["written_at"] = ""
+        logger.debug(
+            "Releasing lock %s after %s", self.lock_uri, _funcs.utc_now() - self.first_acquired_at
+        )
+        _funcs.write(self.blob_store, self.lock_uri, _funcs.json_dumpb(lock_contents))
+        self._maybe_write_debug(lock_contents)
+
+    def _maybe_write_debug(self, lock_contents: LockContents) -> None:
+        """Only do this if the lock was actually acquired."""
+        # this debug bit serves to help us understand when clients actually believed
+        # that they had acquired the lock.  Because we only do this after our first
+        # 'successful' write, it will not impose extra latency during the
+        # latency-critical section.
+        if self.debug and self.first_acquired_at:
+            name = (self.writer_name + ";_") if self.writer_name else ""
+            first_written_at = lock_contents["first_written_at"]
+            hostname = lock_contents["hostname"]
+            pid = lock_contents["pid"]
+            acq_uuid = lock_contents["writer_id"]
+            assert "/" not in acq_uuid, lock_contents
+            debug_uri = self.blob_store.join(
+                self.lock_dir_uri,
+                "writers-debug",
+                f"firstwrite={first_written_at};_uuid={acq_uuid};_host={hostname};_pid={pid}{name}.json",
+            )
+            try:
+                self.blob_store.putbytes(
+                    debug_uri,
+                    _funcs.json_dumpb(lock_contents),
+                    type_hint="application/mops-lock-breadcrumb",
+                )
+            except Exception:
+                logger.warning(f"Problem writing debug lock {debug_uri}")

thds/mops/pure/core/memo/__init__.py

@@ -0,0 +1,6 @@
+from .function_memospace import (  # noqa
+    args_kwargs_content_address,
+    make_function_memospace,
+    parse_memo_uri,
+)
+from .unique_name_for_function import extract_function_logic_key_from_docstr  # noqa: F401

thds/mops/pure/core/memo/function_memospace.py

@@ -0,0 +1,267 @@
+"""A big part of what mops offers is automatic memoization.
+
+It's built on the principle that if we need to be able to transfer
+execution from one system/environment to another, then by definition
+your computation must be a pure function, otherwise the result is not
+reliable. And, because it _is_ a pure function, by definition we can
+memoize your calls to it. More than that, we already _have_ memoized
+them, because in order to transfer the invocation to the worker
+environment, and then the worker's results back to your orchestrator,
+we needed to serialize them somewhere, and those serialized invocation
+and result will (in theory) be there the next time we look for them.
+
+In a perfect world with pure functions, this memoization would be
+omnipresent and completely transparent to the user. However, we don't
+live in a perfect world. There are at least two common ways in which
+always-on memoization could lead to incorrect behavior:
+
+1. Your code changes between calls to the same function.
+
+   We can't reliably detect this, because we're not actually able to
+   serialize or otherwise derive a key from the full code,
+   recursively, of your function and everything it
+   references/calls.
+
+   Therefore, we allow you to notify us of these changes in one of
+   several ways, but the most common is by using mops without
+   explicitly setting a `pipeline_id` for your application's run.
+
+   If you don't set a `pipeline_id`, then one will be
+   non-deterministically generated for you at every application start;
+   essentially, you'll get no memoization of any kind, because you
+   haven't confirmed (via pipeline_id) that your code has not
+   changed. But if you do set the same pipeline_id consistently when
+   running your function, you'll be able to take advantage of the
+   memoization that is already occurring under the hood.
+
+2. Your function writes its true results as side effects to some other
+   storage location, and the returned result from the function merely
+   _references_ the true result, which is stored in that external
+   system.
+
+   In other words, your function is not truly pure.
+
+   In this case, the actual source of erroneous behavior would be if
+   the external storage system is mutable. If it is not mutable, or
+   if, by convention, the storage can reliably be treated as
+   representing immutable, persistent data, then aside from network
+   errors or other sources of retryable non-determinism, your
+   application can be expected to reliably reuse memoized results from
+   this technically impure but pure-in-practice function.
+
+   In general, this source of non-determinism is probably the easier
+   to deal with, as it requires only the one convention - namely, that
+   certain ADLS storage accounts/containers should never have new and
+   different data written over top of existing data.
+
+
+The code that follows helps address point #1 above. Code changes are
+endemic to software development and data science, and it cannot be
+expected that memoization will only be used after code is "set in
+stone".
+
+The approach taken here is that it should be possible to run a given
+process, with a known or even an auto-generated pipeline id, and then
+simply record that pipeline id for later, such that a future caller of
+the function can opt into the memoized results of that 'known run'
+simply by calling the function.
+
+The implementation detail is that this will be done out of band -
+instead of modifying the code (either the called code or the call
+site), we will allow this to be 'injected' via configuration, on a
+per-function (rather than per-application, or per-function-call)
+basis.
+
+- per-application is rejected because it's what pipeline_id already
+  does - if you simply want to opt in to an entire 'universe' of
+  memoized results, you can reuse the pipeline_id corresponding to
+  that universe. We're trying to solve for a case where multiple
+  'universes' need to be stitched together in a later re-use of
+  memoized results.  - per-function-call is rejected because there are
+  no currently-anticipated use cases for it - as an implementation
+  detail this would not be particularly hard to achieve, but it also
+  seems likely to be more 'developer overhead' than anybody would
+  really want to use in practice.
+
+The memoization/cache key for `use_runner` (mops) function calls is made up of three parts or levels:
+
+- The top level is the global storage config, including SA, container,
+  and a version-specific base path provided by the `mops` runner.
+  This level is not semantically derived from the function call
+  itself; it's present purely as a technical reality.
+
+  In the configuration and in the code, the configurable part of this
+  is referred to as the storage_root. Once a mops runner adds its own
+  base path, it becomes the runner prefix.
+
+- The middle level is the 'code' memoization, which provides users granular ways of
+  invalidating caches across runs sharing a runner prefix by changing one of:
+---- pipeline_id
+---- name of function being memoized
+---- cache key in docstring for function being memoized
+  to indicate that something about the _code being run_ has changed.
+
+- The bottom level is the 'arguments' memoization,
+  whereby we serialize and then hash the full set of arguments to the function,
+  such that different calls to the same function will memoize differently as expected.
+
+Of the three levels, our per-function memoization config should only need to 'deal' with the top two levels.
+
+- A previous call to the function in question might have used a
+  different storage root than is configured by the application for the
+  default case, so it must be necessary to specify where we want to
+  look for memoized results.
+
+- The pipeline_id used for a known result may be different for various
+  different functions that we intend to call.
+
+- If a codebase has undergone refactoring, such that a function lives
+  in a different module than it previously did, but you wish to reuse
+  memoized results, it should be possible to provide a translation
+  layer for the name itself.
+
+- In rare cases, the (optional) value of a function's
+  function-logic-key (embedded in the docstring) may have changed
+  compared to the version we're able to import, but we may still wish
+  to pick up the result of a different configuration.
+
+Notably, we do _not_ propose to allow configuration of the hashed
+args/kwargs itself, which would amount to a full redirect of the
+function call to a known result. It's not that there might not be some
+use case for this functionality; we simply don't foresee what that
+would be and decline to prematurely implement such functionality.
+
+"""
+
+import hashlib
+import re
+import typing as ty
+
+from thds import humenc
+from thds.core import config
+
+from ..pipeline_id_mask import (
+    extract_from_docstr,
+    get_pipeline_id,
+    get_pipeline_id_mask,
+    pipeline_id_mask,
+)
+from ..uris import lookup_blob_store
+from .unique_name_for_function import make_unique_name_including_docstring_key, parse_unique_name
+
+
+class _PipelineMemospaceHandler(ty.Protocol):
+    def __call__(self, __callable_name: str, __runner_prefix: str) -> ty.Optional[str]:
+        ...
+
+
+_PIPELINE_MEMOSPACE_HANDLERS: ty.List[_PipelineMemospaceHandler] = list()
+
+
+def add_pipeline_memospace_handlers(*handlers: _PipelineMemospaceHandler) -> None:
+    """Add one or more handlers that will be tested in order to determine whether an
+    application wishes to override all or part of the "pipeline memospace" (the runner
+    prefix plus the pipeline id) for a given fully-qualified function name.
+
+    Does _not_ provide access to the invocation-specific `function_id` information; this
+    capability is not offered by mops.
+    """
+    _PIPELINE_MEMOSPACE_HANDLERS.extend(handlers)
+
+
+def matching_mask_pipeline_id(pipeline_id: str, callable_regex: str) -> _PipelineMemospaceHandler:
+    """Set the function memospace to be:
+
+    the current runner prefix
+    + the supplied pipeline_id, OR the set pipeline_id (not the
+      pipeline_id_mask!) if the supplied pipeline_id is empty
+      (thus allowing for this to fall back to an auto-generated pipeline_id if you want to force a run)
+    + the callable name (including docstring key).
+
+    Note this uses re.match, which means your regex must match the _beginning_ of the
+    callable name. If you want fullmatch, write your own. :)
+
+    """
+
+    def _handler(callable_name: str, runner_prefix: str) -> ty.Optional[str]:
+        if re.match(callable_regex, callable_name):
+            return lookup_blob_store(runner_prefix).join(
+                runner_prefix, pipeline_id or get_pipeline_id(), callable_name
+            )
+        return None
+
+    return _handler
+
+
+def _lookup_pipeline_memospace(runner_prefix: str, callable_name: str) -> ty.Optional[str]:
+    """The pipeline memospace is everything up until but not including the hash of the (args, kwargs) tuple."""
+    try:
+        config_memospace = config.config_by_name(f"mops.memo.{callable_name}.memospace")()
+    except KeyError:
+        config_memospace = ""
+    if config_memospace:
+        return config_memospace
+    for handler in _PIPELINE_MEMOSPACE_HANDLERS:
+        pipeline_memospace = handler(callable_name, runner_prefix)
+        if pipeline_memospace:
+            return pipeline_memospace
+    return None
+
+
+def make_function_memospace(runner_prefix: str, f: ty.Callable) -> str:
+    callable_name = make_unique_name_including_docstring_key(f)
+    # always default to the function docstring if no other mask is currently provided.
+    with pipeline_id_mask(extract_from_docstr(f, require=False)):
+        return _lookup_pipeline_memospace(runner_prefix, callable_name) or lookup_blob_store(
+            runner_prefix
+        ).join(
+            runner_prefix,
+            get_pipeline_id_mask(),
+            callable_name,
+        )
+
+
+class MemoUriComponents(ty.NamedTuple):
+    runner_prefix: str
+    pipeline_id: str
+    function_module: str
+    function_name: str
+    function_logic_key: str
+    args_hash: str
+
+
+def parse_memo_uri(
+    memo_uri: str,
+    runner_prefix: str = "",  # the part up to but not including the pipeline_id
+    separator: str = "/",
+    backward_compat_split: str = "mops2-mpf",
+) -> MemoUriComponents:
+    if not runner_prefix:
+        # this is in order to help with backward compatibilty for mops summaries that
+        # didn't store any of this. providing memospace is a more precise way to handle this.
+        if backward_compat_split not in memo_uri:
+            raise ValueError("Cannot determine the components of a memo URI with no memospace")
+        parts = memo_uri.split(backward_compat_split, 1)
+        assert len(parts) > 1, parts
+        runner_prefix = separator.join((parts[0].rstrip(separator), backward_compat_split))
+
+    runner_prefix = runner_prefix.rstrip(separator)
+    rest, args_hash = memo_uri.rsplit(separator, 1)  # args hash is last component
+    rest, full_function_name = rest.rsplit(separator, 1)
+    pipeline_id = rest[len(runner_prefix) :]
+    pipeline_id = pipeline_id.strip(separator)
+
+    function_parts = parse_unique_name(full_function_name)
+
+    return MemoUriComponents(
+        runner_prefix,
+        pipeline_id,
+        function_parts.module,
+        function_parts.name,
+        function_parts.function_logic_key,
+        args_hash,
+    )
+
+
+def args_kwargs_content_address(args_kwargs_bytes: bytes) -> str:
+    return humenc.encode(hashlib.sha256(args_kwargs_bytes).digest())

thds/mops/pure/core/memo/keyfunc.py

@@ -0,0 +1,53 @@
+"""Definitions of basic keyfuncs."""
+import inspect
+import typing as ty
+
+from ..types import Args, Kwargs
+
+
+class Keyfunc(ty.Protocol):
+    """A function which, when called with (c, args, kwargs),
+    returns either the same or a different callable, and the same or
+    different args and kwargs, such that the returned three-tuple is
+    what will get used to construct the full memoization key.
+
+    The args, kwargs returned _must_ be bindable to the parameters of
+    the callable returned.  However, the callable will not be actually
+    invoked, so it is not important that they bind in a semantically
+    meaningful way - if you're just trying to drop certain arguments
+    that can't be pickled, your best bet will be to return a `None`
+    placeholder for those.
+
+    The identity function (lambda c, a, k: c, a k) is equivalent to
+    the unchanged default behavior from MemoizingPicklingRunner.
+    """
+
+    def __call__(
+        self, c: ty.Callable, __args: Args, __kwargs: Kwargs
+    ) -> ty.Tuple[ty.Callable, Args, Kwargs]:
+        ...  # pragma: nocover
+
+
+ArgsOnlyKeyfunc = ty.Callable[..., ty.Tuple[Args, Kwargs]]
+
+
+def args_only(keyfunc: ty.Union[ArgsOnlyKeyfunc, Keyfunc]) -> Keyfunc:
+    def funcpassthrough_keyfunc(
+        c: ty.Callable, args: Args, kwargs: Kwargs
+    ) -> ty.Tuple[ty.Callable, Args, Kwargs]:
+        return c, *keyfunc(*args, **kwargs)  # type: ignore
+
+    return funcpassthrough_keyfunc
+
+
+def autowrap_args_only_keyfunc(keyfunc: ty.Union[ArgsOnlyKeyfunc, Keyfunc]) -> Keyfunc:
+    """This exists only to 'sweeten' the API, so that in most cases a
+    'normal-looking' function can be passed in that does not have
+    access to the `func` parameter and gets Pythonic access to the
+    splatted args and kwargs, rather than a tuple and a dictionary.
+    """
+    keyfunc_params = inspect.signature(keyfunc).parameters
+    is_full_keyfunc = len(keyfunc_params) == 3 and next(iter(keyfunc_params.values())).name == "c"
+    if is_full_keyfunc:
+        return ty.cast(Keyfunc, keyfunc)
+    return args_only(keyfunc)