thds.mops 3.9.20250721225429__py3-none-any.whl → 3.9.20250721231027__py3-none-any.whl

thds/mops/k8s/uncertain_future.py

@@ -0,0 +1,160 @@
+import collections
+import threading
+import time
+import typing as ty
+
+# we use concurrent.futures.Future as an implementation detail, but it's communicated
+# as core.futures.PFuture to give us the flexibility to change the implementation later if needed.
+from concurrent.futures import Future
+from dataclasses import dataclass
+from uuid import uuid4
+
+from typing_extensions import Self
+
+from thds import core
+
+R_0 = ty.TypeVar("R_0", contravariant=True)  # R-naught - the thing that might resolve a Future.
+# a value for this type may never be None.
+
+R = ty.TypeVar("R")
+# the Result type of the Future. These are allowed to be None, since some Futures may
+# resolve but not return a value.
+
+
+class NotYetDone:
+    pass
+
+
+_LastSeenAt = float  # type alias for the last seen time of the Future, in seconds since epoch
+
+
+FutureInterpreter = ty.Callable[[ty.Optional[R_0], _LastSeenAt], ty.Union[R, NotYetDone]]
+# a FutureInterpreter is a function that takes an object R_0 and the time.monotonic() at
+# which it was last seen, and returns either NotYetDone (if the status is still in progress) or
+# the actual Future result of type R, or, if the status is failure,
+# _raises_ an appropriate Exception.
+
+
+class _FutureInterpretationShim(ty.Generic[R_0, R]):
+    def __init__(self, interpreter: FutureInterpreter[R_0, ty.Union[NotYetDone, R]]) -> None:
+        self.future = Future[R]()
+        self._interpreter = interpreter
+        self._id = uuid4().hex  # has an id so it can be hashed and therefore easily found in a set
+
+    def __hash__(self) -> int:
+        return hash(self._id)
+
+    def __call__(self, r_0: ty.Optional[R_0], last_seen_at: float) -> ty.Optional[Self]:
+        """First and foremost - this _must_ be treated as an object that the creator
+        is ultimately responsible for calling on a semi-regular basis. It represents a
+        likely deadlock for the holder of the Future if it is never called.
+
+        Return False if the Future is still in progress and should not be unregistered.
+        Return True if the Future is done and should be unregistered.
+        """
+        try:
+            interpretation = self._interpreter(r_0, last_seen_at)
+            if isinstance(interpretation, NotYetDone):
+                return None  # do nothing and do not unregister - the status is still in progress.
+
+            self.future.set_result(interpretation)
+        except Exception as e:
+            self.future.set_exception(e)
+
+        return self
+
+
+K = ty.TypeVar("K")  # Key type for the UncertainFuturesTracker
+
+
+@dataclass
+class _FuturesState(ty.Generic[R_0]):
+    """Represents a single 'observable' that may have multiple Futures (and therefore interpretations) associated with it."""
+
+    futshims: list[_FutureInterpretationShim[R_0, ty.Any]]
+    last_seen_at: float
+
+
+def official_timer() -> float:
+    # we don't need any particular meaning to the time.
+    return time.monotonic()
+
+
+class UncertainFuturesTracker(ty.Generic[K, R_0]):
+    """This class represents a kind of Future where we cannot be guaranteed that we will ever see
+    any further information about it, because we do not control the source of the data.
+
+    A good example would be a Kubernetes object that we are watching - we may _think_ that a Job will be created,
+    but there are race conditions galore in terms of actually looking for that object.
+
+    However, if we _do_ see it at a some point, then we can interpret future 'missingness'
+    as a tentative success.
+
+    The danger with this uncertainty is that Futures represent implicit deadlocks - if we
+    never resolve the Future, then a caller may be waiting for it forever. Therefore, we
+    ask the original requestor of the Future to specify how long they are willing to wait
+    to get a result, after which point we will resolve the Future as an exception.
+    """
+
+    def __init__(self, allowed_stale_seconds: float) -> None:
+        self._keyed_futures_state = collections.OrderedDict[K, _FuturesState[R_0]]()
+        self._lock = threading.Lock()  # i don't trust ordered dict operations to be thread-safe.
+        self._check_stale_seconds = allowed_stale_seconds
+
+    def create(self, key: K, interpreter: FutureInterpreter[R_0, R]) -> core.futures.PFuture[R]:
+        futshim = _FutureInterpretationShim(interpreter)
+        with self._lock:
+            if key not in self._keyed_futures_state:
+                self._keyed_futures_state[key] = _FuturesState(
+                    [futshim],
+                    last_seen_at=official_timer() + self._check_stale_seconds,
+                    # we provide a double margin for objects that we have never seen before.
+                )
+                self._keyed_futures_state.move_to_end(key, last=False)
+                # never seen and therefore should be at the beginning (most stale)
+            else:
+                # maintain our ordered dict so we can handle garbage collection of stale Futures.
+                self._keyed_futures_state[key].futshims.append(futshim)
+
+        return futshim.future
+
+    def update(self, key: ty.Optional[K], r_0: ty.Optional[R_0]) -> None:
+        """Update the keyed Futures based on their interpreters.
+
+        Also check any stale Futures - Futures that have not seen an update (via their key) in a while.
+
+        If `key` is None, we will update all Futures that have been created so far.
+        """
+
+        def check_resolution(fut_state: _FuturesState[R_0], inner_r_0: ty.Optional[R_0]) -> None:
+            for future_shim_that_is_done in core.parallel.yield_results(
+                [
+                    core.thunks.thunking(futshim)(inner_r_0, fut_state.last_seen_at)
+                    for futshim in fut_state.futshims
+                ],
+                progress_logger=core.log.getLogger(__name__).debug,
+                named="UncertainFuturesTracker.update",
+            ):
+                if future_shim_that_is_done is not None:
+                    # the Future is done, so we can remove it from the list of Futures.
+                    fut_state.futshims.remove(future_shim_that_is_done)
+
+        if key is not None:
+            with self._lock:
+                if key not in self._keyed_futures_state:
+                    self._keyed_futures_state[key] = _FuturesState(list(), last_seen_at=official_timer())
+                else:
+                    # maintain our ordered dict so we can handle garbage collection of stale Futures.
+                    self._keyed_futures_state.move_to_end(key)
+                    self._keyed_futures_state[key].last_seen_at = official_timer()
+
+            fut_state = self._keyed_futures_state[key]
+            check_resolution(fut_state, r_0)
+
+        # 'garbage collect' any Futures that haven't been updated in a while.
+        for futs_state in self._keyed_futures_state.values():
+            if futs_state.last_seen_at + self._check_stale_seconds < official_timer():
+                check_resolution(futs_state, None)
+            else:  # these are ordered, so once we see one that's not stale, we can stop checking.
+                # this prevents us from having to do O(N) checks for every update.
+                break

thds/mops/k8s/watch.py

@@ -12,17 +12,20 @@ import urllib3
 from kubernetes import client
 from kubernetes import watch as k8s_watch
 
-from thds.core import scope
+from thds.core import futures, scope
 from thds.core.log import getLogger, logger_context
 from thds.termtool.colorize import colorized
 
 from . import config
 from .auth import load_config
 from .too_old_resource_version import parse_too_old_resource_version
+from .uncertain_future import FutureInterpreter, UncertainFuturesTracker
 
 logger = getLogger(__name__)
 
 T = ty.TypeVar("T")
+K = ty.TypeVar("K")
+R = ty.TypeVar("R")
 
 
 class V1List(ty.Protocol[T]):
@@ -115,10 +118,6 @@ def callback_events(
             break
 
 
-def _make_name(namespace: str, name: str) -> str:
-    return f"{namespace}/{name}"
-
-
 def _default_get_name(obj: ty.Any) -> str:
     return obj.metadata.name
 
@@ -148,9 +147,15 @@ class OneShotLimiter:
             self._names.add(name)
 
 
+def _watch_timer() -> float:
+    # in this context, monotonicity (actual timing) is most useful because we don't need sentinels.
+    return time.monotonic()
+
+
 def is_stale(api_last_update_time: float, obj_last_seen_time: float) -> bool:
-    now = time.monotonic()
+    now = _watch_timer()
     allowed_stale_seconds = config.k8s_watch_object_stale_seconds()
+    # about 5 minutes by default as of 2025-07-15.
     if (time_since_api_update := now - api_last_update_time) > allowed_stale_seconds:  # noqa: F841
         # we haven't heard anything from the API in a while; probably
         # the API is down. Ignore object staleness to avoid false positives.
@@ -223,93 +228,146 @@ def watch_forever(
             break
 
 
-class WatchingObjectSource(ty.Generic[T]):
-    """Efficiently 'get' objects by reliably watching for changes to all such objects in a given namespace.
-
-    This is network-efficient for observing many different objects,
-    but not memory efficient if you really only need to fetch details
-    for a few objects.
+class _SeenObjectContainer(ty.Generic[K, T]):
+    """Splits some of the logic for 'get' out of WatchingObjectSource
+    so that we can have it be a simpler container for both this and the UncertainFuturesTracker.
     """
 
     def __init__(
         self,
-        get_list_method: GetListMethod[T],
-        get_name: ty.Callable[[T], str] = ty.cast(  # noqa: B008
-            ty.Callable[[T], str], _default_get_name
-        ),
-        backup_fetch: ty.Optional[ty.Callable[[str, str], T]] = None,
-        typename: str = "object",
-        starting: ty.Callable[[str], str] = STARTING,
+        backup_fetch: ty.Optional[ty.Callable[[K], ty.Optional[T]]] = None,
     ) -> None:
-        self.get_list_method = get_list_method
-        self.get_name = get_name
-        self.backup_fetch = backup_fetch
-        self.typename = typename
-        self._objs_by_name: ty.Dict[str, T] = dict()
+        self._objs: ty.Dict[K, T] = dict()
         # ^ is a possibly big/expensive local cache of the most recent
         # state for all of the event type in the namespace.  Don't use
         # this class if you can't afford the memory overhead of
         # observing everything in your namespace and keeping the last
         # known copy of everything forever.
-        self._last_seen_time_by_name: ty.Dict[str, float] = dict()
+        self._last_seen_times: ty.Dict[K, float] = dict()
         self._last_api_update_time = 0.0
-        self._limiter = OneShotLimiter()
-
-    def _start_thread(self, namespace: str) -> None:
-        create_watch_thread(
-            self.get_list_method, self._add_object, namespace, typename=self.typename
-        ).start()
-
-    def _add_object(self, namespace: str, obj: T, _event_type: EventType) -> None:
-        """This is where we receive updates from the k8s API."""
-        self._last_api_update_time = time.monotonic()
-
-        if not obj:
-            logger.warning(f"Received null/empty {self.typename}")
-            return
+        self.backup_fetch = backup_fetch
 
-        name = _make_name(namespace, self.get_name(obj))
-        logger.debug(f"{self.typename} {name} updated")
-        self._last_seen_time_by_name[name] = time.monotonic()
-        self._objs_by_name[name] = obj
+    def set_object(self, key: K, obj: T) -> None:
+        """Set an object in the cache, updating the last seen time."""
+        now = _watch_timer()
+        self._last_api_update_time = now
+        self._last_seen_times[key] = now
+        self._objs[key] = obj
 
-    def _is_stale(self, name: str) -> bool:
-        return is_stale(self._last_api_update_time, self._last_seen_time_by_name.get(name) or 0)
-
-    @scope.bound
-    def get(self, obj_name: str, namespace: str = "") -> ty.Optional[T]:
-        namespace = namespace or config.k8s_namespace()
-        name = _make_name(namespace, obj_name)
-        scope.enter(logger_context(name=obj_name, namespace=namespace))
+    def _is_stale(self, key: K) -> bool:
+        return is_stale(self._last_api_update_time, self._last_seen_times.get(key) or 0)
 
+    def get(self, key: K) -> ty.Optional[T]:
         # first try is looking in our local cache
-        if (obj := self._objs_by_name.get(name)) and not self._is_stale(name):
+        if (obj := self._objs.get(key)) and not self._is_stale(key):
             return obj
 
         # second try is making sure the namespace watcher is running, sleeping, and then looking in the cache again.
         # This is much more efficient than a manual fetch.
-        self._limiter(namespace, self._start_thread)
         time.sleep(config.k8s_monitor_delay())
-        if (obj := self._objs_by_name.get(name)) and not self._is_stale(name):
+        if (obj := self._objs.get(key)) and not self._is_stale(key):
             return obj
 
         # if that doesn't work, try a manual fetch.
         if self.backup_fetch:
-            logger.warning(f"Manually fetching {self.typename}...")
+            logger.warning(f"Manually fetching {key}...")
             # doing a lot of manual fetches may indicate that the k8s API is having trouble keeping up...
             try:
-                if obj := self.backup_fetch(namespace, obj_name):
-                    self._add_object(namespace, obj, "FETCH")  # updates last seen, too
+                if obj := self.backup_fetch(key):
+                    self.set_object(key, obj)  # updates last seen, too
                     return obj
 
             except Exception:
-                logger.exception(f"Unexpected error during manual fetch of {self.typename}.")
+                logger.exception(f"Unexpected error during manual fetch of {key}.")
 
-        if self._is_stale(name):
+        if self._is_stale(key):
             logger.warning(
-                f"Could not refresh {name}, and our record of it is stale - dropping stale object!"
+                f"Could not refresh {key}, and our record of it is stale - dropping stale object!"
             )
-            self._objs_by_name.pop(name, None)
-            self._last_seen_time_by_name.pop(name, None)
+            self._objs.pop(key, None)
+            self._last_seen_times.pop(key, None)
 
         return None
+
+
+class WatchingObjectSource(ty.Generic[T]):
+    """Efficiently 'get' objects by launching a single thread to
+    watch for changes to all such objects in a given namespace.
+
+    Also provide a way to create a future that will be resolved according to the logic
+    provided by the caller whenever an object is updated, or if the object has not been
+    updated in a while.
+
+    Importantly, the Futures are only prevented from deadlocking (never awakening their
+    condition variable) by the fact that we very occasionally will go through the list
+    of seen objects and raise Exceptions for objects that have not been updated in a while.
+    This is vaguely akin to garbage collection, in that it will occasionally
+    cause a 'pause' in the watcher thread as it tries to collect stale objects.
+
+    This is network-efficient for observing many different objects,
+    but not memory efficient if you really only need to fetch details
+    for a few objects, because we retain the last known state for every observed object indefinitely.
+    """
+
+    def __init__(
+        self,
+        get_list_method: GetListMethod[T],
+        get_name: ty.Callable[[T], str] = ty.cast(  # noqa: B008
+            ty.Callable[[T], str], _default_get_name
+        ),
+        backup_fetch: ty.Optional[ty.Callable[[str, str], ty.Optional[T]]] = None,
+        typename: str = "object",
+    ) -> None:
+        self.get_list_method = get_list_method
+        self.get_name = get_name
+        self.typename = typename
+        self._limiter = OneShotLimiter()
+        self._uncertain_futures = UncertainFuturesTracker[tuple[str, str], T](
+            config.k8s_watch_object_stale_seconds()
+        )
+        self._seen_objects = _SeenObjectContainer[tuple[str, str], T](
+            lambda namespace_and_name: backup_fetch(*namespace_and_name) if backup_fetch else None
+        )
+
+    def _add_object(self, namespace: str, obj: T, _event_type: EventType) -> None:
+        """This is where we receive updates from the k8s API."""
+        if not obj:
+            logger.warning(f"Received null/empty {self.typename}")
+            return
+
+        key = (namespace, self.get_name(obj))
+        self._seen_objects.set_object(key, obj)
+        self._uncertain_futures.update(key, obj)
+        logger.debug("%s %s updated", self.typename, key)
+
+    def _start_namespace_watcher_thread(self, namespace: str) -> None:
+        create_watch_thread(
+            self.get_list_method, self._add_object, namespace, typename=self.typename
+        ).start()
+
+    @scope.bound
+    def get(self, obj_name: str, namespace: str = "") -> ty.Optional[T]:
+        """May block for a little while if a manual fetch is required."""
+        namespace = namespace or config.k8s_namespace()
+        scope.enter(logger_context(name=obj_name, namespace=namespace))
+        self._limiter(namespace, self._start_namespace_watcher_thread)
+        return self._seen_objects.get((namespace, obj_name))
+
+    def create_future(
+        self,
+        interpreter: FutureInterpreter[T, R],
+        obj_name: str,
+        *,
+        namespace: str = "",
+    ) -> futures.PFuture[R]:
+        """Create a future that will be resolved when the object is available according to
+        the interpreter.
+
+        The FutureInterpreter must:
+         - raise an exception if it wishes the future to raise.
+         - return a Done with the result if it wishes the future to resolve successfully.
+          -return None if the status is still in progress.
+        """
+        namespace = namespace or config.k8s_namespace()
+        self._limiter(namespace, self._start_namespace_watcher_thread)
+        return self._uncertain_futures.create((namespace, obj_name), interpreter)

thds/mops/pure/__init__.py

@@ -7,6 +7,7 @@
 from . import adls  # noqa
 from ._magic.api import magic  # noqa
 from .core.entry import register_entry_handler
+from .core.lock.maintain import no_maintain as no_maintain_locks  # noqa: F401
 from .core.memo import results  # noqa
 from .core.memo.function_memospace import (  # noqa
     add_pipeline_memospace_handlers,
@@ -21,7 +22,7 @@ from .core.use_runner import use_runner  # noqa
 from .pickling.memoize_only import memoize_in  # noqa
 from .pickling.mprunner import MemoizingPicklingRunner  # noqa
 from .runner.simple_shims import samethread_shim, subprocess_shim  # noqa
-from .runner.types import Shim, ShimBuilder  # noqa
+from .runner.types import FutureShim, Shim, ShimBuilder  # noqa
 
 
 def _register_things() -> None:

thds/mops/pure/_magic/sauce.py

@@ -6,7 +6,7 @@ import typing as ty
 
 from typing_extensions import ParamSpec
 
-from thds.core import stack_context
+from thds.core import futures, stack_context
 from thds.mops._utils import config_tree
 
 from ..core import file_blob_store, pipeline_id, pipeline_id_mask, uris
@@ -109,7 +109,7 @@ class Magic(ty.Generic[P, R]):
     def _is_off(self) -> bool:
         return self._shim_builder_or_off is None
 
-    def _shimbuilder(self, f: ty.Callable[P, R], args: P.args, kwargs: P.kwargs) -> Shim:
+    def _shimbuilder(self, f: ty.Callable[P, R], args: P.args, kwargs: P.kwargs) -> Shim:  # type: ignore[valid-type]
         # this can be set using a stack-local context, or set globally as specifically
         # or generally as the user needs. We prefer stack local over everything else.
         sb = self._shim_builder_or_off
@@ -123,8 +123,16 @@ class Magic(ty.Generic[P, R]):
     def _pipeline_id(self) -> str:
         return self.config.pipeline_id.getv(self._func_config_path)
 
+    def submit(self, *args: P.args, **kwargs: P.kwargs) -> futures.PFuture[R]:
+        """A futures-based interface that doesn't block on the result of the wrapped
+        function call, but returns a PFuture once either a result has been found or a a
+        new invocation has been started.
+        """
+        with pipeline_id.set_pipeline_id_for_stack(self._pipeline_id):
+            return self.runner.submit(self.__wrapped__, *args, **kwargs)
+
     def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R:
-        """This is the wrapped function."""
+        """This is the wrapped function - call this as though it were the function itself."""
         with pipeline_id.set_pipeline_id_for_stack(self._pipeline_id):
             return self._func(*args, **kwargs)
 

thds/mops/pure/_magic/shims.py

@@ -4,14 +4,14 @@ from thds import core
 
 from ..runner.shim_builder import make_builder
 from ..runner.simple_shims import samethread_shim, subprocess_shim
-from ..runner.types import Shim, ShimBuilder
+from ..runner.types import FutureShim, Shim, ShimBuilder
 
 ShimName = ty.Literal[
     "samethread",  # memoization and coordination, but run in the same thread as the caller.
     "subprocess",  # memoization and coordination, but transfer to a subprocess rather than remote.
     "off",  # equivalent to None - disables use of mops.
 ]
-ShimOrBuilder = ty.Union[ShimBuilder, Shim]
+ShimOrBuilder = ty.Union[ShimBuilder, Shim, FutureShim]
 logger = core.log.getLogger(__name__)
 
 

thds/mops/pure/core/deferred_work.py

@@ -32,15 +32,7 @@ def open_context() -> ty.Iterator[None]:
     The idea is that you'd call perform_all() inside your Shim which transfers
     execution to a remote environment, but _not_ call it if you're transferring execution
     to a local environment, as the upload will not be needed.
-
-    This is not re-entrant. If this is called while the dictionary is non-empty, an
-    exception will be raised. This is only because I can think of no reason why anyone
-    would want it to be re-entrant, so it seems better to raise an error. If for some
-    reason re-entrancy were desired, we could just silently pass if the dictionary already
-    has deferred work.
     """
-    existing_work = _DEFERRED_INVOCATION_WORK()
-    assert existing_work is None, f"deferred work context is not re-entrant! {existing_work}"
     with _DEFERRED_INVOCATION_WORK.set(dict()):
         logger.debug("Opening deferred work context")
         yield

thds/mops/pure/core/entry/runner_registry.py

@@ -5,14 +5,6 @@ In practice we only have a single Runner type registered, the MemoizingPicklingR
 
 import typing as ty
 
-from thds.core import stack_context
-
-RUNNER_ENTRY_COUNT = stack_context.StackContext("runner_entry_count", 0)
-
-
-def entry_count() -> int:
-    return RUNNER_ENTRY_COUNT()
-
 
 class EntryHandler(ty.Protocol):
     def __call__(self, *__args: str) -> ty.Any:
@@ -27,5 +19,4 @@ def register_entry_handler(name: str, mh: EntryHandler) -> None:
 
 
 def run_named_entry_handler(name: str, *args: str) -> None:
-    with RUNNER_ENTRY_COUNT.set(RUNNER_ENTRY_COUNT() + 1):
-        ENTRY_HANDLERS[name](*args)
+    ENTRY_HANDLERS[name](*args)

thds/mops/pure/core/lock/__init__.py

@@ -2,6 +2,7 @@ from ._acquire import acquire  # noqa: F401
 from .maintain import (  # noqa: F401
     CannotMaintainLock,
     launch_daemon_lock_maintainer,
+    maintain_to_release,
     remote_lock_maintain,
 )
 from .types import LockAcquired  # noqa: F401

thds/mops/pure/core/lock/_acquire.py

@@ -32,7 +32,7 @@ from thds.core import log
 from . import _funcs
 from .read import get_writer_id, make_read_lockfile
 from .types import LockAcquired, LockContents
-from .write import LockfileWriter, make_lock_contents
+from .write import LockEmitter, LockfileWriter
 
 logger = log.getLogger(__name__)
 
@@ -106,7 +106,7 @@ def acquire(  # noqa: C901
     lockfile_writer = LockfileWriter(
         my_writer_id,
         lock_dir_uri,
-        make_lock_contents(my_writer_id, expire),
+        LockEmitter(my_writer_id, expire),
         expire.total_seconds(),
         debug=debug,
     )

thds/mops/pure/core/lock/maintain.py

@@ -15,12 +15,14 @@ from datetime import datetime, timedelta
 from functools import partial
 from threading import Thread
 
-from thds.core import log
+from thds.core import config, log
 
 from ._funcs import make_lock_uri
 from .read import get_writer_id, make_read_lockfile
 from .types import LockAcquired
-from .write import LockfileWriter, make_lock_contents
+from .write import LockEmitter, LockfileWriter
+
+MAINTAIN_LOCKS = config.item("thds.mops.pure.local.maintain_locks", default=True, parse=config.tobool)
 
 logger = log.getLogger(__name__)
 
@@ -103,7 +105,7 @@ def remote_lock_maintain(lock_dir_uri: str, expected_writer_id: str = "") -> Loc
     lockfile_writer = LockfileWriter(
         current_writer_id,
         lock_dir_uri,
-        make_lock_contents(get_writer_id(lock_contents), timedelta(seconds=expire_s)),
+        LockEmitter(get_writer_id(lock_contents), timedelta(seconds=expire_s)),
         expire_s,
         writer_name="remote",
     )
@@ -148,3 +150,20 @@ def launch_daemon_lock_maintainer(lock_acq: LockAcquired) -> ty.Callable[[], Non
         lock_acq.release()
 
     return stop_maintaining
+
+
+def maintain_to_release(
+    acquired_lock: LockAcquired,
+) -> ty.Callable[[], None]:
+    """Depending on configuration, potentially start maintaining the lock.
+
+    Return a callable that will release the lock when called.
+    """
+    if MAINTAIN_LOCKS():
+        return launch_daemon_lock_maintainer(acquired_lock)
+
+    return acquired_lock.release
+
+
+def no_maintain() -> None:
+    MAINTAIN_LOCKS.set_global(False)

thds/mops/pure/core/lock/write.py

@@ -1,5 +1,6 @@
 import os
 import typing as ty
+from dataclasses import dataclass
 from datetime import datetime, timedelta
 
 from thds.core import hostname, log
@@ -10,38 +11,37 @@ from .types import LockContents
 logger = log.getLogger(__name__)
 
 
-def make_lock_contents(
-    writer_id: str, expire: timedelta
-) -> ty.Callable[[ty.Optional[datetime]], LockContents]:
-    """Impure - Resets written_at to 'right now' to keep the lock 'live'."""
-    write_count = 0
-    first_written_at = ""
+@dataclass
+class LockEmitter:
+    writer_id: str
+    expire: timedelta
 
-    assert (
-        "/" not in writer_id
-    ), f"{writer_id} should not contain a slash - maybe you passed a URI instead?"
+    write_count: int = 0
+    first_written_at: str = ""
 
-    def lock_contents(first_acquired_at: ty.Optional[datetime]) -> LockContents:
-        nonlocal write_count, first_written_at
-        write_count += 1
+    def __post_init__(self) -> None:
+        assert (
+            "/" not in self.writer_id
+        ), f"{self.writer_id} should not contain a slash - maybe you passed a URI instead?"
+
+    def __call__(self, first_acquired_at: ty.Optional[datetime]) -> LockContents:
+        self.write_count += 1
         now = _funcs.utc_now().isoformat()
-        first_written_at = first_written_at or now
+        self.first_written_at = self.first_written_at or now
 
         return {
-            "writer_id": writer_id,
+            "writer_id": self.writer_id,
             "written_at": now,
-            "expire_s": expire.total_seconds(),
+            "expire_s": self.expire.total_seconds(),
             # debug stuff:
-            "write_count": write_count,
+            "write_count": self.write_count,
             "hostname": hostname.friendly(),
             "pid": str(os.getpid()),
-            "first_written_at": first_written_at,
+            "first_written_at": self.first_written_at,
             "first_acquired_at": first_acquired_at.isoformat() if first_acquired_at else "",
             "released_at": "",
         }
 
-    return lock_contents
-
 
 class LockfileWriter:
     """The core purpose of this class is to allow setting of first_acquired_at immediately

thds/mops/pure/core/memo/__init__.py

@@ -1,4 +1,4 @@
-from . import calls, unique_name_for_function  # noqa: F401
+from . import calls, results, unique_name_for_function  # noqa: F401
 from .function_memospace import (  # noqa
     args_kwargs_content_address,
     make_function_memospace,