thds.mops 3.9.20250919153256__py3-none-any.whl → 3.10.20251104012416__py3-none-any.whl

thds/mops/_utils/config_tree.py

@@ -154,6 +154,10 @@ class ConfigTree(ty.Generic[V]):
     def __setitem__(self, key: str, value: V) -> None:
         self.setv(value, pathable=key)
 
+    def __contains__(self, key: str) -> bool:
+        """Only answers the specific question - does this exact key exist in the config?"""
+        return key in self.registry
+
     def load_config(self, config: ty.Mapping[str, ty.Any]) -> None:
         """Loads things with an inner key matching this name into the config.
 

thds/mops/k8s/apply_yaml.py

@@ -1,22 +1,45 @@
+import subprocess
 import tempfile
 
+import kubernetes
 from kubernetes import client, utils
+from packaging import version
 
 
 def format_yaml(yaml_template_str: str, **template_values: str) -> str:
     return yaml_template_str.format(**template_values)
 
 
+def kubectl_apply_file(yaml_path: str) -> None:
+    subprocess.run(["kubectl", "apply", "-f", yaml_path], check=True)
+
+
+def kubectl_apply(yaml_string: str) -> None:
+    with tempfile.NamedTemporaryFile("w", prefix="kubectl-yaml") as f:
+        f.write(yaml_string)
+        f.flush()
+        kubectl_apply_file(f.name)
+
+
+def apply_yaml(yaml_path: str) -> None:
+    if version.parse(kubernetes.__version__) < version.parse("32.0.0"):
+        kubectl_apply_file(yaml_path)  # best effort
+        return
+
+    # NOTE: Prior to 32.0.0, this function doesn't actually server-side apply.
+    # https://github.com/kubernetes-client/python/pull/2252
+    # Hence the check above to use kubectl for older versions.
+    utils.create_from_yaml(client.ApiClient(), yaml_path, apply=True)
+
+
 def create_yaml_template(yaml_str: str, **template_values: str) -> None:
     """Format a YAML template with the given keyword arguments, then apply it to the Kubernetes cluster.
 
     You must already have set up your SDK config.
 
-    NOTE: This function doesn't actually apply, and can't until the next release of the K8S SDK:
-    https://github.com/kubernetes-client/python/pull/2252
     """
     formatted_yaml = format_yaml(yaml_str, **template_values)
     with tempfile.NamedTemporaryFile("w", prefix="kubectl-yaml") as f:
         f.write(formatted_yaml)
         f.flush()
-        utils.create_from_yaml(client.ApiClient(), f.name)
+        apply_yaml(f.name)

thds/mops/k8s/uncertain_future.py

@@ -44,20 +44,20 @@ class _FutureInterpretationShim(ty.Generic[R_0, R]):
     def __hash__(self) -> int:
         return hash(self._id)
 
-    def __call__(self, r_0: ty.Optional[R_0], last_seen_at: float) -> ty.Optional[Self]:
+    def interpret(self, r_0: ty.Optional[R_0], last_seen_at: float) -> ty.Optional[Self]:
         """First and foremost - this _must_ be treated as an object that the creator
         is ultimately responsible for calling on a semi-regular basis. It represents a
         likely deadlock for the holder of the Future if it is never called.
 
-        Return False if the Future is still in progress and should not be unregistered.
-        Return True if the Future is done and should be unregistered.
+        Return None if the Future is still in progress and should not be unregistered.
+        Return self if the Future is done and should be unregistered.
         """
         try:
             interpretation = self._interpreter(r_0, last_seen_at)
             if isinstance(interpretation, NotYetDone):
                 return None  # do nothing and do not unregister - the status is still in progress.
 
-            self.future.set_result(interpretation)
+            self.future.set_result(interpretation)  # resolved successfully!
         except Exception as e:
             self.future.set_exception(e)
 
@@ -71,7 +71,7 @@ K = ty.TypeVar("K")  # Key type for the UncertainFuturesTracker
 class _FuturesState(ty.Generic[R_0]):
     """Represents a single 'observable' that may have multiple Futures (and therefore interpretations) associated with it."""
 
-    futshims: list[_FutureInterpretationShim[R_0, ty.Any]]
+    futshims: set[_FutureInterpretationShim[R_0, ty.Any]]
     last_seen_at: float
 
 
@@ -94,10 +94,16 @@ class UncertainFuturesTracker(ty.Generic[K, R_0]):
     never resolve the Future, then a caller may be waiting for it forever. Therefore, we
     ask the original requestor of the Future to specify how long they are willing to wait
     to get a result, after which point we will resolve the Future as an exception.
+
+    Notably, once we have seen an object, we will not ever remove it from our tracking list.
+    This implies a certain amount of memory growth over time, but it avoids race conditions
+    between producers and consumers of the Futures.
     """
 
     def __init__(self, allowed_stale_seconds: float) -> None:
         self._keyed_futures_state = collections.OrderedDict[K, _FuturesState[R_0]]()
+        # ordered from least-recently-seen to most-recently-seen, so that we can easily garbage collect
+        # potentially stale Futures, which will be at the front of the OrderedDict.
         self._lock = threading.Lock()  # i don't trust ordered dict operations to be thread-safe.
         self._check_stale_seconds = allowed_stale_seconds
 
@@ -105,16 +111,11 @@ class UncertainFuturesTracker(ty.Generic[K, R_0]):
         futshim = _FutureInterpretationShim(interpreter)
         with self._lock:
             if key not in self._keyed_futures_state:
-                self._keyed_futures_state[key] = _FuturesState(
-                    [futshim],
-                    last_seen_at=official_timer() + self._check_stale_seconds,
-                    # we provide a double margin for objects that we have never seen before.
-                )
-                self._keyed_futures_state.move_to_end(key, last=False)
-                # never seen and therefore should be at the beginning (most stale)
+                self._keyed_futures_state[key] = _FuturesState({futshim}, last_seen_at=official_timer())
+                # never seen and therefore should be at the end (least stale)
             else:
                 # maintain our ordered dict so we can handle garbage collection of stale Futures.
-                self._keyed_futures_state[key].futshims.append(futshim)
+                self._keyed_futures_state[key].futshims.add(futshim)
 
         return futshim.future
 
@@ -126,10 +127,10 @@ class UncertainFuturesTracker(ty.Generic[K, R_0]):
         If `key` is None, we will update all Futures that have been created so far.
         """
 
-        def check_resolution(fut_state: _FuturesState[R_0], inner_r_0: ty.Optional[R_0]) -> None:
+        def interpret_event(fut_state: _FuturesState[R_0], inner_r_0: ty.Optional[R_0]) -> None:
             for future_shim_that_is_done in core.parallel.yield_results(
                 [
-                    core.thunks.thunking(futshim)(inner_r_0, fut_state.last_seen_at)
+                    core.thunks.thunking(futshim.interpret)(inner_r_0, fut_state.last_seen_at)
                     for futshim in fut_state.futshims
                 ],
                 progress_logger=core.log.getLogger(__name__).debug,
@@ -137,24 +138,33 @@ class UncertainFuturesTracker(ty.Generic[K, R_0]):
             ):
                 if future_shim_that_is_done is not None:
                     # the Future is done, so we can remove it from the list of Futures.
-                    fut_state.futshims.remove(future_shim_that_is_done)
+                    fut_state.futshims.discard(future_shim_that_is_done)
 
         if key is not None:
             with self._lock:
                 if key not in self._keyed_futures_state:
-                    self._keyed_futures_state[key] = _FuturesState(list(), last_seen_at=official_timer())
+                    self._keyed_futures_state[key] = _FuturesState(set(), last_seen_at=official_timer())
                 else:
                     # maintain our ordered dict so we can handle garbage collection of stale Futures.
                     self._keyed_futures_state.move_to_end(key)
                     self._keyed_futures_state[key].last_seen_at = official_timer()
 
-            fut_state = self._keyed_futures_state[key]
-            check_resolution(fut_state, r_0)
+            if fut_state := self._keyed_futures_state.get(key):
+                interpret_event(fut_state, r_0)
 
+        #
         # 'garbage collect' any Futures that haven't been updated in a while.
-        for futs_state in self._keyed_futures_state.values():
-            if futs_state.last_seen_at + self._check_stale_seconds < official_timer():
-                check_resolution(futs_state, None)
+        #
+        with self._lock:
+            safe_futures = list(self._keyed_futures_state.values())
+            # this avoids holding the lock, but also avoids RuntimeError: OrderedDict mutated during iteration
+        now = official_timer()
+        for futs_state in safe_futures:
+            if now > futs_state.last_seen_at + self._check_stale_seconds:
+                interpret_event(futs_state, None)
+                # None means we have no new information about the object.
+                # the interpreter must decide what to do with that, plus the last seen time.
+
             else:  # these are ordered, so once we see one that's not stale, we can stop checking.
                 # this prevents us from having to do O(N) checks for every update.
                 break

thds/mops/k8s/watch.py

@@ -112,9 +112,14 @@ def callback_events(
 ) -> None:
     """Suitable for use with a daemon thread."""
     for namespace, obj, event in event_yielder:
-        should_exit = on_event(namespace, obj, event)
-        if should_exit:
-            break
+        try:
+            should_exit = on_event(namespace, obj, event)
+            if should_exit:
+                break
+        except Exception:
+            logger.exception(
+                "Exception in k8s watch event callback [probably a bug in mops] - continuing..."
+            )
 
 
 def _default_get_name(obj: ty.Any) -> str:

thds/mops/parallel.py

@@ -1,7 +1,7 @@
 import concurrent
 import typing as ty
 
-from thds.core import parallel
+from thds.core import inspect, log, parallel
 from thds.core.parallel import (  # noqa: F401; for backward-compatibility, since these came from here originally.
     IterableWithLen,
     IteratorWithLen,
@@ -29,6 +29,7 @@ def parallel_yield_results(
         error_fmt=ERROR,
         success_fmt=DONE,
         named=named,
+        progress_logger=log.getLogger(inspect.caller_module_name(__name__) or __name__).info,
     )
 
 

thds/mops/pure/_magic/api.py

@@ -50,6 +50,11 @@ class _MagicApi:
         pipeline_id: str = "",
         calls: ty.Collection[ty.Callable] = tuple(),
     ) -> ty.Callable[[ty.Callable[P, R]], sauce.Magic[P, R]]:
+        """This is the main pure.magic() decorator.  It is designed to be applied directly
+        at the site of function definition, i.e. on the `def`.  We dynamically capture
+        the fully qualified name of the function being decorated and use that to
+        look up the appropriate 'magic' configuration at the time of each call to the function.
+        """
         return sauce.make_magic(_get_config(), shim_or_builder, blob_root, pipeline_id, calls)
 
     @staticmethod
@@ -58,10 +63,29 @@ class _MagicApi:
         *,
         blob_root: uris.UriResolvable = "",
         pipeline_id: str = "",
+        config_path: str = "",
     ) -> ty.Callable[[F], F]:  # cleaner type for certain use cases
+        """This alternative API is designed for more dynamic use cases - rather than
+        decorating a function def directly, you can use this to create a more generic
+        decorator that can be applied within other code (not at module-level).
+
+        However, you must never apply pure.magic.deco to the same function from multiple
+        places, as this means that you have multiple different uses sharing the same
+        configuration path, which will lead to subtle bugs.
+
+        We attempt to detect this and raise an error if it happens. If it does, you should
+        provide an explicit unique config_path for each usage.
+        """
         return ty.cast(
             ty.Callable[[F], F],
-            _MagicApi.__call__(shim_or_builder, blob_root=blob_root, pipeline_id=pipeline_id),
+            sauce.make_magic(
+                _get_config(),
+                shim_or_builder,
+                blob_root=blob_root,
+                pipeline_id=pipeline_id,
+                calls=tuple(),
+                config_path=config_path,
+            ),
         )
 
     @staticmethod
@@ -122,6 +146,10 @@ class _MagicApi:
         m_config.blob_root.load_config(all_config)
         m_config.pipeline_id.load_config(all_config)
 
+    @staticmethod
+    def config_path(func: ty.Callable) -> str:
+        return sauce.make_magic_config_path(func)
+
 
 magic: ty.Final = _MagicApi()
 # we only instantiate this so we can have a call to magic() that is not __init__.

thds/mops/pure/_magic/sauce.py

@@ -6,7 +6,7 @@ import typing as ty
 
 from typing_extensions import ParamSpec
 
-from thds.core import futures, stack_context
+from thds.core import futures, log, stack_context
 from thds.mops._utils import config_tree
 
 from ..core import file_blob_store, pipeline_id, pipeline_id_mask, uris
@@ -38,6 +38,8 @@ class _MagicConfig:
         self.shim_bld[""] = make_builder(samethread_shim)  # default Shim
         self.pipeline_id[""] = "magic"  # default pipeline_id
 
+        self.all_registered_paths: set[str] = set()
+
     def __repr__(self) -> str:
         return f"MagicConfig(shim_bld={self.shim_bld}, blob_root={self.blob_root}, pipeline_id={self.pipeline_id})"
 
@@ -68,15 +70,17 @@ class Magic(ty.Generic[P, R]):
         self,
         func: ty.Callable[P, R],
         config: _MagicConfig,
+        magic_config_path: str,
         calls: ty.Collection[ty.Callable] = frozenset(),
     ):
         functools.update_wrapper(self, func)
-        self._func_config_path = full_name_and_callable(func)[0].replace("--", ".")
+        self._magic_config_path = magic_config_path
 
         self.config = config
+
         if p_id := pipeline_id_mask.extract_from_docstr(func, require=False):
             # this allows the docstring pipeline id to become 'the most specific' config.
-            self.config.pipeline_id.setv(p_id, self._func_config_path)
+            self.config.pipeline_id.setv(p_id, self._magic_config_path)
         self._shim = stack_context.StackContext[ty.Union[None, ShimName, ShimOrBuilder]](
             str(func) + "_SHIM", None  # none means nothing has been set stack-local
         )
@@ -104,7 +108,7 @@ class Magic(ty.Generic[P, R]):
     def _shim_builder_or_off(self) -> ty.Optional[ShimBuilder]:
         if stack_local_shim := self._shim():
             return to_shim_builder(stack_local_shim)
-        return self.config.shim_bld.getv(self._func_config_path)
+        return self.config.shim_bld.getv(self._magic_config_path)
 
     def _is_off(self) -> bool:
         return self._shim_builder_or_off is None
@@ -117,11 +121,11 @@ class Magic(ty.Generic[P, R]):
         return sb(f, args, kwargs)
 
     def _get_blob_root(self) -> str:
-        return self.config.blob_root.getv(self._func_config_path)()
+        return self.config.blob_root.getv(self._magic_config_path)()
 
     @property
     def _pipeline_id(self) -> str:
-        return self.config.pipeline_id.getv(self._func_config_path)
+        return self.config.pipeline_id.getv(self._magic_config_path)
 
     def submit(self, *args: P.args, **kwargs: P.kwargs) -> futures.PFuture[R]:
         """A futures-based interface that doesn't block on the result of the wrapped
@@ -138,26 +142,80 @@ class Magic(ty.Generic[P, R]):
 
     def __repr__(self) -> str:
         return (
-            f"Magic('{self._func_config_path}', shim={self._shim_builder_or_off},"
+            f"Magic('{self._magic_config_path}', shim={self._shim_builder_or_off},"
             f" blob_root='{self._get_blob_root()}', pipeline_id='{self._pipeline_id}')"
         )
 
 
+def make_magic_config_path(func: ty.Callable) -> str:
+    return full_name_and_callable(func)[0].replace("--", ".")
+
+
+class MagicReregistrationError(ValueError):
+    pass
+
+
 def make_magic(
     config: _MagicConfig,
     shim_or_builder: ty.Union[ShimName, ShimOrBuilder, None],
     blob_root: uris.UriResolvable,
     pipeline_id: str,
     calls: ty.Collection[ty.Callable],
+    *,
+    config_path: str = "",
 ) -> ty.Callable[[ty.Callable[P, R]], Magic[P, R]]:
+    """config_path is a dot-separated path that must be unique throughout your application.
+
+    By default it will be set to the thds.other.module.function_name of the decorated function.
+    """
+    error_logger = log.auto(__name__, "thds.mops.pure._magic.api").error
+    err_msg = (
+        "You are probably using pure.magic(.deco) from multiple places on the same function. You will need to specify a unique config_path for each usage."
+        if not config_path
+        else f"You supplied a config_path ({config_path}) but you reused the decorator on different functions with the same config_path."
+    )
+    err_msg += " See the comment in mops.pure._magic.sauce for more details."
+
+    def must_not_remagic_same_func(msg: str) -> None:
+        error_logger(f"{msg}; {err_msg}")
+        # if you see either of the above messages, consider whether you really need the magic
+        # configurability of pure.magic, or whether it might be better to instantiate and use
+        # MemoizingPicklingRunner directly without configurability. The reason overwriting
+        # configs, by applying pure.magic to the same callable from more than one location is
+        # disallowed is that you will get 'spooky action at a distance' between different parts
+        # of your application that are overwriting the base config for the same function.
+        # Another approach would be to use a wrapper `def` with a static @pure.magic decorator
+        # on it that calls the inner function, so that they are completely different functions
+        # as far as pure.magic is concerned.
+        raise MagicReregistrationError(msg)
+
+    magic_config_path_cache: set[str] = set()
+    # the reason for this cache is that there are cases where you may want to apply the _exact
+    # same_ base config to the same function multiple times - just for ease of use. And
+    # since this is the exact same config, we should allow it and treat it as though you
+    # had only applied it once.  Of course, if you later try to configure these
+    # applications separately, it won't work - these _are_ the same magic config path, so
+    # they're bound together via that config.
+
     def deco(func: ty.Callable[P, R]) -> Magic[P, R]:
-        fully_qualified_name = full_name_and_callable(func)[0].replace("--", ".")
+        fully_qualified_name = make_magic_config_path(func)
+        magic_config_path = config_path or fully_qualified_name
+
+        def deco_being_reapplied_to_same_func() -> bool:
+            return fully_qualified_name in magic_config_path_cache
+
+        if magic_config_path in config.all_registered_paths and not deco_being_reapplied_to_same_func():
+            must_not_remagic_same_func(f"Cannot re-register {magic_config_path} using pure.magic")
+
         if shim_or_builder is not None:
-            config.shim_bld[fully_qualified_name] = to_shim_builder(shim_or_builder)
+            config.shim_bld[magic_config_path] = to_shim_builder(shim_or_builder)
         if blob_root:  # could be empty string
-            config.blob_root[fully_qualified_name] = uris.to_lazy_uri(blob_root)
+            config.blob_root[magic_config_path] = uris.to_lazy_uri(blob_root)
         if pipeline_id:  # could be empty string
-            config.pipeline_id[fully_qualified_name] = pipeline_id
-        return Magic(func, config, calls)
+            config.pipeline_id[magic_config_path] = pipeline_id
+
+        magic_config_path_cache.add(fully_qualified_name)
+        config.all_registered_paths.add(magic_config_path)
+        return Magic(func, config, magic_config_path, calls)
 
     return deco

thds/mops/pure/core/source.py

@@ -33,6 +33,7 @@ have a Source object returned to it while it performs low-level deserialization.
 """
 
 import io
+import json
 import sys
 import typing as ty
 from functools import partial
@@ -71,19 +72,44 @@ def _hashref_uri(hash: hashing.Hash, type: ty.Literal["local", "remote"]) -> str
     return to_uri(local_hashref)
 
 
-def _read_hashref(hashref_uri: str) -> str:
+class _HashrefMeta(ty.NamedTuple):
+    size: int
+
+    @classmethod
+    def empty(cls) -> "_HashrefMeta":
+        return cls(size=0)
+
+    def serialize(self) -> str:
+        serialized = json.dumps(self._asdict())
+        return serialized
+
+    @classmethod
+    def deserialize(cls, serialized: ty.Union[str, ty.Sequence[str]]) -> "_HashrefMeta":
+        s = serialized if isinstance(serialized, str) else serialized[0]
+        try:
+            return cls(**json.loads(s))
+        except json.JSONDecodeError:
+            logger.warning("Failed to deserialize hashref metadata '%s'", serialized)
+            return cls.empty()
+
+
+def _read_hashref(hashref_uri: str) -> ty.Tuple[str, _HashrefMeta]:
     """Return URI represented by this hashref. Performs IO."""
     uri_bytes = io.BytesIO()
     lookup_blob_store(hashref_uri).readbytesinto(hashref_uri, uri_bytes)
-    uri = uri_bytes.getvalue().decode()
+    content = uri_bytes.getvalue().decode()
+    uri, *rest = content.split("\n")
     assert uri, f"Hashref from {hashref_uri} is empty"
-    return uri
+    if not rest:
+        return uri, _HashrefMeta.empty()
+    return uri, _HashrefMeta.deserialize(rest)
 
 
-def _write_hashref(hashref_uri: str, uri: str) -> None:
+def _write_hashref(hashref_uri: str, uri: str, size: int) -> None:
     """Write URI to this hashref. Performs IO."""
     assert uri, f"Should never encode hashref ({hashref_uri}) pointing to empty URI"
-    lookup_blob_store(hashref_uri).putbytes(hashref_uri, uri.encode(), type_hint="text/plain")
+    content = "\n".join([uri, _HashrefMeta(size=size).serialize()])
+    lookup_blob_store(hashref_uri).putbytes(hashref_uri, content.encode(), type_hint="text/plain")
 
 
 def source_from_hashref(hash: hashing.Hash) -> Source:
@@ -92,7 +118,9 @@ def source_from_hashref(hash: hashing.Hash) -> Source:
     local_file_hashref_uri = _hashref_uri(hash, "local")
     remote_hashref_uri = _hashref_uri(hash, "remote")
 
-    def remote_uri(allow_blob_not_found: bool = True) -> str:
+    def remote_uri_and_meta(
+        allow_blob_not_found: bool = True,
+    ) -> ty.Tuple[str, _HashrefMeta]:
         try:
             return _read_hashref(remote_hashref_uri)
         except Exception as e:
@@ -102,7 +130,7 @@ def source_from_hashref(hash: hashing.Hash) -> Source:
                 # 'remote' blob not found is sometimes fine, but anything else is weird
                 # and we should raise.
                 raise
-            return ""
+            return "", _HashrefMeta.empty()
 
     try:
         # we might be on the same machine where this was originally invoked.
@@ -110,7 +138,9 @@ def source_from_hashref(hash: hashing.Hash) -> Source:
         # Then, there's no need to bother grabbing the remote_uri
         # - but for debugging's sake, it's quite nice to actually
         # have the full remote URI as well even if we're ultimately going to use the local copy.
-        return source.from_file(_read_hashref(local_file_hashref_uri), hash=hash, uri=remote_uri())
+        local_uri, _ = _read_hashref(local_file_hashref_uri)
+        remote_uri, _ = remote_uri_and_meta()
+        return source.from_file(local_uri, hash=hash, uri=remote_uri)
     except FileNotFoundError:
         # we are not on the same machine as the local ref. assume we need the remote URI.
         pass
@@ -120,14 +150,17 @@ def source_from_hashref(hash: hashing.Hash) -> Source:
             raise
 
     # no local file, so we assume there must be a remote URI.
-    return source.from_uri(remote_uri(False), hash=hash)
+    remote_uri, meta = remote_uri_and_meta(False)
+    return source.from_uri(remote_uri, hash=hash, size=meta.size)
 
 
-def _upload_and_create_remote_hashref(local_path: Path, remote_uri: str, hash: hashing.Hash) -> None:
+def _upload_and_create_remote_hashref(
+    local_path: Path, remote_uri: str, hash: hashing.Hash, size: int
+) -> None:
     # exists only to provide a local (non-serializable) closure around local_path and remote_uri.
     lookup_blob_store(remote_uri).putfile(local_path, remote_uri)
     # make sure we never overwrite a hashref until it's actually going to be valid.
-    _write_hashref(_hashref_uri(hash, "remote"), remote_uri)
+    _write_hashref(_hashref_uri(hash, "remote"), remote_uri, size)
 
 
 def _auto_remote_uri(hash: hashing.Hash) -> str:
@@ -155,7 +188,7 @@ def prepare_source_argument(source_: Source) -> ty.Union[str, hashing.Hash]:
         deferred_work.add(
             __name__ + "-localhashref",
             source_.hash,
-            partial(_write_hashref, _hashref_uri(source_.hash, "local"), str(local_path)),
+            partial(_write_hashref, _hashref_uri(source_.hash, "local"), str(local_path), source_.size),
         )
         # then also register pending upload - if the URI is a local file, we need to determine a
         # remote URI for this thing automagically; otherwise, use whatever was already
@@ -164,7 +197,9 @@ def prepare_source_argument(source_: Source) -> ty.Union[str, hashing.Hash]:
         deferred_work.add(
             __name__ + "-remotehashref",
             source_.hash,
-            partial(_upload_and_create_remote_hashref, local_path, remote_uri, source_.hash),
+            partial(
+                _upload_and_create_remote_hashref, local_path, remote_uri, source_.hash, source_.size
+            ),
         )
     else:
         # prepare to (later, if necessary) create a remote hashref, because this Source
@@ -172,7 +207,7 @@ def prepare_source_argument(source_: Source) -> ty.Union[str, hashing.Hash]:
         deferred_work.add(
             __name__,
             source_.hash,
-            partial(_write_hashref, _hashref_uri(source_.hash, "remote"), source_.uri),
+            partial(_write_hashref, _hashref_uri(source_.hash, "remote"), source_.uri, source_.size),
         )
 
     return hashing.Hash(algo=sys.intern(source_.hash.algo), bytes=source_.hash.bytes)
@@ -192,15 +227,17 @@ def prepare_source_argument(source_: Source) -> ty.Union[str, hashing.Hash]:
 # just that mops must detect Sources in the return value and must force an upload on them.
 # In essence, this creates a bifurcated code path for Sources during serialization; if
 # we're "on the way out", we avoid uploading until it is clear that the data will be used
-# in a remote environment. Whereas "on the way back", we must always upload, and nothing
-# can or should be deferred; upload should happen at the time of serialization.
+# in a remote environment. Whereas "on the way back", we must always upload -- there, we
+# defer uploads until everything is serialized, then we perform all deferred uploads in
+# parallel, prior to writing the serialized result.
 #
 # Nevertheless, a local caller should still be able to short-circuit the _download_ by
 # using a locally-created File, if on the same machine where the local file was created.
 
 
 class SourceResult(ty.NamedTuple):
-    """Contains the fully-specified local URI and remote URI, plus (probably) a Hash.
+    """Contains the fully-specified local URI and remote URI, plus (probably) a Hash
+    and a size.
 
     Everything is defined right here. No need for any kind of dynamic lookup, and
     optimization buys us nothing, since memoization only operates on arguments.
@@ -210,6 +247,10 @@ class SourceResult(ty.NamedTuple):
     hash: ty.Optional[hashing.Hash]
     file_uri: str
 
+    size: int = 0
+    # instances of older versions of this namedtuple will be missing this field.
+    # we supply a default for backward-compatibility.
+
 
 class DuplicateSourceBasenameError(ValueError):
     """This is not a catchable error - it will be raised inside the mops result-wrapping
@@ -247,7 +288,7 @@ def prepare_source_result(source_: Source, existing_uris: ty.Collection[str] = t
         else:
             file_uri = ""
             logger.debug("Creating a SourceResult for a URI that is presumed to already be uploaded.")
-        return SourceResult(source_.uri, source_.hash, file_uri)
+        return SourceResult(source_.uri, source_.hash, file_uri, source_.size)
 
     # by definition, if this is a file URI, it now needs to be uploaded, because we could
     # be transferring back to an orchestrator on a different machine, but also because a
@@ -277,15 +318,17 @@ def prepare_source_result(source_: Source, existing_uris: ty.Collection[str] = t
         partial(_put_file_to_blob_store, local_path, remote_uri),
     )
     # upload must _always_ happen on remotely-returned Sources, as detailed above.
-    return SourceResult(remote_uri, source_.hash, source_.uri)
+    return SourceResult(remote_uri, source_.hash, source_.uri, source_.size)
 
 
-def source_from_source_result(remote_uri: str, hash: ty.Optional[hashing.Hash], file_uri: str) -> Source:
+def source_from_source_result(
+    remote_uri: str, hash: ty.Optional[hashing.Hash], file_uri: str, size: int
+) -> Source:
     """Call when deserializing a remote function return value on the orchestrator side, to
     replace all SourceResults with the intended Source object.
     """
     if not file_uri:
-        return source.from_uri(remote_uri, hash=hash)
+        return source.from_uri(remote_uri, hash=hash, size=size)
 
     local_path = source.path_from_uri(file_uri)
 
@@ -305,7 +348,7 @@ def source_from_source_result(remote_uri: str, hash: ty.Optional[hashing.Hash],
             logger.warning(
                 f"Unable to reuse destination local path {local_path} when constructing Source {remote_uri}: {e}"
             )
-    return source.from_uri(remote_uri, hash=hash)
+    return source.from_uri(remote_uri, hash=hash, size=size)
 
 
 def create_source_at_uri(filename: StrOrPath, destination_uri: str) -> Source:

thds/mops/pure/pickling/pickles.py

@@ -14,7 +14,7 @@ from pathlib import Path
 from thds.core import hashing, log, source
 
 from ..core.script_support import add_main_module_function, get_main_module_function
-from ..core.source import source_from_hashref, source_from_source_result
+from ..core.source import SourceResult, source_from_hashref, source_from_source_result
 from ..core.uris import get_bytes, lookup_blob_store
 
 logger = log.getLogger(__name__)
@@ -131,17 +131,15 @@ class UnpickleSourceHashrefArgument(ty.NamedTuple):
         return source_from_hashref(self.hash)
 
 
-class UnpickleSourceResult(ty.NamedTuple):
+class UnpickleSourceResult(SourceResult):
     """Stability for this is not critical, as it will only ever exist in the result
     payload, which does not participate in memoization.
     """
 
-    remote_uri: str
-    hash: ty.Optional[hashing.Hash]
-    file_uri: str
-
     def __call__(self) -> source.Source:
-        return source_from_source_result(*self)
+        return source_from_source_result(
+            remote_uri=self.remote_uri, hash=self.hash, file_uri=self.file_uri, size=self.size
+        )
 
 
 class UnpickleFunctionWithLogicKey(ty.NamedTuple):

thds/mops/pure/tools/summarize/run_summary.py

@@ -5,7 +5,7 @@ import pickle
 import typing as ty
 from pathlib import Path
 
-from thds.core import config, log, pickle_visit, source
+from thds.core import config, files, log, pickle_visit, source
 from thds.mops.pure.core.memo import function_memospace
 from thds.mops.pure.core.metadata import get_invoked_by
 
@@ -152,10 +152,7 @@ def log_function_execution(
         log_entry["uris_in_rvalue"] = sorted(source_uris)
 
     try:
-        assert not log_file.exists(), f"Log file '{log_file}' should not already exist"
-        with log_file.open("w") as f:
-            json.dump(log_entry, f, indent=2)
+        with files.atomic_text_writer(log_file) as file:
+            json.dump(log_entry, file, indent=2)
     except Exception:
-        logger.info(
-            f"Unable to write mops function invocation log file at '{log_file}' - you may have multiple callers for the same invocation"
-        )
+        logger.exception(f"Failed to write mops function invocation log file at '{log_file}'")

{thds_mops-3.9.20250919153256.dist-info → thds_mops-3.10.20251104012416.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.mops
-Version: 3.9.20250919153256
+Version: 3.10.20251104012416
 Summary: ML Ops tools for Trilliant Health
 Author-email: Trilliant Health <info@trillianthealth.com>
 Project-URL: Repository, https://github.com/TrilliantHealth/ds-monorepo

{thds_mops-3.9.20250919153256.dist-info → thds_mops-3.10.20251104012416.dist-info}/RECORD

@@ -2,10 +2,10 @@ thds/mops/__about__.py,sha256=IW_3wy8wEdrVducoBdiVgD7oYOY4J8yO1ezBaPtrc6U,215
 thds/mops/__init__.py,sha256=dbujDxVVfHpWP7OyfjEdNVHLtKx99rsNQPYfjTKn5Lg,127
 thds/mops/_compat.py,sha256=fO1YYEu6LF1re-VXl4P_8RXXLeKt4BgI9NTlHTgNpLk,357
 thds/mops/config.py,sha256=T62YskXvzAfxNgpq2jMatHgoIHfRV_z4cvJ8Rl_TZ6E,2015
-thds/mops/parallel.py,sha256=ynzT7uEtF1sfUi7NS9fHg1I5EhQtSs3p5hNzP3xwAWE,931
+thds/mops/parallel.py,sha256=F6vUhSTO--CY82vyYtWFtspmgd0RxoxQ_EUrCnTm93Q,1039
 thds/mops/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-thds/mops/_utils/config_tree.py,sha256=wMwkw81FDOo2Ld9rEmtobZMW2ZCUmxWSI2Bp_p0pwa8,7151
+thds/mops/_utils/config_tree.py,sha256=Q9mPAAolYPqKe6dkvfetg-wmVARyD7WOx123OLQ8_sU,7326
 thds/mops/_utils/exception.py,sha256=Itj6ceieCdGrKZ2JdW_DIM88Wgvvw104cfbH1RNn6Go,394
 thds/mops/_utils/locked_cache.py,sha256=ROIkwu-_FcXlNyCreWQeE5cyL9XrNW7drWsolTgeajM,2523
 thds/mops/_utils/names.py,sha256=tPPaXCyduUXqmbdvIg3ygevERnKM3YIs868BeaKX5XY,824
@@ -18,7 +18,7 @@ thds/mops/impure/runner.py,sha256=UI1NZWMZ_5TQHfFKLnoiSm2zDR3zCunTKFmJoybkyCo,28
 thds/mops/k8s/__init__.py,sha256=zl4GVcCFRvPscyo6gvv5Lx0OKB7d3QjtVFjYurnxMuE,764
 thds/mops/k8s/_launch.py,sha256=hgPty47CdwryPHKMmEnoxSsSvcSpXhHYSVYnLC2QJb0,10956
 thds/mops/k8s/_shared.py,sha256=MR-s6ijWUHZGjxK_fsOpHuRDB6kuofjo5xiIb7ul2VM,86
-thds/mops/k8s/apply_yaml.py,sha256=hVW6dIVbNdzHdbGlc2VAPGkdByv_rH2oPybyIm7tKIM,820
+thds/mops/k8s/apply_yaml.py,sha256=cGjnMkJ3Ny_D9CgN5FrHAiRZx8VRzA_U5AY4OTD8WxA,1474
 thds/mops/k8s/auth.py,sha256=0zs4TQgkD6VPrhDD43xt7JGwP6uWf3ctySGLcPKN7iw,1691
 thds/mops/k8s/batching.py,sha256=Djt17ffxWyTq4Q7XcAKQdCe9JIIfPahHwm0wqgFqevI,8368
 thds/mops/k8s/config.py,sha256=_znocX5BW8kfG_Cbq6f3apx5FqSihD7Tmic-SBkVjMQ,2992
@@ -31,16 +31,16 @@ thds/mops/k8s/namespace.py,sha256=Z6trVTU9WFashto4PqIhTcxu-foOF93W0TpgqCU7WIA,38
 thds/mops/k8s/node_selection.py,sha256=Gy2Jz8IxZblg2LmtGg8-MtKI4RmXz2AMXqFPP8OQyu0,2065
 thds/mops/k8s/retry.py,sha256=JVfP304kItpLs5nrONHE5UWkVWlrFGlV_oFQqhq3zHg,2846
 thds/mops/k8s/too_old_resource_version.py,sha256=S7ltVA-LrxUpQ8Q__AB0nQmezN8Mmnx5oKK62_baAKI,1500
-thds/mops/k8s/uncertain_future.py,sha256=60v9yVlhnCDN_yUv8l4Z4KafR4TsTGxN7dprkGI8pQQ,7152
+thds/mops/k8s/uncertain_future.py,sha256=_ix-4EqZE_MY5sbLjT-lQ9GIa7woQ3iunPVtDapIhi8,7752
 thds/mops/k8s/wait_job.py,sha256=_X5lSn-3CE4V-_ra0kF1WtxkAiOgqSom8mU1-0hhMio,2445
 thds/mops/k8s/warn_image_backoff.py,sha256=ls_zLSnRbJjO4ICjq1Rk21EXh190l2dT6nKg-PT8Das,1934
-thds/mops/k8s/watch.py,sha256=4LBLZ9s9hp4jSOr3OTBDMWWCEDedgxsUoYt89zCLLsw,14020
+thds/mops/k8s/watch.py,sha256=nH9HBoRCvt8FRhUav9at71QnTCh0plWga3rp7aUYb1E,14212
 thds/mops/k8s/tools/krsync.py,sha256=us7pXX0-bRMwD2oAno7Z6BJcPs6FgaUabHW0STyQJYg,1773
 thds/mops/k8s/tools/krsync.sh,sha256=fWgwkdzWnJeTbzEA_uBiIIi-bNU4nXAYj3dNovyRluU,747
 thds/mops/pure/__init__.py,sha256=3xLimQ2JWdeq1YgPs7bPwlwOspzPRwaR2w2KX7vfJU0,1624
 thds/mops/pure/_magic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-thds/mops/pure/_magic/api.py,sha256=kSlediIZQYsmeHB8plP6osjvUuSEVW4NWdY9ADia12Y,5094
-thds/mops/pure/_magic/sauce.py,sha256=xLaseOhoGgntmz6nZdj2te85sOaOiWExBLMGdeNmKsI,6870
+thds/mops/pure/_magic/api.py,sha256=eG1NEl9FR_FE1yLfN0hiLd0uLWu-gKzw1FCIJR5pMbw,6424
+thds/mops/pure/_magic/sauce.py,sha256=BfzQfEnQarcLuTbKyCKLU8F6BtJ-v6kn1SRIf675cTc,9804
 thds/mops/pure/_magic/shims.py,sha256=CXN8wlHv039oKRzDtp5YFDlwGXmmaheWLCi2I95gSeM,1212
 thds/mops/pure/adls/__init__.py,sha256=fw67xxwnizBurScMa-_zWb94lo5gamEVRt27V4bR0jc,54
 thds/mops/pure/adls/_files.py,sha256=9m35Y4elWF0DjgAXVp4oi5CaY6fXWt8n67PilWxWJns,821
@@ -58,7 +58,7 @@ thds/mops/pure/core/pipeline_id_mask.py,sha256=AVAy06TdNAmivxGec1gahBYvkJCn7yn-g
 thds/mops/pure/core/script_support.py,sha256=4VCBL5AfGSHcZWpOxMw6nnAbQyk1B-979G_OjvUg9B0,953
 thds/mops/pure/core/serialize_big_objs.py,sha256=YcOS1ccs82ZWO7nTbeumErMzYVe4hgXCTsfvMggYmd8,2332
 thds/mops/pure/core/serialize_paths.py,sha256=JoVXFGSA68QbL4oY8tQbp9MoizTCKj_nPRCuA3i03i8,6122
-thds/mops/pure/core/source.py,sha256=R36ajrCU1JdWF-8iD8YqAiP-q39ypZqf2DeBsqC9lYo,15105
+thds/mops/pure/core/source.py,sha256=i6SRgOFfkdaidx6uEq6poGFUU1zQWiqaXPIG-l5zY7Q,16580
 thds/mops/pure/core/types.py,sha256=_3gDwztDKV4Xeyw2jvyMRJAjmR6gRsmfYmsRCcZMUwI,5436
 thds/mops/pure/core/uris.py,sha256=qO9_f-ro7kax6haNOPTPe81-_aUSRFELeeZH4PMTTU4,2694
 thds/mops/pure/core/use_runner.py,sha256=m1Mu1XDr3xRf_u_VSiHfTG4TH6fnSg0IqwmtbLKG_oc,2103
@@ -88,7 +88,7 @@ thds/mops/pure/pickling/__init__.py,sha256=WNdG8PdJCk-kYaXkvvPa--hjYGoUlBXG3w2X8
 thds/mops/pure/pickling/_pickle.py,sha256=YB8xbqDiwdk8ccnVZ2_4kQn98V2JSrFqw2E3J-jEHlA,8081
 thds/mops/pure/pickling/memoize_only.py,sha256=oI5CMy6IEJc46Gb_BGWNUuAe3fysS7HxRSTajN0WssI,837
 thds/mops/pure/pickling/mprunner.py,sha256=VWYS_PXLgYJetK69CCZ0-b1109-QBHWssC0MskHww94,8831
-thds/mops/pure/pickling/pickles.py,sha256=CSlnjLssE0Ad8YzqyaKqWCSNyW5LiMFKiXO6hWAZmvU,5097
+thds/mops/pure/pickling/pickles.py,sha256=KYkPexi5mGWjrv9uZxt4iWuBUPyYlME2FQIwRiPlPqc,5134
 thds/mops/pure/pickling/remote.py,sha256=7JXZRGnLI5y5dqElIDrhIlaRv6Q_zQ_78aqNhO7O4KY,8478
 thds/mops/pure/pickling/sha256_b64.py,sha256=HL0cPixHPZYuZDVDBscxsnI-3a2amWEfw-LseOX-PyY,2916
 thds/mops/pure/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -106,11 +106,11 @@ thds/mops/pure/tools/sha256_b64_addressed.py,sha256=SECAiw3xSqpsrBBZix0MgJRTQrbH
 thds/mops/pure/tools/stress.py,sha256=N7C8kLpaGbImeEYlT5jsEl1metvsUu8cnfyQ8vFN0H8,2541
 thds/mops/pure/tools/summarize/__init__.py,sha256=MSmt_5Xg84uHqzTN38JwgseJK8rsJn_11A8WD99VtEo,61
 thds/mops/pure/tools/summarize/cli.py,sha256=7kDtn24ok8oBO3jFjlMmOK3jnZYpMoE_5Y8fmDH8Imc,11524
-thds/mops/pure/tools/summarize/run_summary.py,sha256=w45qiQr7elrHDiK9Hgs85gtU3gwLuXa447ih1Y23BBY,5776
+thds/mops/pure/tools/summarize/run_summary.py,sha256=glEN_YxUGADzp2Ofvr4ZDeHvnZ1znNR7HD7EATn1sPI,5644
 thds/mops/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/testing/deferred_imports.py,sha256=f0ezCgQAtzTqW1yAOb0OWgsB9ZrlztLB894LtpWDaVw,3780
-thds_mops-3.9.20250919153256.dist-info/METADATA,sha256=qFAGHQOOzR874kPcYKsg8bfhifIqP3czu_CtsFF0Jgo,2225
-thds_mops-3.9.20250919153256.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_mops-3.9.20250919153256.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
-thds_mops-3.9.20250919153256.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_mops-3.9.20250919153256.dist-info/RECORD,,
+thds_mops-3.10.20251104012416.dist-info/METADATA,sha256=3SGjSyqzuAPZqDy7pJ9V90TlWz6VzcVlpL3MQKEmXBk,2226
+thds_mops-3.10.20251104012416.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_mops-3.10.20251104012416.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
+thds_mops-3.10.20251104012416.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_mops-3.10.20251104012416.dist-info/RECORD,,