thds.mops 3.7.20250424212724__py3-none-any.whl → 3.8.20250425182948__py3-none-any.whl

thds/mops/pure/_magic/api.py

@@ -48,8 +48,9 @@ class _MagicApi:
         *,
         blob_root: uris.UriResolvable = "",
         pipeline_id: str = "",
+        calls: ty.Collection[ty.Callable] = tuple(),
     ) -> ty.Callable[[ty.Callable[P, R]], sauce.Magic[P, R]]:
-        return sauce.make_magic(_get_config(), shim_or_builder, blob_root, pipeline_id)
+        return sauce.make_magic(_get_config(), shim_or_builder, blob_root, pipeline_id, calls)
 
     @staticmethod
     def deco(

thds/mops/pure/_magic/sauce.py

@@ -68,6 +68,7 @@ class Magic(ty.Generic[P, R]):
         self,
         func: ty.Callable[P, R],
         config: _MagicConfig,
+        calls: ty.Collection[ty.Callable] = frozenset(),
     ):
         functools.update_wrapper(self, func)
         self._func_config_path = full_name_and_callable(func)[0].replace("--", ".")
@@ -80,6 +81,7 @@ class Magic(ty.Generic[P, R]):
             str(func) + "_SHIM", None  # none means nothing has been set stack-local
         )
         self.runner = MemoizingPicklingRunner(self._shimbuilder, self._get_blob_root)
+        self.runner.calls(func, *calls)
         self._func = use_runner(self.runner, self._is_off)(func)
         self.__doc__ = f"{func.__doc__}\n\nMagic class info:\n{self.__class__.__doc__}"
         self.__wrapped__ = func
@@ -138,6 +140,7 @@ def make_magic(
     shim_or_builder: ty.Union[ShimName, ShimOrBuilder, None],
     blob_root: uris.UriResolvable,
     pipeline_id: str,
+    calls: ty.Collection[ty.Callable],
 ) -> ty.Callable[[ty.Callable[P, R]], Magic[P, R]]:
     def deco(func: ty.Callable[P, R]) -> Magic[P, R]:
         fully_qualified_name = full_name_and_callable(func)[0].replace("--", ".")
@@ -147,6 +150,6 @@ def make_magic(
             config.blob_root[fully_qualified_name] = uris.to_lazy_uri(blob_root)
         if pipeline_id:  # could be empty string
             config.pipeline_id[fully_qualified_name] = pipeline_id
-        return Magic(func, config)
+        return Magic(func, config, calls)
 
     return deco

thds/mops/pure/core/memo/__init__.py

@@ -1,3 +1,4 @@
+from . import calls, unique_name_for_function  # noqa: F401
 from .function_memospace import (  # noqa
     args_kwargs_content_address,
     make_function_memospace,

thds/mops/pure/core/memo/calls.py

@@ -0,0 +1,91 @@
+"""This module currently exists only to serve the use case of tracking function logic keys recursively
+for mops-wrapped functions that call other mops-wrapped functions...
+
+which is _often but not always_ an anti-pattern...
+"""
+
+import typing as ty
+
+from .unique_name_for_function import (
+    extract_function_logic_key_from_docstr,
+    make_unique_name_including_docstring_key,
+)
+
+
+def resolve(
+    calls_registry: ty.Mapping[ty.Callable, ty.Collection[ty.Callable]],
+    origin_callable: ty.Callable,
+) -> ty.List[ty.Callable]:
+    """Using the 'edges' defined in the mapping, return a set of all callables recursively
+    reachable from the origin callable, not including the origin callable itself.
+    """
+    visited = list()
+    stack = [origin_callable]
+
+    while stack:
+        current_function = stack.pop()
+        if current_function in visited:
+            continue
+
+        if current_function is not origin_callable:
+            visited.append(current_function)
+
+        stack.extend(calls_registry.get(current_function, []))
+
+    return visited
+
+
+# the below code is operating under the assumption that putting all function full names
+# (module and name) plus their function logic keys inside the memo uri explicitly is
+# better than hashing them all together, because this gives users the standard amount of
+# 'debuggability' that they've come to expect when it comes to things like function names
+# and function-logic-keys.
+#
+# it _will_ lead to longer memo uris, which is unfortunate in some ways.
+#
+# an alternative would be to store these inside the Invocation and also change the _hash_
+# that we're already computing from being a hash of _only_ the args, kwargs to a hash
+# of those plus these.
+#
+# but there's great utility in being able to see that the hash itself comes only
+# from the args, kwargs - and it is not expected that the set of functions 'called'
+# would change during runtime - it should be static like everything else.
+#
+# A third alternative would be to use a second hash (calls-<the hash>) and then embed the
+# actual names in some other place - either the invocation itself, or possibly as
+# metadata. But I also think there's utility in being able to see those function names as
+# part of the overall memo uri, rather than having to go look at metadata to see it.  It
+# also makes it much more obvious that we're doing nesting - which is frowned upon for
+# most use cases, and good to make visible even if the use case is valuable.
+
+
+CALLS_PREFIX = "calls-"
+
+
+def combine_function_logic_keys(functions: ty.Iterable[ty.Callable]) -> tuple[str, ...]:
+    funcs_and_logic_keys = list()
+    for func in functions:
+        flk = extract_function_logic_key_from_docstr(func)
+        if flk:
+            # if the function doesn't have a function logic key, then it can't really 'invalidate'
+            # anything, so we can ignore it.
+            funcs_and_logic_keys.append(CALLS_PREFIX + make_unique_name_including_docstring_key(func))
+    return tuple(sorted(funcs_and_logic_keys))
+
+
+class CallsPieces(ty.NamedTuple):
+    remaining_prefix: str
+    full_function_name: str
+    calls_functions: list[str]
+
+
+def split_off_calls_strings(memo_str_not_including_args_kwargs: str, separator: str) -> CallsPieces:
+    calls_functions = list()
+    rest = memo_str_not_including_args_kwargs
+    while True:
+        rest, full_function_name = rest.rsplit(separator, 1)
+        if full_function_name.startswith(CALLS_PREFIX):
+            calls_functions.append(full_function_name[len(CALLS_PREFIX) :])
+        else:
+            break
+    return CallsPieces(rest, full_function_name, calls_functions)

thds/mops/pure/core/memo/function_memospace.py

@@ -147,6 +147,7 @@ from ..pipeline_id_mask import (
     pipeline_id_mask,
 )
 from ..uris import lookup_blob_store
+from . import calls
 from .unique_name_for_function import make_unique_name_including_docstring_key, parse_unique_name
 
 
@@ -227,6 +228,7 @@ class MemoUriComponents(ty.NamedTuple):
     function_module: str
     function_name: str
     function_logic_key: str
+    calls_functions: ty.List[str]
     args_hash: str
 
 
@@ -247,8 +249,12 @@ def parse_memo_uri(
 
     runner_prefix = runner_prefix.rstrip(separator)
     rest, args_hash = memo_uri.rsplit(separator, 1)  # args hash is last component
-    rest, full_function_name = rest.rsplit(separator, 1)
-    pipeline_id = rest[len(runner_prefix) :]
+
+    remaining_prefix, full_function_name, calls_functions = calls.split_off_calls_strings(
+        rest, separator
+    )
+
+    pipeline_id = remaining_prefix[len(runner_prefix) :]
     pipeline_id = pipeline_id.strip(separator)
 
     function_parts = parse_unique_name(full_function_name)
@@ -259,6 +265,7 @@ def parse_memo_uri(
         function_parts.module,
         function_parts.name,
         function_parts.function_logic_key,
+        calls_functions,
         args_hash,
     )
 

thds/mops/pure/pickling/_pickle.py

@@ -88,7 +88,7 @@ def read_partial_pickle(full_bytes: bytes) -> ty.Tuple[bytes, ty.Any]:
     # non-pickle metadata and embedding it at the beginning of the file.
     first_pickle_pos = full_bytes.find(b"\x80")
     if first_pickle_pos == -1:
-        raise ValueError("Unable to find a pickle in the bytes")
+        raise ValueError(f"Unable to find a pickle in bytes of length {len(full_bytes)}")
     return (
         full_bytes[:first_pickle_pos],
         CallableUnpickler(io.BytesIO(full_bytes[first_pickle_pos:])).load(),
@@ -102,7 +102,10 @@ def make_read_header_and_object(
     type_hint: str, xf_header: ty.Optional[ty.Callable[[bytes], H]] = None
 ) -> ty.Callable[[str], ty.Tuple[H, ty.Any]]:
     def read_object(uri: str) -> ty.Tuple[H, ty.Any]:
-        header, unpickled = read_partial_pickle(get_bytes(uri, type_hint=type_hint))
+        uri_bytes = get_bytes(uri, type_hint=type_hint)
+        if not uri_bytes:
+            raise ValueError(f"{uri} exists but is empty - something is very wrong.")
+        header, unpickled = read_partial_pickle(uri_bytes)
         return (xf_header or (lambda h: h))(header), unpickled  # type: ignore
 
     return read_object

thds/mops/pure/pickling/mprunner.py

@@ -6,8 +6,11 @@ See runner.local.py for the core runner implementation.
 """
 
 import typing as ty
+from collections import defaultdict
 from functools import partial
 
+from typing_extensions import Self
+
 from thds.core import cache, log
 from thds.core.stack_context import StackContext
 
@@ -94,6 +97,23 @@ class MemoizingPicklingRunner:
 
         self._run_directory = run_summary.create_mops_run_directory()
 
+        self._calls_registry: dict[ty.Callable, list[ty.Callable]] = defaultdict(list)
+
+    def calls(self, caller: ty.Callable, *callees: ty.Callable) -> Self:
+        """Register that the first Callable calls the provided Callables(s).
+
+        This is (currently) used to ensure that function-logic-keys on the callees affect
+        the memoization of the caller. Callees that do not have a function-logic-key will
+        be ignored for this purpose; however there are no known reasons why your
+        underlying Callable should not have a function-logic-key, unless it has never been
+        modified since its creation.
+
+        The interface is more general and could in theory be used for other purposes in
+        the future.
+        """
+        self._calls_registry[caller].extend(callees)
+        return self  # returns self mainly to faciliate use with use_runner.
+
     def shared(self, *objs: ty.Any, **named_objs: ty.Any) -> None:
         """Set up memoizing pickle serialization for these objects.
 
@@ -162,6 +182,7 @@ class MemoizingPicklingRunner:
                 self._wrap_shim_builder,
                 _pickle.read_metadata_and_object,
                 self._run_directory,
+                self._calls_registry,
             )(
                 self._rerun_exceptions,
                 memo.make_function_memospace(

thds/mops/pure/runner/local.py

@@ -43,6 +43,7 @@ def invoke_via_shim_or_return_memoized(  # noqa: C901
     shim_builder: types.ShimBuilder,
     get_meta_and_result: types.GetMetaAndResult,
     run_directory: ty.Optional[Path] = None,
+    calls_registry: ty.Mapping[ty.Callable, ty.Collection[ty.Callable]] = dict(),  # noqa: B006
 ) -> ty.Callable[[bool, str, ty.Callable[..., T], Args, Kwargs], T]:
     @scope.bound
     def create_invocation__check_result__wait_shim(
@@ -80,7 +81,12 @@ def invoke_via_shim_or_return_memoized(  # noqa: C901
         scope.enter(deferred_work.open_context())  # optimize Source objects during serialization
 
         args_kwargs_bytes = serialize_args_kwargs(storage_root, func, args, kwargs)
-        memo_uri = fs.join(function_memospace, memo.args_kwargs_content_address(args_kwargs_bytes))
+        memo_uri = fs.join(
+            function_memospace,
+            *memo.calls.combine_function_logic_keys(memo.calls.resolve(calls_registry, func)),
+            # ^ these will embedded as extra nesting.
+            memo.args_kwargs_content_address(args_kwargs_bytes),
+        )
 
         # Define some important and reusable 'chunks of work'
 

{thds_mops-3.7.20250424212724.dist-info → thds_mops-3.8.20250425182948.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.mops
-Version: 3.7.20250424212724
+Version: 3.8.20250425182948
 Summary: ML Ops tools for Trilliant Health
 Author-email: Trilliant Health <info@trillianthealth.com>
 Project-URL: Repository, https://github.com/TrilliantHealth/ds-monorepo

{thds_mops-3.7.20250424212724.dist-info → thds_mops-3.8.20250425182948.dist-info}/RECORD

@@ -36,8 +36,8 @@ thds/mops/k8s/tools/krsync.py,sha256=us7pXX0-bRMwD2oAno7Z6BJcPs6FgaUabHW0STyQJYg
 thds/mops/k8s/tools/krsync.sh,sha256=lskw4COt51Bv1yy2IAYUc8u8uQV-coSyUiOT8rADKkQ,546
 thds/mops/pure/__init__.py,sha256=kbG0lMvXRBS3LGbb2gPPE9-qjYMXrypyb2tJX2__aZc,1533
 thds/mops/pure/_magic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-thds/mops/pure/_magic/api.py,sha256=7FxlBVLmh5kHp2Jro8gw_I1rmNaaNGitJtOnjega1Bs,5034
-thds/mops/pure/_magic/sauce.py,sha256=AdFNqldbaZ41k8o8CSug75boR2FBEdenOwZTnNBxaK8,6173
+thds/mops/pure/_magic/api.py,sha256=kSlediIZQYsmeHB8plP6osjvUuSEVW4NWdY9ADia12Y,5094
+thds/mops/pure/_magic/sauce.py,sha256=LDiv4YnJ4yQ7YW8t4Qr8bB6p3l5KcteSxMuO--BRJ3A,6316
 thds/mops/pure/_magic/shims.py,sha256=JI49ddv6lEUmNVsEl-XkGlsx2RpOMQoIOSSSfootYE8,1188
 thds/mops/pure/adls/__init__.py,sha256=fw67xxwnizBurScMa-_zWb94lo5gamEVRt27V4bR0jc,54
 thds/mops/pure/adls/_files.py,sha256=9m35Y4elWF0DjgAXVp4oi5CaY6fXWt8n67PilWxWJns,821
@@ -71,8 +71,9 @@ thds/mops/pure/core/lock/maintain.py,sha256=2VkqKxyp0bZkfM5wZV4Lz7zmZl7t5TOcCGNY
 thds/mops/pure/core/lock/read.py,sha256=Ct5eYMlkTlEaV5Yhw6HWsDD7VrgdhDZoI6AVIQ0ts-4,1255
 thds/mops/pure/core/lock/types.py,sha256=f32t_e2svMOXUVzcnLkEizw6Q47g3HPQsyAkGT2OKMs,993
 thds/mops/pure/core/lock/write.py,sha256=4z3W9rsRIs5ZI-_g2Q6ZplQdez6DxCGJ-HZikQI3dHo,5614
-thds/mops/pure/core/memo/__init__.py,sha256=7h0BHoHqCj-u8lNtPMgrkocvUe7WVq5gIM3djYIz848,217
-thds/mops/pure/core/memo/function_memospace.py,sha256=36mOPybkj8vlGzo6pxDQCS8ZxccHWoNAcBjhBR8wrRA,11463
+thds/mops/pure/core/memo/__init__.py,sha256=OAgSWsup07EKxITr3yjwJ8eXbhU6-P1DVeZaYIgylgc,277
+thds/mops/pure/core/memo/calls.py,sha256=kvm6kn-CbOLxZuo86BvzEJw69p7VlEJ8_mCiWd6uz-g,3631
+thds/mops/pure/core/memo/function_memospace.py,sha256=ooUoqqCDwsqE_Ni25SUW0iTx57kGiWMcewn1F6vYTEI,11620
 thds/mops/pure/core/memo/keyfunc.py,sha256=FAOEDzMcQ-0JvW4j1eaUzixnemo_373V-16kWZl7_i0,2053
 thds/mops/pure/core/memo/overwrite_params.py,sha256=ltuFxhr8gNo2iBoBz2eFPayjSV23gMdBuoLZD42lIAg,2425
 thds/mops/pure/core/memo/results.py,sha256=272pbb5jLqP0KZ5YL5fNTxl2sP2zALFHQs8du8jCfFo,3143
@@ -81,14 +82,14 @@ thds/mops/pure/joblib/__init__.py,sha256=-3hSs-GsNzE_eNnwrdZBHAR_eaub5Uyl5GPYqBw
 thds/mops/pure/joblib/backend.py,sha256=F__6lrdc1-VcX4n4Pw7Lz1bBgeefShtRy2DQh6Fp-eI,2671
 thds/mops/pure/joblib/batching.py,sha256=tPOATD28-YW7KcWa3IqKm-fhLaILzM792ApvU-_zfnM,2298
 thds/mops/pure/pickling/__init__.py,sha256=WNdG8PdJCk-kYaXkvvPa--hjYGoUlBXG3w2X86yuhGo,156
-thds/mops/pure/pickling/_pickle.py,sha256=91ACcVRK3_DwOuZhqeqjphBFvYdn0iGm2thgrRgpXvg,7347
+thds/mops/pure/pickling/_pickle.py,sha256=oBt2LX3_Bm33lFmQiOdQq0zIdPjDmPY8je2ICWgUQbo,7514
 thds/mops/pure/pickling/memoize_only.py,sha256=oI5CMy6IEJc46Gb_BGWNUuAe3fysS7HxRSTajN0WssI,837
-thds/mops/pure/pickling/mprunner.py,sha256=D-nChnp7ebRfwtdM8oZytbsCLfwOFp9skDrHWFAqaAs,7388
+thds/mops/pure/pickling/mprunner.py,sha256=dVbwQA8hzEL7UiwYXmzoGwN3_jbEtGoHDPMkRmo_UtA,8378
 thds/mops/pure/pickling/pickles.py,sha256=nCg7L7CqReNWDF8FAdEmCcuXVC_kLT5zuyW3V8Vvvs4,4704
 thds/mops/pure/pickling/remote.py,sha256=XMR4-DdhxhBgbskmXCC2G9BEXJ15j6i_Rcg_-JQi_W8,6025
 thds/mops/pure/pickling/sha256_b64.py,sha256=HL0cPixHPZYuZDVDBscxsnI-3a2amWEfw-LseOX-PyY,2916
 thds/mops/pure/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-thds/mops/pure/runner/local.py,sha256=eq2Z99oT--fm0e6toL7gFKvmzHUsQy9oFtiW1w38o0U,11678
+thds/mops/pure/runner/local.py,sha256=qnA8d4j6cVfSqPvc_LwaPTWX-GucezGpULlE-ft0nOA,11958
 thds/mops/pure/runner/shim_builder.py,sha256=DkOXbPaOWPj2uUsJhjlWmh8ijG9OQc4ciHqa-vHPfXw,709
 thds/mops/pure/runner/simple_shims.py,sha256=oJ8sC5EVD-JFZx8CYE3_QwaQTuFa5F3IYH5PJ9mdMtY,702
 thds/mops/pure/runner/strings.py,sha256=PYAYMxZ2ehgahKIBXJilENNE6OrdNkueNBel8LPsoh8,26
@@ -103,8 +104,8 @@ thds/mops/pure/tools/summarize/cli.py,sha256=gaechsJhRZsOxGJGG1dQsW5dMBlgSv2sUmE
 thds/mops/pure/tools/summarize/run_summary.py,sha256=ujJC24J0XsF5W5P-eHiIq-4gmedmFXk2g1uljuvqOvc,5373
 thds/mops/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/testing/deferred_imports.py,sha256=f0ezCgQAtzTqW1yAOb0OWgsB9ZrlztLB894LtpWDaVw,3780
-thds_mops-3.7.20250424212724.dist-info/METADATA,sha256=2O_smajVfBbKK79ksUfWq_FwirszWtefasw6P_LdX0c,2158
-thds_mops-3.7.20250424212724.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-thds_mops-3.7.20250424212724.dist-info/entry_points.txt,sha256=GShNqjcjbq0TAJuwpyeCI5XCltiwdZxnNHkBpmYbNkU,329
-thds_mops-3.7.20250424212724.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_mops-3.7.20250424212724.dist-info/RECORD,,
+thds_mops-3.8.20250425182948.dist-info/METADATA,sha256=YXR7rice7lax6IE10hBe1NY8qQoCS_aqe3G4vgYDCKM,2158
+thds_mops-3.8.20250425182948.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+thds_mops-3.8.20250425182948.dist-info/entry_points.txt,sha256=GShNqjcjbq0TAJuwpyeCI5XCltiwdZxnNHkBpmYbNkU,329
+thds_mops-3.8.20250425182948.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_mops-3.8.20250425182948.dist-info/RECORD,,