thds.mops 3.9.20251021234412__py3-none-any.whl → 3.9.20251023163418__py3-none-any.whl

thds/mops/_utils/config_tree.py

@@ -154,6 +154,10 @@ class ConfigTree(ty.Generic[V]):
     def __setitem__(self, key: str, value: V) -> None:
         self.setv(value, pathable=key)
 
+    def __contains__(self, key: str) -> bool:
+        """Only answers the specific question - does this exact key exist in the config?"""
+        return key in self.registry
+
     def load_config(self, config: ty.Mapping[str, ty.Any]) -> None:
         """Loads things with an inner key matching this name into the config.
 

thds/mops/pure/_magic/api.py

@@ -50,6 +50,11 @@ class _MagicApi:
         pipeline_id: str = "",
         calls: ty.Collection[ty.Callable] = tuple(),
     ) -> ty.Callable[[ty.Callable[P, R]], sauce.Magic[P, R]]:
+        """This is the main pure.magic() decorator.  It is designed to be applied directly
+        at the site of function definition, i.e. on the `def`.  We dynamically capture
+        the fully qualified name of the function being decorated and use that to
+        look up the appropriate 'magic' configuration at the time of each call to the function.
+        """
         return sauce.make_magic(_get_config(), shim_or_builder, blob_root, pipeline_id, calls)
 
     @staticmethod
@@ -58,10 +63,29 @@ class _MagicApi:
         *,
         blob_root: uris.UriResolvable = "",
         pipeline_id: str = "",
+        config_path: str = "",
     ) -> ty.Callable[[F], F]:  # cleaner type for certain use cases
+        """This alternative API is designed for more dynamic use cases - rather than
+        decorating a function def directly, you can use this to create a more generic
+        decorator that can be applied within other code (not at module-level).
+
+        However, you must never apply pure.magic.deco to the same function from multiple
+        places, as this means that you have multiple different uses sharing the same
+        configuration path, which will lead to subtle bugs.
+
+        We attempt to detect this and raise an error if it happens. If it does, you should
+        provide an explicit unique config_path for each usage.
+        """
         return ty.cast(
             ty.Callable[[F], F],
-            _MagicApi.__call__(shim_or_builder, blob_root=blob_root, pipeline_id=pipeline_id),
+            sauce.make_magic(
+                _get_config(),
+                shim_or_builder,
+                blob_root=blob_root,
+                pipeline_id=pipeline_id,
+                calls=tuple(),
+                config_path=config_path,
+            ),
         )
 
     @staticmethod
@@ -122,6 +146,10 @@ class _MagicApi:
         m_config.blob_root.load_config(all_config)
         m_config.pipeline_id.load_config(all_config)
 
+    @staticmethod
+    def config_path(func: ty.Callable) -> str:
+        return sauce.make_magic_config_path(func)
+
 
 magic: ty.Final = _MagicApi()
 # we only instantiate this so we can have a call to magic() that is not __init__.

thds/mops/pure/_magic/sauce.py

@@ -6,7 +6,7 @@ import typing as ty
 
 from typing_extensions import ParamSpec
 
-from thds.core import futures, stack_context
+from thds.core import futures, log, stack_context
 from thds.mops._utils import config_tree
 
 from ..core import file_blob_store, pipeline_id, pipeline_id_mask, uris
@@ -38,6 +38,8 @@ class _MagicConfig:
         self.shim_bld[""] = make_builder(samethread_shim)  # default Shim
         self.pipeline_id[""] = "magic"  # default pipeline_id
 
+        self.all_registered_paths: set[str] = set()
+
     def __repr__(self) -> str:
         return f"MagicConfig(shim_bld={self.shim_bld}, blob_root={self.blob_root}, pipeline_id={self.pipeline_id})"
 
@@ -68,15 +70,17 @@ class Magic(ty.Generic[P, R]):
         self,
         func: ty.Callable[P, R],
         config: _MagicConfig,
+        magic_config_path: str,
         calls: ty.Collection[ty.Callable] = frozenset(),
     ):
         functools.update_wrapper(self, func)
-        self._func_config_path = full_name_and_callable(func)[0].replace("--", ".")
+        self._magic_config_path = magic_config_path
 
         self.config = config
+
         if p_id := pipeline_id_mask.extract_from_docstr(func, require=False):
             # this allows the docstring pipeline id to become 'the most specific' config.
-            self.config.pipeline_id.setv(p_id, self._func_config_path)
+            self.config.pipeline_id.setv(p_id, self._magic_config_path)
         self._shim = stack_context.StackContext[ty.Union[None, ShimName, ShimOrBuilder]](
             str(func) + "_SHIM", None  # none means nothing has been set stack-local
         )
@@ -104,7 +108,7 @@ class Magic(ty.Generic[P, R]):
     def _shim_builder_or_off(self) -> ty.Optional[ShimBuilder]:
         if stack_local_shim := self._shim():
             return to_shim_builder(stack_local_shim)
-        return self.config.shim_bld.getv(self._func_config_path)
+        return self.config.shim_bld.getv(self._magic_config_path)
 
     def _is_off(self) -> bool:
         return self._shim_builder_or_off is None
@@ -117,11 +121,11 @@ class Magic(ty.Generic[P, R]):
         return sb(f, args, kwargs)
 
     def _get_blob_root(self) -> str:
-        return self.config.blob_root.getv(self._func_config_path)()
+        return self.config.blob_root.getv(self._magic_config_path)()
 
     @property
     def _pipeline_id(self) -> str:
-        return self.config.pipeline_id.getv(self._func_config_path)
+        return self.config.pipeline_id.getv(self._magic_config_path)
 
     def submit(self, *args: P.args, **kwargs: P.kwargs) -> futures.PFuture[R]:
         """A futures-based interface that doesn't block on the result of the wrapped
@@ -138,26 +142,80 @@ class Magic(ty.Generic[P, R]):
 
     def __repr__(self) -> str:
         return (
-            f"Magic('{self._func_config_path}', shim={self._shim_builder_or_off},"
+            f"Magic('{self._magic_config_path}', shim={self._shim_builder_or_off},"
             f" blob_root='{self._get_blob_root()}', pipeline_id='{self._pipeline_id}')"
         )
 
 
+def make_magic_config_path(func: ty.Callable) -> str:
+    return full_name_and_callable(func)[0].replace("--", ".")
+
+
+class MagicReregistrationError(ValueError):
+    pass
+
+
 def make_magic(
     config: _MagicConfig,
     shim_or_builder: ty.Union[ShimName, ShimOrBuilder, None],
     blob_root: uris.UriResolvable,
     pipeline_id: str,
     calls: ty.Collection[ty.Callable],
+    *,
+    config_path: str = "",
 ) -> ty.Callable[[ty.Callable[P, R]], Magic[P, R]]:
+    """config_path is a dot-separated path that must be unique throughout your application.
+
+    By default it will be set to the thds.other.module.function_name of the decorated function.
+    """
+    error_logger = log.auto(__name__, "thds.mops.pure._magic.api").error
+    err_msg = (
+        "You are probably using pure.magic(.deco) from multiple places on the same function. You will need to specify a unique config_path for each usage."
+        if not config_path
+        else f"You supplied a config_path ({config_path}) but you reused the decorator on different functions with the same config_path."
+    )
+    err_msg += " See the comment in mops.pure._magic.sauce for more details."
+
+    def must_not_remagic_same_func(msg: str) -> None:
+        error_logger(f"{msg}; {err_msg}")
+        # if you see either of the above messages, consider whether you really need the magic
+        # configurability of pure.magic, or whether it might be better to instantiate and use
+        # MemoizingPicklingRunner directly without configurability. The reason overwriting
+        # configs, by applying pure.magic to the same callable from more than one location is
+        # disallowed is that you will get 'spooky action at a distance' between different parts
+        # of your application that are overwriting the base config for the same function.
+        # Another approach would be to use a wrapper `def` with a static @pure.magic decorator
+        # on it that calls the inner function, so that they are completely different functions
+        # as far as pure.magic is concerned.
+        raise MagicReregistrationError(msg)
+
+    magic_config_path_cache: set[str] = set()
+    # the reason for this cache is that there are cases where you may want to apply the _exact
+    # same_ base config to the same function multiple times - just for ease of use. And
+    # since this is the exact same config, we should allow it and treat it as though you
+    # had only applied it once.  Of course, if you later try to configure these
+    # applications separately, it won't work - these _are_ the same magic config path, so
+    # they're bound together via that config.
+
     def deco(func: ty.Callable[P, R]) -> Magic[P, R]:
-        fully_qualified_name = full_name_and_callable(func)[0].replace("--", ".")
+        fully_qualified_name = make_magic_config_path(func)
+        magic_config_path = config_path or fully_qualified_name
+
+        def deco_being_reapplied_to_same_func() -> bool:
+            return fully_qualified_name in magic_config_path_cache
+
+        if magic_config_path in config.all_registered_paths and not deco_being_reapplied_to_same_func():
+            must_not_remagic_same_func(f"Cannot re-register {magic_config_path} using pure.magic")
+
         if shim_or_builder is not None:
-            config.shim_bld[fully_qualified_name] = to_shim_builder(shim_or_builder)
+            config.shim_bld[magic_config_path] = to_shim_builder(shim_or_builder)
         if blob_root:  # could be empty string
-            config.blob_root[fully_qualified_name] = uris.to_lazy_uri(blob_root)
+            config.blob_root[magic_config_path] = uris.to_lazy_uri(blob_root)
         if pipeline_id:  # could be empty string
-            config.pipeline_id[fully_qualified_name] = pipeline_id
-        return Magic(func, config, calls)
+            config.pipeline_id[magic_config_path] = pipeline_id
+
+        magic_config_path_cache.add(fully_qualified_name)
+        config.all_registered_paths.add(magic_config_path)
+        return Magic(func, config, magic_config_path, calls)
 
     return deco

{thds_mops-3.9.20251021234412.dist-info → thds_mops-3.9.20251023163418.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.mops
-Version: 3.9.20251021234412
+Version: 3.9.20251023163418
 Summary: ML Ops tools for Trilliant Health
 Author-email: Trilliant Health <info@trillianthealth.com>
 Project-URL: Repository, https://github.com/TrilliantHealth/ds-monorepo

{thds_mops-3.9.20251021234412.dist-info → thds_mops-3.9.20251023163418.dist-info}/RECORD

@@ -5,7 +5,7 @@ thds/mops/config.py,sha256=T62YskXvzAfxNgpq2jMatHgoIHfRV_z4cvJ8Rl_TZ6E,2015
 thds/mops/parallel.py,sha256=F6vUhSTO--CY82vyYtWFtspmgd0RxoxQ_EUrCnTm93Q,1039
 thds/mops/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-thds/mops/_utils/config_tree.py,sha256=wMwkw81FDOo2Ld9rEmtobZMW2ZCUmxWSI2Bp_p0pwa8,7151
+thds/mops/_utils/config_tree.py,sha256=Q9mPAAolYPqKe6dkvfetg-wmVARyD7WOx123OLQ8_sU,7326
 thds/mops/_utils/exception.py,sha256=Itj6ceieCdGrKZ2JdW_DIM88Wgvvw104cfbH1RNn6Go,394
 thds/mops/_utils/locked_cache.py,sha256=ROIkwu-_FcXlNyCreWQeE5cyL9XrNW7drWsolTgeajM,2523
 thds/mops/_utils/names.py,sha256=tPPaXCyduUXqmbdvIg3ygevERnKM3YIs868BeaKX5XY,824
@@ -39,8 +39,8 @@ thds/mops/k8s/tools/krsync.py,sha256=us7pXX0-bRMwD2oAno7Z6BJcPs6FgaUabHW0STyQJYg
 thds/mops/k8s/tools/krsync.sh,sha256=fWgwkdzWnJeTbzEA_uBiIIi-bNU4nXAYj3dNovyRluU,747
 thds/mops/pure/__init__.py,sha256=3xLimQ2JWdeq1YgPs7bPwlwOspzPRwaR2w2KX7vfJU0,1624
 thds/mops/pure/_magic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-thds/mops/pure/_magic/api.py,sha256=kSlediIZQYsmeHB8plP6osjvUuSEVW4NWdY9ADia12Y,5094
-thds/mops/pure/_magic/sauce.py,sha256=xLaseOhoGgntmz6nZdj2te85sOaOiWExBLMGdeNmKsI,6870
+thds/mops/pure/_magic/api.py,sha256=eG1NEl9FR_FE1yLfN0hiLd0uLWu-gKzw1FCIJR5pMbw,6424
+thds/mops/pure/_magic/sauce.py,sha256=BfzQfEnQarcLuTbKyCKLU8F6BtJ-v6kn1SRIf675cTc,9804
 thds/mops/pure/_magic/shims.py,sha256=CXN8wlHv039oKRzDtp5YFDlwGXmmaheWLCi2I95gSeM,1212
 thds/mops/pure/adls/__init__.py,sha256=fw67xxwnizBurScMa-_zWb94lo5gamEVRt27V4bR0jc,54
 thds/mops/pure/adls/_files.py,sha256=9m35Y4elWF0DjgAXVp4oi5CaY6fXWt8n67PilWxWJns,821
@@ -109,8 +109,8 @@ thds/mops/pure/tools/summarize/cli.py,sha256=7kDtn24ok8oBO3jFjlMmOK3jnZYpMoE_5Y8
 thds/mops/pure/tools/summarize/run_summary.py,sha256=glEN_YxUGADzp2Ofvr4ZDeHvnZ1znNR7HD7EATn1sPI,5644
 thds/mops/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/testing/deferred_imports.py,sha256=f0ezCgQAtzTqW1yAOb0OWgsB9ZrlztLB894LtpWDaVw,3780
-thds_mops-3.9.20251021234412.dist-info/METADATA,sha256=QTTSJBqR1RAdWViv4J397KtVvEYHHja7hhImWPkVqnM,2225
-thds_mops-3.9.20251021234412.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_mops-3.9.20251021234412.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
-thds_mops-3.9.20251021234412.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_mops-3.9.20251021234412.dist-info/RECORD,,
+thds_mops-3.9.20251023163418.dist-info/METADATA,sha256=tazNjfQKEr0nFkGcn5FarA96_iLdpUxywauSIBV6vlk,2225
+thds_mops-3.9.20251023163418.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_mops-3.9.20251023163418.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
+thds_mops-3.9.20251023163418.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_mops-3.9.20251023163418.dist-info/RECORD,,