thds.mops 3.6.20250219172032__py3-none-any.whl

thds/mops/parallel.py

@@ -0,0 +1,36 @@
+import concurrent
+import typing as ty
+
+from thds.core import parallel
+from thds.core.parallel import (  # noqa: F401; for backward-compatibility, since these came from here originally.
+    IterableWithLen,
+    IteratorWithLen,
+)
+from thds.core.thunks import (  # noqa: F401; for backward-compatibility, since these came from here originally.
+    Thunk,
+    thunking,
+)
+
+from ._utils.colorize import colorized
+
+ERROR = colorized(fg="white", bg="red")
+DONE = colorized(fg="white", bg="blue")
+R = ty.TypeVar("R")
+
+
+def parallel_yield_results(
+    thunks: ty.Iterable[ty.Callable[[], R]],
+    *,
+    executor_cm: ty.Optional[ty.ContextManager[concurrent.futures.Executor]] = None,
+    named: str = "",
+) -> ty.Iterator[R]:
+    yield from parallel.yield_results(
+        thunks,
+        executor_cm=executor_cm,
+        error_fmt=ERROR,
+        success_fmt=DONE,
+        named=named,
+    )
+
+
+yield_results = parallel_yield_results

thds/mops/pure/__init__.py

@@ -0,0 +1,43 @@
+# This module is the supported interface and everything not exported here is subject to
+# backward-incompatible change without notice.
+#
+# The single exception is the joblib module, which is not exported by default
+# to avoid requiring the additional dependency.
+
+from . import adls  # noqa
+from ._magic.api import magic  # noqa
+from .core.entry import register_entry_handler
+from .core.memo import results  # noqa
+from .core.memo.function_memospace import (  # noqa
+    add_pipeline_memospace_handlers,
+    matching_mask_pipeline_id,
+)
+from .core.pipeline_id import get_pipeline_id, set_pipeline_id  # noqa
+from .core.pipeline_id_mask import pipeline_id_mask  # noqa
+from .core.source import create_source_at_uri  # noqa
+from .core.types import Args, BlobStore, Kwargs, Runner  # noqa
+from .core.uris import UriIsh, UriResolvable, register_blob_store  # noqa
+from .core.use_runner import use_runner  # noqa
+from .pickling.memoize_only import memoize_in  # noqa
+from .pickling.mprunner import MemoizingPicklingRunner  # noqa
+from .runner.simple_shims import samethread_shim, subprocess_shim  # noqa
+from .runner.types import Shim, ShimBuilder  # noqa
+
+
+def _register_things() -> None:
+    from . import pickling
+    from .core.uris import load_plugin_blobstores
+
+    register_entry_handler(
+        pickling.mprunner.RUNNER_NAME,
+        pickling.remote.run_pickled_invocation,  # type: ignore
+    )
+
+    load_plugin_blobstores()
+
+
+_register_things()
+
+
+Shell = Shim  # deprecated alias
+ShellBuilder = ShimBuilder  # deprecated alias

thds/mops/pure/_magic/api.py

@@ -0,0 +1,114 @@
+"""Magic is an attempt at a new interface for mops designed to make it even less sticky
+and easier to get things done with.
+
+It's designed to combine the most common workflows into a single wrapper function
+requiring an absolute minimum of boilerplate/config.
+
+Unlike the more open-ended interface of use_runner plus BYO Runner, this one assumes
+MemoizingPicklingRunner, and the most likely non-default config will be a runtime Shim or
+ShimBuilder.  If you don't supply one, it will default to the same-thread shim.
+"""
+
+import typing as ty
+from pathlib import Path
+
+from thds import core
+from thds.mops import config
+from thds.mops._utils import config_tree
+
+from ..core import uris
+from ..runner.types import ShimBuilder
+from . import sauce
+from .sauce import P, R
+from .shims import ShimName, ShimOrBuilder, to_shim_builder
+
+_MAGIC_CONFIG: ty.Final = sauce.new_config()
+
+
+def _get_config() -> sauce._MagicConfig:  # for testing
+    return _MAGIC_CONFIG
+
+
+class _MagicApi:
+    """The public API for pure.magic.
+
+    Each of these methods makes a global change to your application, so they're designed
+    to be used at import time or in other situations where no functions have been called.
+
+    If you want to apply a shim, blob_root, or pipeline_id to a single function, prefer
+    the @pure.magic(shim, blob_root=your_blob_root, pipeline_id='lazing/sunday') decorator
+    approach rather than configuring them after the fact, to keep the definition as close
+    as possible to the site of use.
+    """
+
+    @staticmethod
+    def __call__(
+        shim_or_builder: ty.Union[ShimName, ShimOrBuilder, None] = None,
+        *,
+        blob_root: uris.UriResolvable = "",
+        pipeline_id: str = "",
+    ) -> ty.Callable[[ty.Callable[P, R]], sauce.Magic[P, R]]:
+        return sauce.make_magic(_get_config(), shim_or_builder, blob_root, pipeline_id)
+
+    @staticmethod
+    def blob_root(
+        blob_root_uri: uris.UriResolvable, pathable: config_tree.Pathable = None, *, mask: bool = False
+    ) -> core.config.ConfigItem[ty.Callable[[], str]]:
+        """Sets the root URI for the blob store and control files for a specific module or function."""
+        return _get_config().blob_root.setv(uris.to_lazy_uri(blob_root_uri), pathable, mask=mask)
+
+    @staticmethod
+    def shim(
+        shim: ty.Union[ShimName, ShimOrBuilder],
+        pathable: config_tree.Pathable = None,
+        *,
+        mask: bool = False,
+    ) -> core.config.ConfigItem[ty.Optional[ShimBuilder]]:
+        """Use the provided shim for everything matching the pathable,
+        unless there's a more specific path that matches.
+
+        e.g.:
+        - magic.shim('samethread') would turn off mops for everything within
+          or below the current module.
+        - magic.shim('subprocess', 'foo.bar.baz') would use the subprocess shim for
+          everything within or below the foo.bar.baz module.
+        - magic.shim(my_shim_builder, my_func) would use my_shim_builder for just my_func.
+
+        To instead _mask_ everything at this level and below regardless of more specific
+        config, pass mask=True.
+        """
+        return _get_config().shim_bld.setv(to_shim_builder(shim), pathable, mask=mask)
+
+    @staticmethod
+    def off(pathable: config_tree.Pathable = None, *, mask: bool = False) -> None:
+        """Turn off mops for everything matching the pathable.
+
+        A shortcut for shim(None).
+        """
+        _MagicApi.shim("off", pathable, mask=mask)
+
+    @staticmethod
+    def pipeline_id(
+        pipeline_id: str, pathable: config_tree.Pathable = None, *, mask: bool = False
+    ) -> core.config.ConfigItem[str]:
+        """Sets the pipeline_id for a specific module or function."""
+        return _get_config().pipeline_id.setv(pipeline_id, pathable, mask=mask)
+
+    @staticmethod
+    def load_config_file(magic_config: ty.Optional[Path] = None) -> None:
+        """Call this to load pure.magic config from the nearest .mops.toml file upward,
+        or the path you provide.
+
+        Should be called only once, in the `__main__` block of your program,
+        and after all imports are resolved.
+        """
+        all_config = config.load(magic_config or config.first_found_config_file(), name="pure.magic")
+        m_config = _get_config()
+        m_config.shim_bld.load_config(all_config)
+        m_config.blob_root.load_config(all_config)
+        m_config.pipeline_id.load_config(all_config)
+
+
+magic: ty.Final = _MagicApi()
+# we only instantiate this so we can have a call to magic() that is not __init__.
+# there is no state whatsoever in this object.

thds/mops/pure/_magic/sauce.py

@@ -0,0 +1,152 @@
+"""The magic sauce for most of what pure.magic does."""
+
+import contextlib
+import functools
+import typing as ty
+
+from typing_extensions import ParamSpec
+
+from thds.core import stack_context
+from thds.mops._utils import config_tree
+
+from ..core import file_blob_store, pipeline_id_mask, uris
+from ..core.memo.unique_name_for_function import full_name_and_callable
+from ..core.use_runner import use_runner
+from ..pickling.mprunner import MemoizingPicklingRunner
+from ..runner.shim_builder import make_builder
+from ..runner.simple_shims import samethread_shim
+from ..runner.types import Shim, ShimBuilder
+from .shims import ShimName, ShimOrBuilder, to_shim_builder
+
+_local_root = lambda: f"file://{file_blob_store.MOPS_ROOT()}"  # noqa: E731
+P = ParamSpec("P")
+R = ty.TypeVar("R")
+
+
+class _MagicConfig:
+    def __init__(self) -> None:
+        # these ConfigTree objects apply configuration to callables wrapped with pure.magic
+        # based on the fully-qualified path to the callable, e.g. foo.bar.baz.my_func
+        self.shim_bld = config_tree.ConfigTree[ty.Optional[ShimBuilder]](
+            "mops.pure.magic.shim", parse=to_shim_builder  # type: ignore
+        )
+        self.blob_root = config_tree.ConfigTree[ty.Callable[[], str]](
+            "mops.pure.magic.blob_root", parse=uris.to_lazy_uri
+        )
+        self.pipeline_id = config_tree.ConfigTree[str]("mops.pure.magic.pipeline_id")
+        self.blob_root[""] = _local_root  # default Blob Store
+        self.shim_bld[""] = make_builder(samethread_shim)  # default Shim
+        self.pipeline_id[""] = "magic"  # default pipeline_id
+
+    def __repr__(self) -> str:
+        return f"MagicConfig(shim_bld={self.shim_bld}, blob_root={self.blob_root}, pipeline_id={self.pipeline_id})"
+
+
+def new_config() -> _MagicConfig:
+    return _MagicConfig()
+
+
+class Magic(ty.Generic[P, R]):
+    """Magic adds mops' powers (memoization, coordination, remote execution) to a callable.
+
+    If you want to _change_ which runtime shim the function is using, that can be set globally
+    to the program with pure.magic.shim(other_shim, my_magic_func), and it can also be set
+    as a stack-local variable in a context manager provided by this object:
+
+    with my_magic_func.shim("subprocess"):
+        my_magic_func(1, 2, 3)
+
+    You can completely disable mops magic for a function in the same ways, either with a contextmanager
+    or globally, using `off()`, like so:
+
+    with my_magic_func.off():
+        ...
+        my_magic_func(1, 2, 3)
+    """
+
+    def __init__(
+        self,
+        func: ty.Callable[P, R],
+        config: _MagicConfig,
+    ):
+        functools.update_wrapper(self, func)
+        self._func_config_path = full_name_and_callable(func)[0].replace("--", ".")
+
+        self.config = config
+        if p_id := pipeline_id_mask.extract_from_docstr(func, require=False):
+            # this allows the docstring pipeline id to become 'the most specific' config.
+            self.config.pipeline_id.setv(p_id, self._func_config_path)
+        self._shim = stack_context.StackContext[ty.Union[None, ShimName, ShimOrBuilder]](
+            str(func) + "_SHIM", None  # none means nothing has been set stack-local
+        )
+        self.runner = MemoizingPicklingRunner(self._shimbuilder, self._get_blob_root)
+        self._func = use_runner(self.runner, self._is_off)(func)
+        self.__doc__ = f"{func.__doc__}\n\nMagic class info:\n{self.__class__.__doc__}"
+        self.__wrapped__ = func
+
+    @contextlib.contextmanager
+    def shim(self, shim_or_builder: ty.Union[None, ShimName, ShimOrBuilder]) -> ty.Iterator[None]:
+        """If None is passed, no change will be made."""
+        with self._shim.set(shim_or_builder or self._shim()):
+            yield
+
+    @contextlib.contextmanager
+    def off(self) -> ty.Iterator[None]:
+        """off is an API for setting the shim to None,
+        effectively turning off mops for the wrapped function.
+        """
+        with self.shim("off"):
+            yield
+
+    @property
+    def _shim_builder_or_off(self) -> ty.Optional[ShimBuilder]:
+        if stack_local_shim := self._shim():
+            return to_shim_builder(stack_local_shim)
+        return self.config.shim_bld.getv(self._func_config_path)
+
+    def _is_off(self) -> bool:
+        return self._shim_builder_or_off is None
+
+    def _shimbuilder(self, f: ty.Callable[P, R], args: P.args, kwargs: P.kwargs) -> Shim:
+        # this can be set using a stack-local context, or set globally as specifically
+        # or generally as the user needs. We prefer stack local over everything else.
+        sb = self._shim_builder_or_off
+        assert sb is not None, "This should have been handled by use_runner(self._off)"
+        return sb(f, args, kwargs)
+
+    def _get_blob_root(self) -> str:
+        return self.config.blob_root.getv(self._func_config_path)()
+
+    @property
+    def _pipeline_id(self) -> str:
+        return self.config.pipeline_id.getv(self._func_config_path)
+
+    def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R:
+        """This is the wrapped function."""
+        with pipeline_id_mask.pipeline_id_mask(self._pipeline_id):
+            return self._func(*args, **kwargs)
+
+    def __repr__(self) -> str:
+        return (
+            f"Magic('{self._func_config_path}', shim={self._shim_builder_or_off},"
+            f" blob_root='{self._get_blob_root()}', pipeline_id='{self._pipeline_id}')"
+        )
+
+
+def make_magic(
+    config: _MagicConfig,
+    shim_or_builder: ty.Union[ShimName, ShimOrBuilder, None],
+    blob_root: uris.UriResolvable,
+    pipeline_id: str,
+) -> ty.Callable[[ty.Callable[P, R]], Magic[P, R]]:
+    def deco(func: ty.Callable[P, R]) -> Magic[P, R]:
+        fully_qualified_name = full_name_and_callable(func)[0].replace("--", ".")
+        if shim_or_builder is not None:
+            config.shim_bld[fully_qualified_name] = to_shim_builder(shim_or_builder)
+        if blob_root:  # could be empty string
+            config.blob_root[fully_qualified_name] = uris.to_lazy_uri(blob_root)
+        if pipeline_id:  # could be empty string
+            config.pipeline_id[fully_qualified_name] = pipeline_id
+        return Magic(func, config)
+
+    return deco

thds/mops/pure/_magic/shims.py

@@ -0,0 +1,34 @@
+import typing as ty
+
+from thds import core
+
+from ..runner.shim_builder import make_builder
+from ..runner.simple_shims import samethread_shim, subprocess_shim
+from ..runner.types import Shim, ShimBuilder
+
+ShimName = ty.Literal[
+    "samethread",  # memoization and coordination, but run in the same thread as the caller.
+    "subprocess",  # memoization and coordination, but transfer to a subprocess rather than remote.
+    "off",  # equivalent to None - disables use of mops.
+]
+ShimOrBuilder = ty.Union[ShimBuilder, Shim]
+logger = core.log.getLogger(__name__)
+
+
+def _shim_name_to_builder(shim_name: ShimName) -> ty.Optional[ShimBuilder]:
+    if shim_name == "samethread":
+        return make_builder(samethread_shim)
+    if shim_name == "subprocess":
+        return make_builder(subprocess_shim)
+    if shim_name == "off":
+        return None
+    logger.warning("Unrecognized shim name: %s; mops will be turned off.", shim_name)
+    return None
+
+
+def to_shim_builder(shim: ty.Union[None, ShimName, ShimOrBuilder]) -> ty.Optional[ShimBuilder]:
+    if shim is None:
+        return None
+    if isinstance(shim, str):
+        return _shim_name_to_builder(shim)
+    return make_builder(shim)

thds/mops/pure/adls/__init__.py

@@ -0,0 +1 @@
+from .output_fqn import invocation_output_fqn  # noqa

thds/mops/pure/adls/_files.py

@@ -0,0 +1,22 @@
+import typing as ty
+
+import azure.core.exceptions
+from azure.storage.filedatalake import FileSystemClient
+
+
+def yield_files(fsc: FileSystemClient, adls_root: str) -> ty.Iterable[ty.Any]:
+    """Yield files (including directories) from the root."""
+    with fsc as client:
+        try:
+            yield from client.get_paths(adls_root)
+        except azure.core.exceptions.ResourceNotFoundError as rnfe:
+            if rnfe.response and rnfe.response.status_code == 404:
+                return  # no paths
+            raise
+
+
+def yield_filenames(fsc: FileSystemClient, adls_root: str) -> ty.Iterable[str]:
+    """Yield only real file (not directory) names recursively from the root."""
+    for azure_file in yield_files(fsc, adls_root):
+        if not azure_file.get("is_directory"):
+            yield azure_file["name"]

thds/mops/pure/adls/blob_store.py

@@ -0,0 +1,185 @@
+"""This abstraction matches what is required by the BlobStore abstraction in pure.core.uris"""
+
+import logging
+import typing as ty
+from pathlib import Path
+
+from azure.core.exceptions import HttpResponseError
+from azure.storage.filedatalake import DataLakeFileClient
+
+from thds.adls import ADLS_SCHEME, AdlsFqn, download, join, resource, ro_cache
+from thds.adls.cached_up_down import download_to_cache, upload_through_cache
+from thds.adls.errors import blob_not_found_translation, is_blob_not_found
+from thds.adls.global_client import get_global_fs_client
+from thds.core import config, fretry, home, link, log, scope
+
+from ..._utils.on_slow import LogSlow, on_slow
+from ..core.types import DISABLE_CONTROL_CACHE, AnyStrSrc, BlobStore
+
+T = ty.TypeVar("T")
+ToBytes = ty.Callable[[T, ty.BinaryIO], ty.Any]
+FromBytes = ty.Callable[[ty.BinaryIO], T]
+_5_MB = 5 * 2**20
+
+
+# suppress very noisy INFO logs in azure library.
+# This mirrors thds.adls.
+log.getLogger("azure.core").setLevel(logging.WARNING)
+logger = log.getLogger(__name__)
+
+
+def _selective_upload_path(path: Path, fqn: AdlsFqn) -> None:
+    if path.stat().st_size > _5_MB:
+        upload_through_cache(fqn, path)
+    else:
+        resource.upload(fqn, path)
+
+
+def is_creds_failure(exc: Exception) -> bool:
+    return isinstance(exc, HttpResponseError) and not is_blob_not_found(exc)
+
+
+_azure_creds_retry = fretry.retry_sleep(is_creds_failure, fretry.expo(retries=9, delay=1.0))
+# sometimes Azure Cli credentials expire but would succeed if retried
+# and the azure library does not seem to retry these on its own.
+
+
+class AdlsBlobStore(BlobStore):
+    def control_root(self, uri: str) -> str:
+        return str(AdlsFqn.parse(uri).root())
+
+    def _client(self, fqn: AdlsFqn) -> DataLakeFileClient:
+        return get_global_fs_client(fqn.sa, fqn.container).get_file_client(fqn.path)
+
+    @_azure_creds_retry
+    @scope.bound
+    def readbytesinto(self, remote_uri: str, stream: ty.IO[bytes], type_hint: str = "bytes") -> None:
+        fqn = AdlsFqn.parse(remote_uri)
+        scope.enter(log.logger_context(download=fqn))
+        logger.debug(f"<----- downloading {type_hint}")
+        with blob_not_found_translation(fqn):
+            on_slow(
+                lambda elapsed_s: LogSlow(f"Took {int(elapsed_s)}s to download {type_hint}"),
+            )(lambda: self._client(fqn).download_file().readinto(stream))()
+
+    @_azure_creds_retry
+    @scope.bound
+    def getfile(self, remote_uri: str) -> Path:
+        scope.enter(log.logger_context(download="mops-getfile"))
+        return download_to_cache(AdlsFqn.parse(remote_uri))
+
+    @_azure_creds_retry
+    @scope.bound
+    def putbytes(
+        self, remote_uri: str, data: AnyStrSrc, type_hint: str = "application/octet-stream"
+    ) -> None:
+        """Upload data to a remote path."""
+        resource.upload(AdlsFqn.parse(remote_uri), data, content_type=type_hint)
+
+    @_azure_creds_retry
+    @scope.bound
+    def putfile(self, path: Path, remote_uri: str) -> None:
+        scope.enter(log.logger_context(upload="mops-putfile"))
+        _selective_upload_path(path, AdlsFqn.parse(remote_uri))
+
+    @_azure_creds_retry
+    @scope.bound
+    def exists(self, remote_uri: str) -> bool:
+        fqn = AdlsFqn.parse(remote_uri)
+        scope.enter(log.logger_context(exists=fqn))
+        return on_slow(
+            lambda secs: LogSlow(f"Took {int(secs)}s to check if file exists."),
+            slow_seconds=1.2,
+        )(lambda: self._client(fqn).exists())()
+
+    def join(self, *parts: str) -> str:
+        return join(*parts).rstrip("/")
+
+    def split(self, uri: str) -> ty.List[str]:
+        fqn = AdlsFqn.parse(uri)
+        return [str(fqn.root()), *fqn.path.split("/")]
+
+    def is_blob_not_found(self, exc: Exception) -> bool:
+        return is_blob_not_found(exc)
+
+    def list(self, uri: str) -> ty.List[str]:
+        fqn = AdlsFqn.parse(uri)
+        return [
+            str(AdlsFqn(fqn.sa, fqn.container, path.name))
+            for path in get_global_fs_client(fqn.sa, fqn.container).get_paths(fqn.path, recursive=False)
+        ]
+
+
+class DangerouslyCachingStore(AdlsBlobStore):
+    """This BlobStore will cache _everything_ locally
+    and anything it finds locally it will return without question.
+
+    This maximally avoids network operations if for some reason you feel the need
+    to do that, but it will 100% lead to false positive cache hits, because it is no longer
+    checking the hash of the locally-cached file against what ADLS itself advertises.
+
+    It is now believed that this is not as dangerous as originally thought, because mops
+    control files are not intended to be mutable, and if mutated in some kind of
+    distributed systems context (e.g. two parallel runs of the same thing), the results
+    are intended to be at least 'equivalent' even when they're not byte-identical.
+
+    Therefore, this is now enabled by default and considered generally safe to use in any
+    environment, including automated ones.
+    """
+
+    def __init__(self, root: Path):
+        self._cache = ro_cache.Cache(root.resolve(), ("ref", "hard"))
+
+    def exists(self, remote_uri: str) -> bool:
+        cache_path = self._cache.path(AdlsFqn.parse(remote_uri))
+        if cache_path.exists():
+            return True
+        return super().exists(remote_uri)
+
+    def readbytesinto(self, remote_uri: str, stream: ty.IO[bytes], type_hint: str = "bytes") -> None:
+        # readbytesinto is used for _almost_ everything in mops - but almost everything is a 'control file'
+        # of some sort. We use a completely separate cache for all of these things, because
+        # in previous implementations, none of these things would have been cached at all.
+        # (see comment on getfile below...)
+        fqn = AdlsFqn.parse(remote_uri)
+        cache_path = self._cache.path(fqn)
+        if not cache_path.exists():
+            download.download_or_use_verified(
+                get_global_fs_client(fqn.sa, fqn.container), fqn.path, cache_path, cache=self._cache
+            )
+        with cache_path.open("rb") as f:
+            stream.write(f.read())
+
+    def getfile(self, remote_uri: str) -> Path:
+        # (continued from comment on readbytesinto...)
+        #
+        # whereas, for getfile, it is really only used for optimizations on larger file
+        # downloads (e.g. Paths), and those were previously subject to long-term caching.
+        # So for getfile, our primary source will be the parent implementation of getfile,
+        # including any caching it already did.
+        #
+        # We still dangerously short-circuit the hash check, and we make a 'cheap copy'
+        # (a link, usually) to our separate cache directory so that it's possible to
+        # completely empty this particular mops cache (and all its 'dangerous' behavior)
+        # simply by deleting that one cache directory.
+        cache_path = self._cache.path(AdlsFqn.parse(remote_uri))
+        if cache_path.exists():
+            return cache_path
+        outpath = super().getfile(remote_uri)
+        link.link(outpath, cache_path)
+        return outpath
+
+
+_DEFAULT_CONTROL_CACHE = config.item(
+    "thds.mops.pure.adls.control_cache_root", default=home.HOMEDIR() / ".mops-adls-control-cache"
+)
+
+
+def get_adls_blob_store(uri: str) -> ty.Optional[AdlsBlobStore]:
+    if not uri.startswith(ADLS_SCHEME):
+        return None
+
+    if DISABLE_CONTROL_CACHE() or not _DEFAULT_CONTROL_CACHE():
+        return AdlsBlobStore()
+
+    return DangerouslyCachingStore(_DEFAULT_CONTROL_CACHE())

thds/mops/pure/adls/output_fqn.py

@@ -0,0 +1,17 @@
+from thds.adls import AdlsFqn
+
+from ..core import uris
+from ..core.output_naming import invocation_output_uri
+
+
+def invocation_output_fqn(storage_root: uris.UriIsh = "", name: str = "") -> AdlsFqn:
+    """If your function only outputs a single blob to ADLS, you can safely
+    use this without providing a name.  However, if you have multiple outputs
+    from the same invocation, you must provide a meaningful name for each one.
+
+    As an example:
+
+    <pipeline> <function mod/name  > <your name     > <args,kwargs hash                                   >
+    nppes/2023/thds.nppes.intake:run/<name goes here>/CoastOilAsset.IVZ9KplQKlNgxQHav0jIMUS9p4Kbn3N481e0Uvs
+    """
+    return AdlsFqn.parse(invocation_output_uri(storage_root, name=name))

thds/mops/pure/core/content_addressed.py

@@ -0,0 +1,31 @@
+import typing as ty
+
+from thds import humenc
+from thds.core.hashing import Hash
+
+from .uris import active_storage_root
+
+B64_ADDRESSED = "{algo}-b64-addressed"
+# we can save on storage and simplify lots of internals if we just
+# hash all blobs and upload them to a key that is their hash.
+
+
+def storage_content_addressed(hash_str: str, algo: str, storage_root: str = "") -> str:
+    hash_namespace = B64_ADDRESSED.format(algo=algo)
+    return f"{storage_root or active_storage_root()}/{hash_namespace}/{hash_str}"
+
+
+class ContentAddressed(ty.NamedTuple):
+    bytes_uri: str
+    debug_uri: str
+
+
+def wordybin_content_addressed(
+    hash: Hash, storage_root: str = "", debug_name: str = ""
+) -> ContentAddressed:
+    """This should be used any time you have access to the raw bytes, so that we can stick
+    with the Human Base 64 format.
+    """
+    base_uri = storage_content_addressed(humenc.encode(hash.bytes), hash.algo, storage_root)
+    # corresponds with '_bytes' as used in `serialize_paths.py`
+    return ContentAddressed(f"{base_uri}/_bytes", f"{base_uri}/{debug_name}" if debug_name else "")