pipelex-api 0.75.0__tar.gz → 0.76.0__tar.gz

{pipelex_api-0.75.0 → pipelex_api-0.76.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pipelex-api
-Version: 0.75.0
+Version: 0.76.0
 Summary: Pipelex API
 Project-URL: Homepage, https://pipelex.com
 Project-URL: Repository, https://github.com/Pipelex/pipelex
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: <3.15,>=3.11
 Requires-Dist: fastapi<0.142,>=0.118.0
-Requires-Dist: pipelex[anthropic,bedrock,fal,google,google-genai,mistralai]==0.75.0
+Requires-Dist: pipelex[anthropic,bedrock,fal,google,google-genai,mistralai]==0.76.0
 Requires-Dist: pyjwt>=2.10.1
 Requires-Dist: uvicorn>=0.37.0
 Provides-Extra: dev

pipelex_api-0.76.0/pipelex_api/bundle.py

@@ -0,0 +1,433 @@
+"""Materialize a caller-supplied method bundle into a temporary library directory.
+
+A run request may carry the whole method — the `.mthds` bundle plus its PipeFunc
+Python (`pipe_func.py`) and a `requirements.txt` — instead of only
+the inline `mthds_contents` text. Two transport forms are accepted, exactly one
+per request:
+
+  - `bundle_b64`: a base64-encoded zip archive of the bundle directory.
+  - `files`: a `{relative_path: text_content}` map (the zip's contents, unzipped).
+
+The two are equivalent: `files` ≡ the zip's entries. This module decodes either
+form, enforces the ingest guards (both transport forms are refused together; a
+hard file-count and total-size ceiling; per-entry path-safety against absolute
+paths and `..` traversal; a zip-bomb guard that bounds actual decompression),
+writes the surviving files into a fresh temp directory, and hands that directory
+back so the runner can load it via `library_dirs`.
+
+A bundle may also ship the method packages it calls by address, each under
+`.mthds/methods/<name>/` at its root with its `METHODS.toml` there, the layout the
+standard names the local method cache. `partition_bundle_entries` sorts the entries
+into the bundle's own `.mthds` files, its other files, and those packages, refusing
+every other `.mthds` path with a `422 InvalidBundle` naming the entry, and two
+packages declaring the same address with one naming both directories, and
+`materialized_methods_dir` writes the packages into a temp directory of their own,
+`<tmp>/<name>/…`, which the runner hands the engine as `methods_dirs`. In a sandbox-hosted
+deployment the load path reads every `.py` as source text, never importing it:
+it refuses a bundle whose Python declares a structure class
+(`MethodStructuresRefusedError`, a 403) and captures the rest onto the crate for
+the sandbox. The caller is responsible for the hosted-mode gate.
+
+Nothing here imports or executes the bundle's Python — it only writes bytes to
+disk. The caller cleans the directory up via the `materialized_bundle` context
+manager's guaranteed teardown.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import shutil
+import tempfile
+import zipfile
+from contextlib import contextmanager
+from io import BytesIO
+from pathlib import Path, PurePosixPath
+from typing import TYPE_CHECKING, NamedTuple, NoReturn
+
+from mthds.package.discovery import MANIFEST_FILENAME
+from mthds.package.exceptions import ManifestError
+from mthds.package.manifest.parser import parse_methods_toml
+from pipelex import log
+from pydantic import ConfigDict
+from pydantic.dataclasses import dataclass
+
+from pipelex_api.error_types import ErrorType
+from pipelex_api.errors import raise_bad_request, raise_payload_too_large, raise_validation_error
+from pipelex_api.limits import MAX_BUNDLE_FILES, MAX_BUNDLE_TOTAL_BYTES
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+
+@dataclass(frozen=True, config=ConfigDict(arbitrary_types_allowed=True))
+class MaterializedBundle:
+    """A bundle written to disk: the directory to load and the relpaths written."""
+
+    directory: Path
+    relpaths: tuple[str, ...]
+
+    @property
+    def has_python_sources(self) -> bool:
+        """True when the bundle ships any `.py` — the trigger for the hosted-mode gate."""
+        return any(relpath.endswith(".py") for relpath in self.relpaths)
+
+
+def _safe_relpath(name: str) -> PurePosixPath:
+    """Validate one bundle entry name and return it as a normalized relative POSIX path.
+
+    Rejects anything that could escape the destination directory: absolute paths,
+    Windows drive/backslash forms (a bare drive prefix like `C:foo` has no slash or
+    backslash yet is drive-relative on Windows, so `:` is rejected outright), and any
+    `..` component. Directory-only entries (trailing slash) return an empty path and
+    are filtered by the caller.
+    """
+    if not name or name in {".", "./"}:
+        return PurePosixPath()
+    if "\\" in name:
+        msg = f"Bundle entry {name!r} uses backslashes; use forward-slash relative paths only"
+        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+    if ":" in name:
+        msg = f"Bundle entry {name!r} contains ':' (a Windows drive/stream form); use plain relative paths only"
+        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+    pure = PurePosixPath(name)
+    if pure.is_absolute():
+        msg = f"Bundle entry {name!r} is an absolute path; only relative paths are allowed"
+        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+    if any(part == ".." for part in pure.parts):
+        msg = f"Bundle entry {name!r} escapes the bundle root via '..'; not allowed"
+        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+    return pure
+
+
+def _guard_count(count: int) -> None:
+    if count == 0:
+        raise_validation_error(message="Bundle is empty (no files)", error_type=ErrorType.INVALID_BUNDLE)
+    if count > MAX_BUNDLE_FILES:
+        raise_payload_too_large(message=f"Bundle exceeds the {MAX_BUNDLE_FILES}-file limit (got {count})")
+
+
+def _guard_running_total(total_bytes: int) -> None:
+    if total_bytes > MAX_BUNDLE_TOTAL_BYTES:
+        raise_payload_too_large(message=f"Bundle exceeds the {MAX_BUNDLE_TOTAL_BYTES // 1024} KiB decompressed-size limit")
+
+
+# Base64 inflates by 4/3; a zip is compressed, so a bundle whose DECOMPRESSED content is within
+# the ceiling encodes to well under this. Bounding the base64 string BEFORE decoding stops a
+# ~100 MiB request-body (the only other bound) from being expanded into ~75 MiB of heap just to
+# be rejected later by the decompressed-size guard. Slack (+4) covers padding.
+_MAX_BUNDLE_B64_CHARS = MAX_BUNDLE_TOTAL_BYTES * 4 // 3 + 4
+
+
+def _entries_from_zip(bundle_b64: str) -> list[tuple[PurePosixPath, bytes]]:
+    """Decode a base64 zip and return its (safe relpath, bytes) file entries.
+
+    Zip-bomb guard: each member is read through a bounded stream so a lying
+    uncompressed-size header cannot force unbounded decompression — the running
+    total is checked against `MAX_BUNDLE_TOTAL_BYTES` as bytes are pulled. A cheap
+    length check on the still-encoded string runs FIRST, so an oversized payload is
+    refused before it is buffered into memory as decoded bytes.
+    """
+    if len(bundle_b64) > _MAX_BUNDLE_B64_CHARS:
+        raise_payload_too_large(message=f"bundle_b64 exceeds the {MAX_BUNDLE_TOTAL_BYTES // 1024} KiB compressed-size limit")
+    try:
+        raw = base64.b64decode(bundle_b64, validate=True)
+    except (binascii.Error, ValueError) as decode_error:
+        log.warning(f"bundle: invalid base64 ({decode_error})")
+        raise_bad_request(message="bundle_b64 is not valid base64", error_type=ErrorType.INVALID_BASE64)
+
+    try:
+        archive = zipfile.ZipFile(BytesIO(raw))
+    except zipfile.BadZipFile as zip_error:
+        log.warning(f"bundle: corrupt zip ({zip_error})")
+        raise_validation_error(message="bundle_b64 is not a valid zip archive", error_type=ErrorType.INVALID_BUNDLE)
+
+    entries: list[tuple[PurePosixPath, bytes]] = []
+    total_bytes = 0
+    with archive:
+        members = [info for info in archive.infolist() if not info.is_dir()]
+        _guard_count(len(members))
+        budget = MAX_BUNDLE_TOTAL_BYTES
+        for info in members:
+            relpath = _safe_relpath(info.filename)
+            if not relpath.parts:
+                continue
+            # Bounded read: pull at most (remaining budget + 1) bytes so a zip bomb whose header
+            # under-reports its size still cannot decompress past the ceiling.
+            with archive.open(info) as member:
+                data = member.read(budget + 1)
+            total_bytes += len(data)
+            _guard_running_total(total_bytes)
+            budget = MAX_BUNDLE_TOTAL_BYTES - total_bytes
+            entries.append((relpath, data))
+    return entries
+
+
+def _entries_from_files(files: dict[str, str]) -> list[tuple[PurePosixPath, bytes]]:
+    r"""Validate a {relpath: text} map and return its (safe relpath, bytes) entries.
+
+    JSON can carry a lone surrogate as an escape (`"\ud800"`), which has no UTF-8 form, so an entry whose name or
+    content holds one is refused with a `422` naming it rather than failing later, when it is encoded or written.
+    """
+    _guard_count(len(files))
+    entries: list[tuple[PurePosixPath, bytes]] = []
+    total_bytes = 0
+    for name, content in files.items():
+        try:
+            name.encode("utf-8")
+        except UnicodeEncodeError:
+            msg = f"Bundle entry {name!r} has a name holding a lone surrogate, which is not valid Unicode text"
+            raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+        relpath = _safe_relpath(name)
+        if not relpath.parts:
+            msg = f"Bundle entry {name!r} has no filename"
+            raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+        try:
+            data = content.encode("utf-8")
+        except UnicodeEncodeError:
+            msg = f"Bundle entry {name!r} holds a lone surrogate, which is not valid Unicode text"
+            raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+        total_bytes += len(data)
+        _guard_running_total(total_bytes)
+        entries.append((relpath, data))
+    return entries
+
+
+def _refuse_colliding_paths(entries: list[tuple[PurePosixPath, bytes]]) -> None:
+    """Refuse two entries of one path, and an entry that is also the directory of another.
+
+    Entry names are normalized (`a/./b` and `a//b` are `a/b`) and a zip may hold one member name twice, so two entries
+    can name one file: a check reading the first copy, as the shipped packages' manifest check does, would pass content
+    that differs from the last copy, which is the one written to disk and read by the engine. A file that another entry
+    is placed under could not be written at all.
+    """
+    paths: set[PurePosixPath] = set()
+    for relpath, _ in entries:
+        if relpath in paths:
+            msg = f"Bundle has more than one entry for the path '{relpath.as_posix()}'; each file may appear only once"
+            raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+        paths.add(relpath)
+    for relpath, _ in entries:
+        for parent in relpath.parents:
+            if parent in paths:
+                msg = f"Bundle entry '{parent.as_posix()}' is a file, and also the directory of the entry '{relpath.as_posix()}'"
+                raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+
+
+class ParsedBundle(NamedTuple):
+    """A decoded-and-validated bundle held in memory, NOT yet written to disk.
+
+    Splitting parse from materialize lets the caller apply the sandbox-hosted gate
+    (which keys on `has_python_sources`) BEFORE any disk write — so a bundle destined
+    for a 403 never touches the filesystem. Entries are `(safe relpath, bytes)`.
+    """
+
+    entries: tuple[tuple[PurePosixPath, bytes], ...]
+
+    @property
+    def has_python_sources(self) -> bool:
+        """True when the bundle ships any `.py` — the trigger for the hosted-mode gate."""
+        return any(str(relpath).endswith(".py") for relpath, _ in self.entries)
+
+
+def parse_bundle(*, bundle_b64: str | None, files: dict[str, str] | None) -> ParsedBundle:
+    """Decode + guard a bundle into an in-memory `ParsedBundle` (no disk writes).
+
+    Exactly one of `bundle_b64` / `files` must be supplied; supplying both is a
+    caller mistake (they are the same content in two forms) and is refused. All
+    ingest guards (base64, size, count, path-safety, zip-bomb, colliding paths) run
+    here, so the caller can inspect `has_python_sources` and reject BEFORE
+    materializing to disk.
+    """
+    if bundle_b64 is not None and files is not None:
+        msg = "Provide either bundle_b64 or files, not both"
+        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+    if bundle_b64 is not None:
+        entries = _entries_from_zip(bundle_b64)
+    elif files is not None:
+        entries = _entries_from_files(files)
+    else:
+        msg = "No bundle supplied (bundle_b64 and files are both absent)"
+        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+    _refuse_colliding_paths(entries)
+    return ParsedBundle(entries=tuple(entries))
+
+
+# Where a bundle ships the method packages it calls: `.mthds/methods/<name>/`, at the bundle's root and nowhere else.
+_MTHDS_DIR_NAME = ".mthds"
+_METHODS_DIR_NAME = "methods"
+_VENDORING_LAYOUT = (
+    "A method package shipped with the bundle goes under '.mthds/methods/<name>/' at the bundle's root, "
+    "with its manifest at '.mthds/methods/<name>/METHODS.toml'."
+)
+
+
+class VendoredMethodPackage(NamedTuple):
+    """A method package a bundle ships under `.mthds/methods/<name>/`, held in memory."""
+
+    name: str
+    """The package's directory name under `.mthds/methods/`."""
+
+    full_address: str
+    """The address its manifest declares, `address/name`, as a reference names it and a refusal names the package."""
+
+    entries: tuple[tuple[PurePosixPath, bytes], ...]
+    """Every file of the package, by its path relative to the package directory."""
+
+
+class BundlePartition(NamedTuple):
+    """A bundle's entries sorted into its own `.mthds` files, its other files, and the packages it ships."""
+
+    mthds_entries: tuple[tuple[PurePosixPath, bytes], ...]
+    library_entries: tuple[tuple[PurePosixPath, bytes], ...]
+    vendored_packages: tuple[VendoredMethodPackage, ...]
+
+
+def _refuse_entry(*, owner: str, relpath: PurePosixPath, reason: str) -> NoReturn:
+    msg = f"{owner} entry '{relpath.as_posix()}' {reason}. {_VENDORING_LAYOUT}"
+    raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+
+
+def _vendored_package(*, owner: str, name: str, entries: list[tuple[PurePosixPath, bytes]]) -> VendoredMethodPackage:
+    """Check one shipped package's directory and read the address its manifest declares."""
+    package_label = f"'.mthds/methods/{name}/'"
+    if name.startswith("."):
+        msg = f"{owner} ships {package_label}, a hidden directory, which is never searched for a package. {_VENDORING_LAYOUT}"
+        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+    manifest_bytes = next((data for relpath, data in entries if relpath == PurePosixPath(MANIFEST_FILENAME)), None)
+    if manifest_bytes is None:
+        msg = f"{owner} ships {package_label} without a {MANIFEST_FILENAME}, so no reference could find it. {_VENDORING_LAYOUT}"
+        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+    manifest_path = f".mthds/methods/{name}/{MANIFEST_FILENAME}"
+    try:
+        manifest = parse_methods_toml(manifest_bytes.decode("utf-8"))
+    except UnicodeDecodeError:
+        msg = f"{owner} entry '{manifest_path}' is not valid UTF-8. {_VENDORING_LAYOUT}"
+        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+    except ManifestError as exc:
+        msg = f"{owner} entry '{manifest_path}' is not a valid method package manifest: {exc.message} {_VENDORING_LAYOUT}"
+        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+    # The name a reference matches, as the engine's discovery gives it: the manifest's, else the directory's.
+    package_name = manifest.name if manifest.name is not None else name
+    return VendoredMethodPackage(name=name, full_address=f"{manifest.address}/{package_name}", entries=tuple(entries))
+
+
+def _refuse_shared_identities(*, owner: str, packages: tuple[VendoredMethodPackage, ...]) -> None:
+    """Refuse shipped packages a reference could not tell apart, rather than let the directory order pick one.
+
+    A reference is matched against a package's full address without regard to case, as the engine's lookup compares it,
+    so two packages whose full addresses differ only in case are one identity too.
+    """
+    by_identity: dict[str, list[VendoredMethodPackage]] = {}
+    for package in packages:
+        by_identity.setdefault(package.full_address.casefold(), []).append(package)
+    for sharing in by_identity.values():
+        if len(sharing) < 2:
+            continue
+        directories = ", ".join(f"'.mthds/methods/{package.name}/'" for package in sharing)
+        msg = (
+            f"{owner} ships several method packages declaring the address '{sharing[0].full_address}': {directories}. "
+            "A reference matches a package by its address without regard to case, so it could not tell them apart: "
+            "ship one package per address."
+        )
+        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+
+
+def partition_bundle_entries(entries: tuple[tuple[PurePosixPath, bytes], ...], *, owner: str = "Bundle") -> BundlePartition:
+    """Sort a bundle's entries into its own `.mthds` files, its other files, and the method packages it ships.
+
+    An entry under `.mthds/methods/<name>/`, whatever its extension, belongs to the shipped package `<name>`, which
+    leaves the bundle's own content and its library directory. Every other `.mthds` path component is refused with a
+    `422 InvalidBundle` naming the entry rather than dropped, since nothing could ever find what it holds: a file
+    elsewhere under `.mthds/`, a `.mthds/methods/` store below the root, a file directly under `.mthds/methods/`, and a
+    `.mthds/` inside a shipped package, whose own dependencies are not loaded. A shipped package must be a directory
+    whose name does not start with a dot and whose `METHODS.toml` parses, and the address that manifest declares is read
+    here, to name the package in a refusal. Two shipped packages declaring the same full address, compared without regard
+    to case as a reference is matched, are refused too, naming the address and both directories, since nothing but their
+    directory order would decide which one answers. Counting the bundle's own `.mthds` files is the caller's: a bundle
+    whose only `.mthds` files are shipped ones has no content of its own.
+
+    Args:
+        entries: The bundle's `(safe relpath, bytes)` entries.
+        owner: How a refusal names the bundle, for a method package fetched by address as for a request's bundle.
+
+    Returns:
+        The three groups, each in the bundle's entry order.
+    """
+    mthds_entries: list[tuple[PurePosixPath, bytes]] = []
+    library_entries: list[tuple[PurePosixPath, bytes]] = []
+    vendored_entries: dict[str, list[tuple[PurePosixPath, bytes]]] = {}
+    for relpath, data in entries:
+        parts = relpath.parts
+        if parts[0] == _MTHDS_DIR_NAME:
+            if len(parts) < 4 or parts[1] != _METHODS_DIR_NAME:
+                _refuse_entry(owner=owner, relpath=relpath, reason="is under '.mthds/' but not inside a shipped method package")
+            package_relpath = PurePosixPath(*parts[3:])
+            if _MTHDS_DIR_NAME in package_relpath.parts:
+                _refuse_entry(
+                    owner=owner,
+                    relpath=relpath,
+                    reason="is a '.mthds/' directory inside a shipped method package, whose own dependencies are not loaded",
+                )
+            vendored_entries.setdefault(parts[2], []).append((package_relpath, data))
+        elif _MTHDS_DIR_NAME in parts:
+            _refuse_entry(owner=owner, relpath=relpath, reason="is under a '.mthds/' directory below the bundle's root, which is never searched")
+        elif relpath.suffix == ".mthds":
+            mthds_entries.append((relpath, data))
+        else:
+            library_entries.append((relpath, data))
+    vendored_packages = tuple(
+        _vendored_package(owner=owner, name=name, entries=package_entries) for name, package_entries in vendored_entries.items()
+    )
+    _refuse_shared_identities(owner=owner, packages=vendored_packages)
+    return BundlePartition(mthds_entries=tuple(mthds_entries), library_entries=tuple(library_entries), vendored_packages=vendored_packages)
+
+
+@contextmanager
+def materialize_parsed(parsed: ParsedBundle, *, prefix: str = "pipelex-bundle-") -> Generator[MaterializedBundle, None, None]:
+    """Write an already-parsed bundle into a fresh temp directory, cleaned up on exit.
+
+    The yielded `MaterializedBundle.directory` is safe to pass as a `library_dirs`
+    entry; it is removed when the context exits, on both the happy and error path.
+    """
+    directory = Path(tempfile.mkdtemp(prefix=prefix))
+    try:
+        root = directory.resolve()
+        relpaths: list[str] = []
+        for relpath, data in parsed.entries:
+            target = (directory / relpath).resolve()
+            # Defense-in-depth: even after per-part validation, confirm the resolved target stays
+            # under the temp root before writing (guards against symlink/edge normalization surprises).
+            if root != target and root not in target.parents:
+                msg = f"Bundle entry {str(relpath)!r} resolves outside the bundle root"
+                raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_bytes(data)
+            relpaths.append(relpath.as_posix())
+        yield MaterializedBundle(directory=directory, relpaths=tuple(relpaths))
+    finally:
+        shutil.rmtree(directory, ignore_errors=True)
+
+
+@contextmanager
+def materialized_methods_dir(packages: tuple[VendoredMethodPackage, ...]) -> Generator[Path, None, None]:
+    """Write shipped method packages into a temp directory of their own, as `<tmp>/<name>/…`, cleaned up on exit.
+
+    The directory is laid out like `.mthds/methods/`, one directory per package, which is what the engine's
+    `methods_dirs` take, and its path holds no `.mthds/methods` a walk-up from a bundle could take for a store.
+    """
+    entries = tuple((PurePosixPath(package.name) / relpath, data) for package in packages for relpath, data in package.entries)
+    with materialize_parsed(ParsedBundle(entries=entries), prefix="pipelex-methods-") as materialized:
+        yield materialized.directory
+
+
+@contextmanager
+def materialized_bundle(*, bundle_b64: str | None, files: dict[str, str] | None) -> Generator[MaterializedBundle, None, None]:
+    """Parse AND materialize a bundle in one step (convenience for callers that don't gate).
+
+    Equivalent to `parse_bundle(...)` followed by `materialize_parsed(...)`; a caller
+    that must apply the sandbox-hosted gate before disk writes should use the two
+    steps directly instead.
+    """
+    with materialize_parsed(parse_bundle(bundle_b64=bundle_b64, files=files)) as bundle:
+        yield bundle

{pipelex_api-0.75.0 → pipelex_api-0.76.0}/pipelex_api/method_source.py

@@ -12,11 +12,15 @@ manifest-identity package location, the bounds, and the structures check all liv
   files travel as `mthds_contents` (paired with their real relative paths as
   `mthds_sources`, so diagnostics carry true per-file labels), and only the non-`.mthds`
   files are materialized into a temporary `library_dirs` entry — exactly the split the
-  method-bundle transport uses, so a fetched package runs the same proven path.
+  method-bundle transport uses, so a fetched package runs the same proven path. The
+  packages it ships under its own `.mthds/methods/<name>/` are partitioned out of both, as
+  a bundle's are, into a temporary `methods_dirs` entry: the run routes hand it to the
+  engine, and `/validate` does not, resolving such a dependency through the installed store
+  or fetch-on-miss.
 - The tooling routes (`/resolve`, `/codegen`, `/pipe-io`) use
-  :func:`fetch_method_mthds_files`: only the `.mthds` files, as `files[]` items, paired with
-  the manifest's `main_pipe` so the per-pipe route defaults its selector exactly as a run
-  does. No Python ever loads there, so the execution-locus gate does not apply.
+  :func:`fetch_method_mthds_files`: only the package's own `.mthds` files, as `files[]` items,
+  paired with the manifest's `main_pipe` so the per-pipe route defaults its selector exactly
+  as a run does. No Python ever loads there, so the execution-locus gate does not apply.
 
 The security gate (packaging invariant 7 — execution locus decides): `.mthds` content is
 data, always acceptable. On a deployment that is NOT sandbox-hosted, a fetched package
@@ -34,9 +38,9 @@ can never race a running pipeline.
 """
 
 from collections.abc import Generator
-from contextlib import contextmanager
+from contextlib import ExitStack, contextmanager
 from pathlib import Path, PurePosixPath
-from typing import NamedTuple
+from typing import NamedTuple, NoReturn
 
 from mthds.package.discovery import MANIFEST_FILENAME
 from pipelex import log
@@ -46,7 +50,7 @@ from pipelex.methods.method_ref import parse_method_ref
 from pipelex.methods.structures_check import ensure_no_structured_content_python
 from pydantic import BaseModel, ConfigDict, Field
 
-from pipelex_api.bundle import ParsedBundle, materialize_parsed
+from pipelex_api.bundle import BundlePartition, ParsedBundle, materialize_parsed, materialized_methods_dir, partition_bundle_entries
 from pipelex_api.error_types import ErrorType
 from pipelex_api.errors import raise_forbidden, raise_validation_error
 from pipelex_api.limits import MAX_MTHDS_FILE_BYTES
@@ -64,6 +68,9 @@ class FetchedMethodSource(BaseModel):
     mthds_contents: list[str]
     mthds_sources: list[str]
     library_dirs: list[str] | None = None
+    methods_dirs: list[Path] | None = Field(
+        default=None, description="The methods directory holding the packages the package ships under its own `.mthds/methods/`."
+    )
     main_pipe: str | None = Field(default=None, description="The manifest's declared entry pipe; a request `pipe_code` overrides it.")
     provenance: MethodProvenance
 
@@ -106,17 +113,36 @@ def _package_files(package: FetchedMethodPackage) -> list[Path]:
     return files
 
 
-def _read_mthds_text(file_path: Path, *, relative: str, package_address: str) -> str:
+def _decode_mthds_text(data: bytes, *, relative: str, package_address: str) -> str:
     try:
-        content = file_path.read_text(encoding="utf-8")
+        content = data.decode("utf-8")
     except UnicodeDecodeError:
         raise_validation_error(message=f"File '{relative}' in method package '{package_address}' is not valid UTF-8.")
-    if len(content.encode("utf-8")) > MAX_MTHDS_FILE_BYTES:
+    if len(data) > MAX_MTHDS_FILE_BYTES:
         msg = f"File '{relative}' in method package '{package_address}' exceeds the {MAX_MTHDS_FILE_BYTES // 1024} KiB per-file limit."
         raise_validation_error(message=msg)
     return content
 
 
+def _partition_package(package: FetchedMethodPackage, *, files: list[Path]) -> BundlePartition:
+    """Sort a fetched package's files as a bundle's are: its own `.mthds`, its other files, and the packages it ships.
+
+    The refusals are a bundle's too: a `.mthds` path no reference could find is a `422` naming the file.
+    """
+    entries = tuple((PurePosixPath(file_path.relative_to(package.package_dir).as_posix()), file_path.read_bytes()) for file_path in files)
+    return partition_bundle_entries(entries, owner=f"Method package '{package.full_address}'")
+
+
+def _refuse_no_own_mthds(package: FetchedMethodPackage, *, partition: BundlePartition) -> NoReturn:
+    msg = f"Method package '{package.full_address}' contains no .mthds file."
+    if partition.vendored_packages:
+        msg = (
+            f"Method package '{package.full_address}' contains no .mthds file of its own: "
+            "the files under '.mthds/methods/' are the packages it ships."
+        )
+    raise_validation_error(message=msg)
+
+
 def _apply_execution_locus_gate(package: FetchedMethodPackage, *, python_relpaths: list[str]) -> None:
     """Refuse Python that would execute where it must not (see the module docstring)."""
     if not python_relpaths:
@@ -131,10 +157,11 @@ def _apply_execution_locus_gate(package: FetchedMethodPackage, *, python_relpath
 def fetched_method_source(method_ref: str) -> Generator[FetchedMethodSource, None, None]:
     """Fetch a `method_ref`'s package and shape it for the run/validate path.
 
-    Yields the package's `.mthds` files as `(mthds_contents, mthds_sources)` pairs and its
+    Yields the package's `.mthds` files as `(mthds_contents, mthds_sources)` pairs, its
     non-`.mthds` files (PipeFunc `.py`, `requirements.txt`, …) materialized into a temporary
-    `library_dirs` entry, cleaned up on exit. The execution-locus gate runs before anything
-    touches disk.
+    `library_dirs` entry, and the packages it ships under its own `.mthds/methods/<name>/`
+    materialized into a temporary `methods_dirs` entry, each cleaned up on exit. The
+    execution-locus gate runs before anything touches disk.
 
     Args:
         method_ref: The raw `method_ref` string from the request.
@@ -148,37 +175,35 @@ def fetched_method_source(method_ref: str) -> Generator[FetchedMethodSource, Non
     package = _fetch_package(method_ref)
     files = _package_files(package)
     python_relpaths = [file_path.relative_to(package.package_dir).as_posix() for file_path in files if file_path.suffix == ".py"]
+    # The whole package, the packages it ships included, so a structure class anywhere in it is refused before anything is copied.
     _apply_execution_locus_gate(package, python_relpaths=python_relpaths)
 
+    partition = _partition_package(package, files=files)
     mthds_contents: list[str] = []
     mthds_sources: list[str] = []
-    other_entries: list[tuple[PurePosixPath, bytes]] = []
-    for file_path in files:
-        relative = file_path.relative_to(package.package_dir).as_posix()
-        if file_path.suffix == ".mthds":
-            mthds_contents.append(_read_mthds_text(file_path, relative=relative, package_address=package.full_address))
-            mthds_sources.append(relative)
-        elif file_path.name != MANIFEST_FILENAME:
-            # The manifest is already consumed (identity + `main_pipe`); materializing it into
-            # the library dir would hand the local loader a package boundary it must not see.
-            other_entries.append((PurePosixPath(relative), file_path.read_bytes()))
+    for relpath, data in partition.mthds_entries:
+        relative = relpath.as_posix()
+        mthds_contents.append(_decode_mthds_text(data, relative=relative, package_address=package.full_address))
+        mthds_sources.append(relative)
     if not mthds_contents:
-        raise_validation_error(message=f"Method package '{package.full_address}' contains no .mthds file.")
-
-    if not other_entries:
+        _refuse_no_own_mthds(package, partition=partition)
+    # The manifest is already consumed (identity + `main_pipe`); materializing it into the library dir would hand the
+    # local loader a package boundary it must not see.
+    other_entries = tuple((relpath, data) for relpath, data in partition.library_entries if relpath.name != MANIFEST_FILENAME)
+
+    with ExitStack() as stack:
+        library_dirs: list[str] | None = None
+        if other_entries:
+            library_bundle = stack.enter_context(materialize_parsed(ParsedBundle(entries=other_entries)))
+            library_dirs = [str(library_bundle.directory)]
+        methods_dirs: list[Path] | None = None
+        if partition.vendored_packages:
+            methods_dirs = [stack.enter_context(materialized_methods_dir(partition.vendored_packages))]
         yield FetchedMethodSource(
             mthds_contents=mthds_contents,
             mthds_sources=mthds_sources,
-            library_dirs=None,
-            main_pipe=package.manifest.main_pipe,
-            provenance=package.provenance,
-        )
-        return
-    with materialize_parsed(ParsedBundle(entries=tuple(other_entries))) as bundle:
-        yield FetchedMethodSource(
-            mthds_contents=mthds_contents,
-            mthds_sources=mthds_sources,
-            library_dirs=[str(bundle.directory)],
+            library_dirs=library_dirs,
+            methods_dirs=methods_dirs,
             main_pipe=package.manifest.main_pipe,
             provenance=package.provenance,
         )
@@ -188,9 +213,11 @@ def fetch_method_mthds_files(method_ref: str) -> FetchedMthdsFiles:
     """Fetch a `method_ref`'s package and return its `.mthds` files as `files[]` items, plus its `main_pipe`.
 
     The tooling-route shape: each item pairs the file's content with its real relative path
-    as `source`, so crate provenance and diagnostics carry true per-file labels. Only
-    `.mthds` data travels — the package's Python (if any) never loads on these routes, so
-    the execution-locus gate does not apply here. The manifest's `main_pipe` rides beside the
+    as `source`, so crate provenance and diagnostics carry true per-file labels. Only the
+    package's own `.mthds` data travels, never that of a package it ships under its
+    `.mthds/methods/`, since these routes resolve no address-based dependency. The package's
+    Python (if any) never loads on these routes, so the execution-locus gate does not apply
+    here. The manifest's `main_pipe` rides beside the
     files so a per-pipe projection can default its selector the way a run does — the manifest
     is the package author's declaration of the entry pipe, and dropping it here would make
     `main_pipe` buy them nothing on the tooling routes.
@@ -203,13 +230,12 @@ def fetch_method_mthds_files(method_ref: str) -> FetchedMthdsFiles:
         ApiError: 422 for a package with no `.mthds` file or one this server cannot accept.
     """
     package = _fetch_package(method_ref)
+    partition = _partition_package(package, files=_package_files(package))
     items: list[MthdsFileItem] = []
-    for file_path in _package_files(package):
-        if file_path.suffix != ".mthds":
-            continue
-        relative = file_path.relative_to(package.package_dir).as_posix()
-        content = _read_mthds_text(file_path, relative=relative, package_address=package.full_address)
+    for relpath, data in partition.mthds_entries:
+        relative = relpath.as_posix()
+        content = _decode_mthds_text(data, relative=relative, package_address=package.full_address)
         items.append(MthdsFileItem(content=content, source=relative))
     if not items:
-        raise_validation_error(message=f"Method package '{package.full_address}' contains no .mthds file.")
+        _refuse_no_own_mthds(package, partition=partition)
     return FetchedMthdsFiles(files=items, main_pipe=package.manifest.main_pipe)

{pipelex_api-0.75.0 → pipelex_api-0.76.0}/pipelex_api/routes/pipelex/pipeline.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 import hashlib
 import hmac
-from contextlib import contextmanager
-from pathlib import Path, PurePosixPath
+from contextlib import ExitStack, contextmanager
+from pathlib import Path
 from typing import TYPE_CHECKING, Annotated, Any, NamedTuple, cast
 
 from fastapi import APIRouter, Depends, Request
@@ -11,6 +11,7 @@ from fastapi.responses import JSONResponse
 from mthds.protocol.exceptions import PipelineRequestError
 from pipelex.config import get_config, is_pipe_func_sandbox_hosted
 from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.methods.structures_check import ensure_no_structured_content_python
 from pipelex.pipe_run.delivery_assignment import DeliveryAssignment, StorageTarget, WebhookTarget
 from pipelex.pipe_run.pipe_run_protocol import PipeRunProtocol
 from pipelex.pipeline.pipeline_response import PipelexRunResultExecute, PipelexRunResultStart, RunState
@@ -26,7 +27,14 @@ from pydantic import ValidationError
 from typing_extensions import override
 
 from pipelex_api.api_config import get_api_config, resolve_orchestration_mode
-from pipelex_api.bundle import ParsedBundle, materialize_parsed, parse_bundle
+from pipelex_api.bundle import (
+    ParsedBundle,
+    VendoredMethodPackage,
+    materialize_parsed,
+    materialized_methods_dir,
+    parse_bundle,
+    partition_bundle_entries,
+)
 from pipelex_api.error_types import ErrorType
 from pipelex_api.errors import raise_bad_request, raise_forbidden, raise_validation_error
 from pipelex_api.json_body import decode_json_body
@@ -407,6 +415,8 @@ class ApiRunner(PipelexMTHDSProtocol):
             # here, so a group left out of this call is dropped with no error
             # and the run's spans lose their groups while the ack still says 202.
             extras=self.extras,
+            # The packages the request ships, which the base `execute` threads too.
+            methods_dirs=self.methods_dirs,
             pipeline_run_id=pipeline_run_id,
             request_id=request_id,
         )
@@ -685,31 +695,58 @@ async def _parse_request(request: Request) -> tuple[RunRequest, PipelineApiExtra
     return run_request, extras
 
 
+class _BundleRunSource(NamedTuple):
+    """What a request's run source resolves to, before the routes add its entry pipe and provenance."""
+
+    mthds_contents: list[str] | None
+    library_dirs: list[str] | None
+    methods_dirs: list[Path] | None
+
+
+def _ensure_vendored_packages_hold_no_structures(*, methods_dir: Path, packages: tuple[VendoredMethodPackage, ...]) -> None:
+    """Refuse a shipped package whose Python declares a structure class, as a fetched package is on this deployment.
+
+    The library load refuses the bundle's own structure classes, but never reads a shipped package's Python, which is
+    not in a library directory. Each package is named by the address its manifest declares, and each file by its path
+    inside the package, never by the temporary directory.
+    """
+    for package in packages:
+        if any(relpath.suffix == ".py" for relpath, _ in package.entries):
+            ensure_no_structured_content_python(package_dir=methods_dir / package.name, package_address=package.full_address)
+
+
 @contextmanager
-def _bundle_run_source(run_request: RunRequest) -> Generator[tuple[list[str] | None, list[str] | None], None, None]:
-    """Resolve a run's `(mthds_contents, library_dirs)` from the request.
+def _bundle_run_source(run_request: RunRequest) -> Generator[_BundleRunSource, None, None]:
+    """Resolve a run's `(mthds_contents, library_dirs, methods_dirs)` from the request.
 
-    A method bundle KEEPS the proven run path rather than replacing it: its `.mthds`
+    A method bundle KEEPS the proven run path rather than replacing it: its own `.mthds`
     text travels as `mthds_contents` — so `main_pipe` resolves exactly as it does for a
-    plain (non-bundle) run — and ONLY the non-`.mthds` files (custom PipeFunc `.py`,
+    plain (non-bundle) run — and its other files (custom PipeFunc `.py`,
     `requirements.txt`) are materialized into a temp `library_dirs` entry, where the
     load path reads them without importing them. This mirrors "a normal run, plus the Python", instead of
     handing the engine a bare directory with no `mthds_contents` (which never resolves
     `main_pipe`, since that is only derived from `mthds_contents`).
 
-    No bundle → yields the request's own `mthds_contents` and no library dir (the
-    classic path, unchanged). Bundle with no non-`.mthds` files → yields the `.mthds`
-    texts and no library dir. The temp dir (when created) is cleaned up on exit.
-
-    Security gate (decision 5): a bundle that ships custom Python (`.py`) is only
-    honored on a sandbox-hosted deployment, where the load path reads the source
-    without importing it and refuses, with a 403 `MethodStructuresRefusedError`, a
-    bundle whose Python declares a structure class; PipeFunc source is captured for
-    the sandbox. On a non-hosted deployment, running that code would import it
-    in-process — refused with a 403 rather than executing untrusted code.
+    The method packages the bundle ships under `.mthds/methods/<name>/` belong to neither:
+    every file of theirs is written into a temp directory of its own, as `<tmp>/<name>/…`,
+    which the engine searches first for an address-based reference and never writes, so a
+    shipped package wins over an installed copy and ends with the request. The other `.mthds`
+    paths are refused with a `422` naming the entry (`partition_bundle_entries`), and so are two
+    shipped packages declaring the same address and a bundle whose only `.mthds` files are shipped ones.
+
+    No bundle → yields the request's own `mthds_contents` and no directory (the classic path,
+    unchanged). Each temp dir is created only when it has files, and cleaned up on exit.
+
+    Security gate (decision 5): a bundle that ships custom Python (`.py`), in a shipped package
+    or not, is only honored on a sandbox-hosted deployment, where the load path reads the bundle's
+    own source without importing it and refuses, with a 403 `MethodStructuresRefusedError`, a
+    bundle whose Python declares a structure class; PipeFunc source is captured for the sandbox.
+    A shipped package's Python is checked the same way, as a fetched package's is, and never runs.
+    On a non-hosted deployment, running that code would import it in-process — refused with a 403
+    rather than executing untrusted code.
     """
     if run_request.bundle_b64 is None and run_request.files is None:
-        yield run_request.mthds_contents, None
+        yield _BundleRunSource(mthds_contents=run_request.mthds_contents, library_dirs=None, methods_dirs=None)
         return
     # Parse + guard in memory FIRST, then apply the sandbox-hosted gate BEFORE any disk write —
     # a bundle destined for a 403 on a non-hosted deployment never touches the filesystem.
@@ -717,25 +754,36 @@ def _bundle_run_source(run_request: RunRequest) -> Generator[tuple[list[str] | N
     if parsed.has_python_sources and not is_pipe_func_sandbox_hosted():
         msg = "This bundle ships custom Python (.py); running it requires a sandbox-hosted deployment."
         raise_forbidden(message=msg, error_type=ErrorType.CUSTOM_CODE_REQUIRES_SANDBOX)
-    # Split: `.mthds` text → `mthds_contents` (the proven main_pipe path); everything else
-    # (`.py`, `requirements.txt`) → a temp `library_dirs` entry the load path source-captures.
+    # Split: the bundle's own `.mthds` text → `mthds_contents` (the proven main_pipe path); its other files
+    # (`.py`, `requirements.txt`) → a temp `library_dirs` entry the load path source-captures; the packages it
+    # ships → a temp methods directory.
+    partition = partition_bundle_entries(parsed.entries)
     mthds_contents: list[str] = []
-    other_entries: list[tuple[PurePosixPath, bytes]] = []
-    for relpath, content in parsed.entries:
-        if str(relpath).endswith(".mthds"):
-            try:
-                mthds_contents.append(content.decode("utf-8"))
-            except UnicodeDecodeError:
-                raise_validation_error(message=f"Bundle .mthds file '{relpath}' is not valid UTF-8.", error_type=ErrorType.INVALID_BUNDLE)
-        else:
-            other_entries.append((relpath, content))
+    for relpath, content in partition.mthds_entries:
+        try:
+            mthds_contents.append(content.decode("utf-8"))
+        except UnicodeDecodeError:
+            raise_validation_error(message=f"Bundle .mthds file '{relpath}' is not valid UTF-8.", error_type=ErrorType.INVALID_BUNDLE)
     if not mthds_contents:
-        raise_validation_error(message="Method bundle contains no .mthds file.", error_type=ErrorType.INVALID_BUNDLE)
-    if not other_entries:
-        yield mthds_contents, None
-        return
-    with materialize_parsed(ParsedBundle(entries=tuple(other_entries))) as bundle:
-        yield mthds_contents, [str(bundle.directory)]
+        msg = "Method bundle contains no .mthds file."
+        if partition.vendored_packages:
+            msg = (
+                "Method bundle contains no .mthds file of its own: the files under '.mthds/methods/<name>/' are the method "
+                "packages it ships, and the bundle's own .mthds files go outside '.mthds/'."
+            )
+        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+    with ExitStack() as stack:
+        library_dirs: list[str] | None = None
+        if partition.library_entries:
+            library_bundle = stack.enter_context(materialize_parsed(ParsedBundle(entries=partition.library_entries)))
+            library_dirs = [str(library_bundle.directory)]
+        methods_dirs: list[Path] | None = None
+        if partition.vendored_packages:
+            methods_dir = stack.enter_context(materialized_methods_dir(partition.vendored_packages))
+            if is_pipe_func_sandbox_hosted():
+                _ensure_vendored_packages_hold_no_structures(methods_dir=methods_dir, packages=partition.vendored_packages)
+            methods_dirs = [methods_dir]
+        yield _BundleRunSource(mthds_contents=mthds_contents, library_dirs=library_dirs, methods_dirs=methods_dirs)
 
 
 class _ResolvedRunSource(NamedTuple):
@@ -743,6 +791,7 @@ class _ResolvedRunSource(NamedTuple):
 
     mthds_contents: list[str] | None
     library_dirs: list[str] | None
+    methods_dirs: list[Path] | None
     pipe_code: str | None
     method_provenance: MethodProvenance | None
 
@@ -753,7 +802,8 @@ def _run_source(run_request: RunRequest) -> Generator[_ResolvedRunSource, None,
 
     A `method_ref` resolves through `pipelex_api.method_source.fetched_method_source` (fetch → locate →
     execution-locus gate → materialize) into exactly the shape a bundle produces — `.mthds` text
-    as `mthds_contents`, non-`.mthds` files in a temp `library_dirs` entry — so the engine runs
+    as `mthds_contents`, non-`.mthds` files in a temp `library_dirs` entry, the packages the package
+    ships under its own `.mthds/methods/` in a temp `methods_dirs` entry — so the engine runs
     the same proven path. The entry pipe defaults to the fetched manifest's `main_pipe`; a
     request `pipe_code` overrides it. Provenance `(address, tag, commit_sha)` rides back so the
     routes can put it on the response. The other two forms delegate to `_bundle_run_source`,
@@ -764,14 +814,16 @@ def _run_source(run_request: RunRequest) -> Generator[_ResolvedRunSource, None,
             yield _ResolvedRunSource(
                 mthds_contents=fetched.mthds_contents,
                 library_dirs=fetched.library_dirs,
+                methods_dirs=fetched.methods_dirs,
                 pipe_code=run_request.pipe_code or fetched.main_pipe,
                 method_provenance=fetched.provenance,
             )
         return
-    with _bundle_run_source(run_request) as (mthds_contents, library_dirs):
+    with _bundle_run_source(run_request) as bundle_source:
         yield _ResolvedRunSource(
-            mthds_contents=mthds_contents,
-            library_dirs=library_dirs,
+            mthds_contents=bundle_source.mthds_contents,
+            library_dirs=bundle_source.library_dirs,
+            methods_dirs=bundle_source.methods_dirs,
             pipe_code=run_request.pipe_code,
             method_provenance=None,
         )
@@ -825,6 +877,7 @@ async def execute(request: Request) -> JSONResponse:
             read_scope=scopes.read_scope,
             extras=extras.analytics_groups,
             library_dirs=source.library_dirs,
+            methods_dirs=source.methods_dirs,
         )
         response = await runner.execute(
             pipe_code=source.pipe_code,
@@ -916,6 +969,7 @@ async def start(
             read_scope=scopes.read_scope,
             extras=extras.analytics_groups,
             library_dirs=source.library_dirs,
+            methods_dirs=source.methods_dirs,
         )
         start_result = await runner.start(
             pipe_code=source.pipe_code,

{pipelex_api-0.75.0 → pipelex_api-0.76.0}/pipelex_api/schemas/models.py

@@ -71,7 +71,10 @@ class RunRequest(BaseModel):
         bundle_b64: PIPELEX-API EXTENSION — base64-encoded zip of the whole method
             bundle (`.mthds` + `.py` + `structures/*.py` + `requirements.txt`). Lets a
             caller ship custom PipeFunc Python alongside the method; materialized into a
-            temporary library directory before the run. Mutually exclusive with `files`.
+            temporary library directory before the run. The method packages the bundle
+            calls by address may travel with it under `.mthds/methods/<name>/`, each with
+            its `METHODS.toml`, and are searched before the runner's installed copies.
+            Mutually exclusive with `files`.
         files: PIPELEX-API EXTENSION — the bundle as a `{relative_path: text}` map (the
             unzipped equivalent of `bundle_b64`). Mutually exclusive with `bundle_b64`.
         method_ref: PIPELEX-API EXTENSION — run a published method by address instead of
@@ -95,6 +98,8 @@ class RunRequest(BaseModel):
             "PIPELEX-API EXTENSION (not part of the MTHDS Protocol) — base64-encoded zip of the whole "
             "method bundle (`.mthds` + `.py` + `structures/*.py` + `requirements.txt`), materialized into a "
             "temporary library directory before the run so custom PipeFunc Python travels with the method. "
+            "The method packages the bundle calls by address may travel with it under `.mthds/methods/<name>/`, "
+            "each with its `METHODS.toml`; they are searched before the runner's installed copies, for this run alone. "
             "Mutually exclusive with `files`. Custom `.py` is only honored on a sandbox-hosted deployment."
         ),
     )

pipelex_api-0.75.0/pipelex_api/bundle.py

@@ -1,243 +0,0 @@
-"""Materialize a caller-supplied method bundle into a temporary library directory.
-
-A run request may carry the whole method — the `.mthds` bundle plus its PipeFunc
-Python (`pipe_func.py`) and a `requirements.txt` — instead of only
-the inline `mthds_contents` text. Two transport forms are accepted, exactly one
-per request:
-
-  - `bundle_b64`: a base64-encoded zip archive of the bundle directory.
-  - `files`: a `{relative_path: text_content}` map (the zip's contents, unzipped).
-
-The two are equivalent: `files` ≡ the zip's entries. This module decodes either
-form, enforces the ingest guards (both transport forms are refused together; a
-hard file-count and total-size ceiling; per-entry path-safety against absolute
-paths and `..` traversal; a zip-bomb guard that bounds actual decompression),
-writes the surviving files into a fresh temp directory, and hands that directory
-back so the runner can load it via `library_dirs`. In a sandbox-hosted
-deployment the load path reads every `.py` as source text, never importing it:
-it refuses a bundle whose Python declares a structure class
-(`MethodStructuresRefusedError`, a 403) and captures the rest onto the crate for
-the sandbox. The caller is responsible for the hosted-mode gate.
-
-Nothing here imports or executes the bundle's Python — it only writes bytes to
-disk. The caller cleans the directory up via the `materialized_bundle` context
-manager's guaranteed teardown.
-"""
-
-from __future__ import annotations
-
-import base64
-import binascii
-import shutil
-import tempfile
-import zipfile
-from contextlib import contextmanager
-from io import BytesIO
-from pathlib import Path, PurePosixPath
-from typing import TYPE_CHECKING, NamedTuple
-
-from pipelex import log
-from pydantic import ConfigDict
-from pydantic.dataclasses import dataclass
-
-from pipelex_api.error_types import ErrorType
-from pipelex_api.errors import raise_bad_request, raise_payload_too_large, raise_validation_error
-from pipelex_api.limits import MAX_BUNDLE_FILES, MAX_BUNDLE_TOTAL_BYTES
-
-if TYPE_CHECKING:
-    from collections.abc import Generator
-
-
-@dataclass(frozen=True, config=ConfigDict(arbitrary_types_allowed=True))
-class MaterializedBundle:
-    """A bundle written to disk: the directory to load and the relpaths written."""
-
-    directory: Path
-    relpaths: tuple[str, ...]
-
-    @property
-    def has_python_sources(self) -> bool:
-        """True when the bundle ships any `.py` — the trigger for the hosted-mode gate."""
-        return any(relpath.endswith(".py") for relpath in self.relpaths)
-
-
-def _safe_relpath(name: str) -> PurePosixPath:
-    """Validate one bundle entry name and return it as a normalized relative POSIX path.
-
-    Rejects anything that could escape the destination directory: absolute paths,
-    Windows drive/backslash forms (a bare drive prefix like `C:foo` has no slash or
-    backslash yet is drive-relative on Windows, so `:` is rejected outright), and any
-    `..` component. Directory-only entries (trailing slash) return an empty path and
-    are filtered by the caller.
-    """
-    if not name or name in {".", "./"}:
-        return PurePosixPath()
-    if "\\" in name:
-        msg = f"Bundle entry {name!r} uses backslashes; use forward-slash relative paths only"
-        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
-    if ":" in name:
-        msg = f"Bundle entry {name!r} contains ':' (a Windows drive/stream form); use plain relative paths only"
-        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
-    pure = PurePosixPath(name)
-    if pure.is_absolute():
-        msg = f"Bundle entry {name!r} is an absolute path; only relative paths are allowed"
-        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
-    if any(part == ".." for part in pure.parts):
-        msg = f"Bundle entry {name!r} escapes the bundle root via '..'; not allowed"
-        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
-    return pure
-
-
-def _guard_count(count: int) -> None:
-    if count == 0:
-        raise_validation_error(message="Bundle is empty (no files)", error_type=ErrorType.INVALID_BUNDLE)
-    if count > MAX_BUNDLE_FILES:
-        raise_payload_too_large(message=f"Bundle exceeds the {MAX_BUNDLE_FILES}-file limit (got {count})")
-
-
-def _guard_running_total(total_bytes: int) -> None:
-    if total_bytes > MAX_BUNDLE_TOTAL_BYTES:
-        raise_payload_too_large(message=f"Bundle exceeds the {MAX_BUNDLE_TOTAL_BYTES // 1024} KiB decompressed-size limit")
-
-
-# Base64 inflates by 4/3; a zip is compressed, so a bundle whose DECOMPRESSED content is within
-# the ceiling encodes to well under this. Bounding the base64 string BEFORE decoding stops a
-# ~100 MiB request-body (the only other bound) from being expanded into ~75 MiB of heap just to
-# be rejected later by the decompressed-size guard. Slack (+4) covers padding.
-_MAX_BUNDLE_B64_CHARS = MAX_BUNDLE_TOTAL_BYTES * 4 // 3 + 4
-
-
-def _entries_from_zip(bundle_b64: str) -> list[tuple[PurePosixPath, bytes]]:
-    """Decode a base64 zip and return its (safe relpath, bytes) file entries.
-
-    Zip-bomb guard: each member is read through a bounded stream so a lying
-    uncompressed-size header cannot force unbounded decompression — the running
-    total is checked against `MAX_BUNDLE_TOTAL_BYTES` as bytes are pulled. A cheap
-    length check on the still-encoded string runs FIRST, so an oversized payload is
-    refused before it is buffered into memory as decoded bytes.
-    """
-    if len(bundle_b64) > _MAX_BUNDLE_B64_CHARS:
-        raise_payload_too_large(message=f"bundle_b64 exceeds the {MAX_BUNDLE_TOTAL_BYTES // 1024} KiB compressed-size limit")
-    try:
-        raw = base64.b64decode(bundle_b64, validate=True)
-    except (binascii.Error, ValueError) as decode_error:
-        log.warning(f"bundle: invalid base64 ({decode_error})")
-        raise_bad_request(message="bundle_b64 is not valid base64", error_type=ErrorType.INVALID_BASE64)
-
-    try:
-        archive = zipfile.ZipFile(BytesIO(raw))
-    except zipfile.BadZipFile as zip_error:
-        log.warning(f"bundle: corrupt zip ({zip_error})")
-        raise_validation_error(message="bundle_b64 is not a valid zip archive", error_type=ErrorType.INVALID_BUNDLE)
-
-    entries: list[tuple[PurePosixPath, bytes]] = []
-    total_bytes = 0
-    with archive:
-        members = [info for info in archive.infolist() if not info.is_dir()]
-        _guard_count(len(members))
-        budget = MAX_BUNDLE_TOTAL_BYTES
-        for info in members:
-            relpath = _safe_relpath(info.filename)
-            if not relpath.parts:
-                continue
-            # Bounded read: pull at most (remaining budget + 1) bytes so a zip bomb whose header
-            # under-reports its size still cannot decompress past the ceiling.
-            with archive.open(info) as member:
-                data = member.read(budget + 1)
-            total_bytes += len(data)
-            _guard_running_total(total_bytes)
-            budget = MAX_BUNDLE_TOTAL_BYTES - total_bytes
-            entries.append((relpath, data))
-    return entries
-
-
-def _entries_from_files(files: dict[str, str]) -> list[tuple[PurePosixPath, bytes]]:
-    """Validate a {relpath: text} map and return its (safe relpath, bytes) entries."""
-    _guard_count(len(files))
-    entries: list[tuple[PurePosixPath, bytes]] = []
-    total_bytes = 0
-    for name, content in files.items():
-        relpath = _safe_relpath(name)
-        if not relpath.parts:
-            msg = f"Bundle entry {name!r} has no filename"
-            raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
-        data = content.encode("utf-8")
-        total_bytes += len(data)
-        _guard_running_total(total_bytes)
-        entries.append((relpath, data))
-    return entries
-
-
-class ParsedBundle(NamedTuple):
-    """A decoded-and-validated bundle held in memory, NOT yet written to disk.
-
-    Splitting parse from materialize lets the caller apply the sandbox-hosted gate
-    (which keys on `has_python_sources`) BEFORE any disk write — so a bundle destined
-    for a 403 never touches the filesystem. Entries are `(safe relpath, bytes)`.
-    """
-
-    entries: tuple[tuple[PurePosixPath, bytes], ...]
-
-    @property
-    def has_python_sources(self) -> bool:
-        """True when the bundle ships any `.py` — the trigger for the hosted-mode gate."""
-        return any(str(relpath).endswith(".py") for relpath, _ in self.entries)
-
-
-def parse_bundle(*, bundle_b64: str | None, files: dict[str, str] | None) -> ParsedBundle:
-    """Decode + guard a bundle into an in-memory `ParsedBundle` (no disk writes).
-
-    Exactly one of `bundle_b64` / `files` must be supplied; supplying both is a
-    caller mistake (they are the same content in two forms) and is refused. All
-    ingest guards (base64, size, count, path-safety, zip-bomb) run here, so the
-    caller can inspect `has_python_sources` and reject BEFORE materializing to disk.
-    """
-    if bundle_b64 is not None and files is not None:
-        msg = "Provide either bundle_b64 or files, not both"
-        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
-    if bundle_b64 is not None:
-        entries = _entries_from_zip(bundle_b64)
-    elif files is not None:
-        entries = _entries_from_files(files)
-    else:
-        msg = "No bundle supplied (bundle_b64 and files are both absent)"
-        raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
-    return ParsedBundle(entries=tuple(entries))
-
-
-@contextmanager
-def materialize_parsed(parsed: ParsedBundle) -> Generator[MaterializedBundle, None, None]:
-    """Write an already-parsed bundle into a fresh temp directory, cleaned up on exit.
-
-    The yielded `MaterializedBundle.directory` is safe to pass as a `library_dirs`
-    entry; it is removed when the context exits, on both the happy and error path.
-    """
-    directory = Path(tempfile.mkdtemp(prefix="pipelex-bundle-"))
-    try:
-        root = directory.resolve()
-        relpaths: list[str] = []
-        for relpath, data in parsed.entries:
-            target = (directory / relpath).resolve()
-            # Defense-in-depth: even after per-part validation, confirm the resolved target stays
-            # under the temp root before writing (guards against symlink/edge normalization surprises).
-            if root != target and root not in target.parents:
-                msg = f"Bundle entry {str(relpath)!r} resolves outside the bundle root"
-                raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
-            target.parent.mkdir(parents=True, exist_ok=True)
-            target.write_bytes(data)
-            relpaths.append(relpath.as_posix())
-        yield MaterializedBundle(directory=directory, relpaths=tuple(relpaths))
-    finally:
-        shutil.rmtree(directory, ignore_errors=True)
-
-
-@contextmanager
-def materialized_bundle(*, bundle_b64: str | None, files: dict[str, str] | None) -> Generator[MaterializedBundle, None, None]:
-    """Parse AND materialize a bundle in one step (convenience for callers that don't gate).
-
-    Equivalent to `parse_bundle(...)` followed by `materialize_parsed(...)`; a caller
-    that must apply the sandbox-hosted gate before disk writes should use the two
-    steps directly instead.
-    """
-    with materialize_parsed(parse_bundle(bundle_b64=bundle_b64, files=files)) as bundle:
-        yield bundle