pipelex-api 0.84.0__tar.gz → 0.85.0__tar.gz

{pipelex_api-0.84.0 → pipelex_api-0.85.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pipelex-api
-Version: 0.84.0
+Version: 0.85.0
 Summary: Pipelex API
 Project-URL: Homepage, https://pipelex.com
 Project-URL: Repository, https://github.com/Pipelex/pipelex
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: <3.15,>=3.11
 Requires-Dist: fastapi<0.142,>=0.118.0
-Requires-Dist: pipelex[anthropic,bedrock,fal,google,google-genai,mistralai]==0.84.0
+Requires-Dist: pipelex[anthropic,bedrock,fal,google,google-genai,mistralai]==0.85.0
 Requires-Dist: pyjwt>=2.10.1
 Requires-Dist: uvicorn>=0.37.0
 Provides-Extra: dev

pipelex_api-0.85.0/pipelex_api/bundle_resources.py

@@ -0,0 +1,148 @@
+"""Decode the files a request submits with its bundle that its steps read, such as a template file.
+
+They come in two ways. `resources` is a JSON object mapping each file's bundle-relative path, the exact path a step
+names (a `PipeDocGen` step's `template_file`), to the file's bytes in standard base64, beside `mthds_contents` or a
+method bundle. A method bundle (`bundle_b64` or `files`) may also ship the file beside its `.mthds` file, which
+`files_its_steps_read` picks out. Every bundle of the request is read as sitting in one directory, so one path holds
+one file. The decoded files go to the runtime as `BundleResources`, which refuses a file none of the bundle's steps
+names once the bundle is parsed.
+
+Refusals, in the order they are checked:
+
+- more `resources` than `MAX_RESOURCES_PER_REQUEST` → 413 `PayloadTooLarge`;
+- a path that cannot name a file inside the bundle (absolute, a `..` segment, a backslash, a colon, empty, or a
+  second spelling of a path already submitted) → 422 `InvalidResourcePath`, before any value is decoded;
+- more files than `MAX_RESOURCES_PER_REQUEST`, counting those the method bundle ships → 413 `PayloadTooLarge`;
+- a file whose encoded length already exceeds the per-file ceiling → 413 `PayloadTooLarge`, before decoding;
+- a value that is not valid base64 → 400 `InvalidBase64`, as `bundle_b64` is;
+- two different files under one path, sent in `resources` and shipped in the bundle, or shipped beside two of its
+  `.mthds` files → 422 `InvalidResourcePath`;
+- a decoded or shipped file over `MAX_RESOURCE_FILE_BYTES`, or a running total over `MAX_RESOURCES_TOTAL_BYTES` → 413.
+"""
+
+import base64
+import binascii
+from collections.abc import Iterable, Mapping
+from pathlib import PurePosixPath
+from typing import NoReturn
+
+from pipelex import log
+from pipelex.core.bundle_resources.bundle_resource import (
+    BundleResources,
+    canonical_resource_path,
+    canonical_resource_path_or_none,
+    make_bundle_resources,
+)
+from pipelex.core.bundle_resources.exceptions import BundleResourcePathError
+from pipelex.mthds_parsing.exceptions import MthdsParserError
+from pipelex.mthds_parsing.parser import MthdsParser
+from pipelex.tools.log.error_fields import error_fields
+
+from pipelex_api.error_types import ErrorType
+from pipelex_api.errors import raise_bad_request, raise_payload_too_large, raise_validation_error
+from pipelex_api.limits import MAX_RESOURCE_FILE_BYTES, MAX_RESOURCES_PER_REQUEST, MAX_RESOURCES_TOTAL_BYTES
+
+# Base64 inflates by 4/3, so a file within the ceiling encodes to at most this many characters (+4 covers padding).
+# Checking the encoded length first refuses an oversized file before it is decoded into memory.
+_MAX_RESOURCE_B64_CHARS = MAX_RESOURCE_FILE_BYTES * 4 // 3 + 4
+
+
+def _refuse_two_files_under_one_path(*, path: str, where: str) -> NoReturn:
+    msg = (
+        f"Two different files are named '{path}', {where}. Every bundle of a request reads its files as if it sat in one "
+        "directory, so a path names one file: send one of them, or rename one and the step that reads it."
+    )
+    raise_validation_error(message=msg, error_type=ErrorType.INVALID_RESOURCE_PATH)
+
+
+def files_its_steps_read(
+    *, mthds_entries: Iterable[tuple[PurePosixPath, str]], other_entries: Iterable[tuple[PurePosixPath, bytes]]
+) -> dict[str, bytes]:
+    """The files a method bundle ships beside its `.mthds` files that their steps read, keyed by the path a step names.
+
+    Each `.mthds` file is parsed for the files its steps name, such as a `PipeDocGen` step's `template_file`, which is
+    looked up relative to that file's own directory in the bundle. The bundle's `.mthds` files load as content strings,
+    so these files travel with them as the `resources` a request sends do, wherever the run goes. A file a step names
+    that the bundle does not ship is left out, for `resources` to supply or the load to refuse saying so, and a `.mthds`
+    file that does not parse is skipped, for the load to give its verdict on.
+
+    Raises (as a 422 `InvalidResourcePath`): two steps naming one path, beside two `.mthds` files that ship different files.
+    """
+    shipped = dict(other_entries)
+    files: dict[str, bytes] = {}
+    for mthds_relpath, mthds_content in mthds_entries:
+        try:
+            blueprint = MthdsParser.make_pipelex_bundle_blueprint(mthds_content=mthds_content)
+        except MthdsParserError as exc:
+            log.debug("A method bundle's file did not parse, so the files its steps read are left to its load", fields=error_fields(exc=exc))
+            continue
+        for pipe_blueprint in (blueprint.pipe or {}).values():
+            for named_path in pipe_blueprint.bundle_resource_reads:
+                canonical_path = canonical_resource_path_or_none(path=named_path)
+                if canonical_path is None:
+                    continue
+                data = shipped.get(mthds_relpath.parent / canonical_path)
+                if data is None:
+                    continue
+                known = files.get(canonical_path)
+                if known is not None and known != data:
+                    _refuse_two_files_under_one_path(path=canonical_path, where="shipped beside two of the bundle's .mthds files")
+                files[canonical_path] = data
+    return files
+
+
+def decode_bundle_resources(*, resources: dict[str, str] | None, bundle_files: Mapping[str, bytes] | None = None) -> BundleResources | None:
+    """The files the bundle's steps read, decoded and keyed by canonical path, or None when there are none.
+
+    `resources` are the ones the request sends, in base64. `bundle_files` are the ones its method bundle ships, as
+    `files_its_steps_read` picks them out. Both are held to the same limits, together, and a file in both must be the same.
+    """
+    if not resources and not bundle_files:
+        return None
+    sent = resources or {}
+    shipped = bundle_files or {}
+    if len(sent) > MAX_RESOURCES_PER_REQUEST:
+        raise_payload_too_large(message=f"resources exceeds the {MAX_RESOURCES_PER_REQUEST}-file limit (got {len(sent)})")
+    # Every path first, so a second spelling of one file is refused as such before any value is decoded.
+    sent_by_path: dict[str, tuple[str, str]] = {}
+    for path, encoded in sent.items():
+        try:
+            canonical_path = canonical_resource_path(path=path)
+        except BundleResourcePathError as exc:
+            raise_validation_error(message=exc.message, error_type=ErrorType.INVALID_RESOURCE_PATH)
+        if canonical_path in sent_by_path:
+            msg = f"Two files submitted with the bundle name the same file '{canonical_path}'."
+            raise_validation_error(message=msg, error_type=ErrorType.INVALID_RESOURCE_PATH)
+        sent_by_path[canonical_path] = (path, encoded)
+    file_count = len(sent_by_path.keys() | shipped.keys())
+    if file_count > MAX_RESOURCES_PER_REQUEST:
+        msg = (
+            f"The files the bundle's steps read, in resources and in the bundle, exceed the {MAX_RESOURCES_PER_REQUEST}-file limit (got {file_count})"
+        )
+        raise_payload_too_large(message=msg)
+    files: dict[str, bytes] = {}
+    total_bytes = 0
+    for canonical_path, (path, encoded) in sent_by_path.items():
+        if len(encoded) > _MAX_RESOURCE_B64_CHARS:
+            raise_payload_too_large(message=f"The file '{path}' in resources exceeds the {MAX_RESOURCE_FILE_BYTES // 1024} KiB per-file limit")
+        try:
+            data = base64.b64decode(encoded, validate=True)
+        except (binascii.Error, ValueError) as decode_error:
+            # DEBUG: the refusal below is a caller's mistake, which the error handler already logs; this keeps the cause.
+            log.debug("A submitted file's base64 could not be decoded", fields=error_fields(exc=decode_error))
+            raise_bad_request(message=f"The file '{path}' in resources is not valid base64", error_type=ErrorType.INVALID_BASE64)
+        shipped_data = shipped.get(canonical_path)
+        if shipped_data is not None and shipped_data != data:
+            _refuse_two_files_under_one_path(path=canonical_path, where="one sent in resources and one shipped in the bundle")
+        files[canonical_path] = data
+    for canonical_path, data in shipped.items():
+        files.setdefault(canonical_path, data)
+    for canonical_path, data in files.items():
+        if len(data) > MAX_RESOURCE_FILE_BYTES:
+            raise_payload_too_large(
+                message=f"The file '{canonical_path}' a step reads exceeds the {MAX_RESOURCE_FILE_BYTES // 1024} KiB per-file limit"
+            )
+        total_bytes += len(data)
+        if total_bytes > MAX_RESOURCES_TOTAL_BYTES:
+            raise_payload_too_large(message=f"The files the bundle's steps read exceed the {MAX_RESOURCES_TOTAL_BYTES // 1024} KiB total-size limit")
+    return make_bundle_resources(files=files)

{pipelex_api-0.84.0 → pipelex_api-0.85.0}/pipelex_api/error_types.py

@@ -61,6 +61,10 @@ class ErrorType(StrEnum):
     # A run request carried a malformed method bundle (`bundle_b64` / `files`): a corrupt zip, an
     # unsafe entry name (absolute path or `..` traversal), or both transport forms supplied at once.
     INVALID_BUNDLE = "InvalidBundle"
+    # A request's `resources` map keys a submitted file by a path that cannot name a file inside the
+    # bundle: absolute, with a `..` segment, a backslash or a colon, empty, or a second spelling of a
+    # path already submitted. A 422: the key is the path a step names, relative to the bundle.
+    INVALID_RESOURCE_PATH = "InvalidResourcePath"
     # A method carrying custom Python (`.py` in a bundle, or in a fetched `method_ref` package)
     # reached a deployment that is NOT sandbox-hosted. Running customer code in-process is
     # refused (403): custom code is a sandbox-hosted capability only. Use a sandbox-hosted

{pipelex_api-0.84.0 → pipelex_api-0.85.0}/pipelex_api/exception_handlers.py

@@ -341,6 +341,11 @@ _ERROR_TYPE_STATUS_OVERRIDES: dict[str, int] = {
     #     sandboxed PipeFuncs, not in-process Python) → 403, the same status as the bundle
     #     transport's custom-code gate.
     "MethodStructuresRefusedError": 403,
+    # The files submitted in `resources` that the bundle's steps read when the method loads, its HTML templates,
+    # total more than they may, since they travel inline with the method wherever it runs. Only the parsed bundle
+    # says which files those are, so the library refuses them rather than this server's decoding: the request
+    # entity itself is too large → 413, as the decoder's own size refusals are.
+    "BundleResourcesTooLargeError": 413,
 }
 
 

{pipelex_api-0.84.0 → pipelex_api-0.85.0}/pipelex_api/limits.py

@@ -25,6 +25,9 @@ DEFAULT_MAX_CALLBACK_URLS = 5
 DEFAULT_MAX_CALLBACK_URL_LEN = 2048
 DEFAULT_MAX_BUNDLE_FILES = 128  # entries in a materialized method bundle (.mthds + .py + requirements.txt)
 DEFAULT_MAX_BUNDLE_TOTAL_KIB = 8 * 1024  # 8 MiB decompressed across the whole bundle (zip-bomb guard)
+DEFAULT_MAX_RESOURCES_PER_REQUEST = 32  # files submitted with a bundle that its steps read (`resources`)
+DEFAULT_MAX_RESOURCE_FILE_KIB = 5 * 1024  # 5 MiB decoded per submitted file, such as an office template
+DEFAULT_MAX_RESOURCES_TOTAL_KIB = 10 * 1024  # 10 MiB decoded across a request's submitted files
 DEFAULT_MAX_METHOD_CACHE_CLONES = 64  # cached method-package clones (one per resolved commit SHA)
 DEFAULT_MAX_METHOD_CACHE_TOTAL_KIB = 512 * 1024  # 512 MiB across all cached clones
 DEFAULT_MAX_METHOD_CACHE_AGE_HOURS = 24  # a cached clone unused for this long is evicted
@@ -43,6 +46,10 @@ MAX_CALLBACK_URL_LEN = get_positive_int_env(env_var="MAX_CALLBACK_URL_LEN", defa
 MAX_BUNDLE_FILES = get_positive_int_env(env_var="MAX_BUNDLE_FILES", default=DEFAULT_MAX_BUNDLE_FILES)
 MAX_BUNDLE_TOTAL_BYTES = get_positive_int_env(env_var="MAX_BUNDLE_TOTAL_KIB", default=DEFAULT_MAX_BUNDLE_TOTAL_KIB) * 1024
 
+MAX_RESOURCES_PER_REQUEST = get_positive_int_env(env_var="MAX_RESOURCES_PER_REQUEST", default=DEFAULT_MAX_RESOURCES_PER_REQUEST)
+MAX_RESOURCE_FILE_BYTES = get_positive_int_env(env_var="MAX_RESOURCE_FILE_KIB", default=DEFAULT_MAX_RESOURCE_FILE_KIB) * 1024
+MAX_RESOURCES_TOTAL_BYTES = get_positive_int_env(env_var="MAX_RESOURCES_TOTAL_KIB", default=DEFAULT_MAX_RESOURCES_TOTAL_KIB) * 1024
+
 MAX_METHOD_CACHE_CLONES = get_positive_int_env(env_var="MAX_METHOD_CACHE_CLONES", default=DEFAULT_MAX_METHOD_CACHE_CLONES)
 MAX_METHOD_CACHE_TOTAL_BYTES = get_positive_int_env(env_var="MAX_METHOD_CACHE_TOTAL_KIB", default=DEFAULT_MAX_METHOD_CACHE_TOTAL_KIB) * 1024
 MAX_METHOD_CACHE_AGE_SECONDS = get_positive_int_env(env_var="MAX_METHOD_CACHE_AGE_HOURS", default=DEFAULT_MAX_METHOD_CACHE_AGE_HOURS) * 3600

{pipelex_api-0.84.0 → pipelex_api-0.85.0}/pipelex_api/routes/pipelex/crate_ops.py

@@ -27,6 +27,7 @@ from pipelex.pipeline.resolve_bundle import resolve_crate_from_contents
 from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
 from pydantic import BaseModel, Field
 
+from pipelex_api.bundle_resources import decode_bundle_resources
 from pipelex_api.error_types import ErrorType
 from pipelex_api.errors import raise_not_implemented
 from pipelex_api.method_source import fetch_method_mthds_files
@@ -151,6 +152,8 @@ def resolve_requested_crate(request_data: MthdsFilesRequest) -> ResolvedClosure:
     crate = resolve_crate_from_contents(
         mthds_contents=[item.content for item in selection.files],
         mthds_sources=[item.source for item in selection.files],
+        # The files the closure's steps read, such as a PipeDocGen template file, which the load reads.
+        bundle_resources=decode_bundle_resources(resources=request_data.resources),
     )
     return ResolvedClosure(crate=crate, files=selection.files, manifest_main_pipe=selection.manifest_main_pipe)
 

{pipelex_api-0.84.0 → pipelex_api-0.85.0}/pipelex_api/routes/pipelex/pipeline.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import hashlib
 import hmac
 from contextlib import ExitStack, contextmanager
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from typing import TYPE_CHECKING, Annotated, Any, NamedTuple, cast
 
 from fastapi import APIRouter, Depends, Request
@@ -14,11 +14,13 @@ from pipelex.core.pipes.pipe_output import PipeOutput
 from pipelex.methods.structures_check import ensure_no_structured_content_python
 from pipelex.pipe_run.delivery_assignment import DeliveryAssignment, StorageTarget, WebhookTarget
 from pipelex.pipe_run.pipe_run_protocol import PipeRunProtocol
+from pipelex.pipe_run.remote_dispatch import prepare_job_for_remote_dispatch
 from pipelex.pipeline.pipeline_response import PipelexRunResultExecute, PipelexRunResultStart, RunState
 from pipelex.pipeline.pipeline_run_setup import pipeline_run_setup
 from pipelex.pipeline.runner import PipelexMTHDSProtocol
 from pipelex.reporting.usage_records import apply_tokens_usage_wire_shape
 from pipelex.runtime_bridge.exceptions import MissingBundleValidatorError, MissingOrchestratorError
+from pipelex.runtime_bridge.orchestration_mode import DIRECT_ORCHESTRATION_MODE
 from pipelex.runtime_bridge.primitives.hydration import hydrate_working_memory
 from pipelex.runtime_hub import get_bundle_validator_registry, get_orchestrator_registry
 from pipelex.system.environment import get_required_env
@@ -35,6 +37,7 @@ from pipelex_api.bundle import (
     parse_bundle,
     partition_bundle_entries,
 )
+from pipelex_api.bundle_resources import decode_bundle_resources, files_its_steps_read
 from pipelex_api.error_types import ErrorType
 from pipelex_api.errors import raise_bad_request, raise_forbidden, raise_validation_error
 from pipelex_api.json_body import decode_json_body
@@ -227,6 +230,18 @@ def _pipe_output_from_run_output(run_output: PipelexPipeRunOutput) -> PipeOutput
     )
 
 
+async def _prepare_job_for_dispatch(*, pipe_job: PipeJob, orchestration_mode: str) -> PipeJob:
+    """The job as the mode's orchestrator receives it: prepared to leave this process for every mode but `direct`.
+
+    The files submitted with a bundle travel in the job's crate. An orchestrator other than the in-process `direct`
+    one, such as Temporal, runs the job outside this process, so the files its steps read only when they run move to
+    the run's storage first, dry runs included (`prepare_job_for_remote_dispatch`). A `direct` run keeps them inline.
+    """
+    if orchestration_mode == DIRECT_ORCHESTRATION_MODE:
+        return pipe_job
+    return await prepare_job_for_remote_dispatch(pipe_job=pipe_job)
+
+
 class _OrchestratorPipeRun(PipeRunProtocol):
     """Adapts a mode-selected orchestrator to the `PipeRunProtocol` the base `execute` drives.
 
@@ -241,12 +256,14 @@ class _OrchestratorPipeRun(PipeRunProtocol):
     synchronous and always drives this arm, so there is no delivery axis to thread here.
     """
 
-    def __init__(self, *, orchestrator: OrchestratorProtocol) -> None:
+    def __init__(self, *, orchestrator: OrchestratorProtocol, orchestration_mode: str) -> None:
         self._orchestrator = orchestrator
+        self._orchestration_mode = orchestration_mode
 
     @override
     async def run(self, pipe_job: PipeJob, *, delivery_assignment: DeliveryAssignment | None = None) -> PipeOutput:
-        run_output = await self._orchestrator.execute(pipe_job=pipe_job, delivery_assignment=delivery_assignment)
+        dispatched_job = await _prepare_job_for_dispatch(pipe_job=pipe_job, orchestration_mode=self._orchestration_mode)
+        run_output = await self._orchestrator.execute(pipe_job=dispatched_job, delivery_assignment=delivery_assignment)
         return _pipe_output_from_run_output(run_output)
 
 
@@ -313,7 +330,7 @@ class ApiRunner(PipelexMTHDSProtocol):
         # PipeRun, then delegate to the base execute, which owns the full run lifecycle. The
         # ApiRunner is constructed per request, so mutating _pipe_run here is request-scoped.
         # `/execute` is synchronous, so it drives the orchestrator's BLOCKING `execute` arm.
-        self._pipe_run = _OrchestratorPipeRun(orchestrator=orchestrator)
+        self._pipe_run = _OrchestratorPipeRun(orchestrator=orchestrator, orchestration_mode=orchestration_mode)
         return await super().execute(
             pipe_code=pipe_code,
             mthds_contents=mthds_contents,
@@ -417,6 +434,8 @@ class ApiRunner(PipelexMTHDSProtocol):
             extras=self.extras,
             # The packages the request ships, which the base `execute` threads too.
             methods_dirs=self.methods_dirs,
+            # The files submitted with the contents, such as a template file, which the base `execute` threads too.
+            bundle_resources=self.bundle_resources,
             pipeline_run_id=pipeline_run_id,
             request_id=request_id,
         )
@@ -455,7 +474,8 @@ class ApiRunner(PipelexMTHDSProtocol):
         # `PipelexPipeRunInput` (which carries no request_id / output_multiplicity /
         # dynamic_output_concept_ref and skips run registration + telemetry). `start` genuinely
         # enqueues the job and returns a `PipelexPipeDispatchAck` (ids only) immediately.
-        dispatch_ack = await orchestrator.start(pipe_job=pipe_job, delivery_assignment=delivery_assignment)
+        dispatched_job = await _prepare_job_for_dispatch(pipe_job=pipe_job, orchestration_mode=orchestration_mode)
+        dispatch_ack = await orchestrator.start(pipe_job=dispatched_job, delivery_assignment=delivery_assignment)
 
         return PipelexRunResultStart(
             pipeline_run_id=resolved_pipeline_run_id,
@@ -517,6 +537,8 @@ class ApiRunner(PipelexMTHDSProtocol):
             # done for the caller this runner was built for — the user and groups a run would state.
             caller_identity=self.caller_identity,
             graph_pipe_code=graph_pipe_code,
+            # The files submitted with the contents, which the validation's dry run reads, the template check included.
+            bundle_resources=self.bundle_resources,
         )
         # The core seam types its valid arm at the protocol-level ValidationReport (a leaf type)
         # to stay import-acyclic in core; every registered validator in fact produces the canonical
@@ -701,6 +723,8 @@ class _BundleRunSource(NamedTuple):
     mthds_contents: list[str] | None
     library_dirs: list[str] | None
     methods_dirs: list[Path] | None
+    bundle_files: dict[str, bytes]
+    """The files a method bundle ships beside its `.mthds` files that their steps read, keyed by the path a step names."""
 
 
 def _ensure_vendored_packages_hold_no_structures(*, methods_dir: Path, packages: tuple[VendoredMethodPackage, ...]) -> None:
@@ -746,7 +770,7 @@ def _bundle_run_source(run_request: RunRequest) -> Generator[_BundleRunSource, N
     rather than executing untrusted code.
     """
     if run_request.bundle_b64 is None and run_request.files is None:
-        yield _BundleRunSource(mthds_contents=run_request.mthds_contents, library_dirs=None, methods_dirs=None)
+        yield _BundleRunSource(mthds_contents=run_request.mthds_contents, library_dirs=None, methods_dirs=None, bundle_files={})
         return
     # Parse + guard in memory FIRST, then apply the sandbox-hosted gate BEFORE any disk write —
     # a bundle destined for a 403 on a non-hosted deployment never touches the filesystem.
@@ -758,12 +782,13 @@ def _bundle_run_source(run_request: RunRequest) -> Generator[_BundleRunSource, N
     # (`.py`, `requirements.txt`) → a temp `library_dirs` entry the load path source-captures; the packages it
     # ships → a temp methods directory.
     partition = partition_bundle_entries(parsed.entries)
-    mthds_contents: list[str] = []
+    mthds_entries: list[tuple[PurePosixPath, str]] = []
     for relpath, content in partition.mthds_entries:
         try:
-            mthds_contents.append(content.decode("utf-8"))
+            mthds_entries.append((relpath, content.decode("utf-8")))
         except UnicodeDecodeError:
             raise_validation_error(message=f"Bundle .mthds file '{relpath}' is not valid UTF-8.", error_type=ErrorType.INVALID_BUNDLE)
+    mthds_contents = [content for _, content in mthds_entries]
     if not mthds_contents:
         msg = "Method bundle contains no .mthds file."
         if partition.vendored_packages:
@@ -772,6 +797,8 @@ def _bundle_run_source(run_request: RunRequest) -> Generator[_BundleRunSource, N
                 "packages it ships, and the bundle's own .mthds files go outside '.mthds/'."
             )
         raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
+    # The `.mthds` files load as content strings, so a template file shipped beside one travels with it as a resource.
+    bundle_files = files_its_steps_read(mthds_entries=mthds_entries, other_entries=partition.library_entries)
     with ExitStack() as stack:
         library_dirs: list[str] | None = None
         if partition.library_entries:
@@ -783,7 +810,7 @@ def _bundle_run_source(run_request: RunRequest) -> Generator[_BundleRunSource, N
             if is_pipe_func_sandbox_hosted():
                 _ensure_vendored_packages_hold_no_structures(methods_dir=methods_dir, packages=partition.vendored_packages)
             methods_dirs = [methods_dir]
-        yield _BundleRunSource(mthds_contents=mthds_contents, library_dirs=library_dirs, methods_dirs=methods_dirs)
+        yield _BundleRunSource(mthds_contents=mthds_contents, library_dirs=library_dirs, methods_dirs=methods_dirs, bundle_files=bundle_files)
 
 
 class _ResolvedRunSource(NamedTuple):
@@ -794,6 +821,8 @@ class _ResolvedRunSource(NamedTuple):
     methods_dirs: list[Path] | None
     pipe_code: str | None
     method_provenance: MethodProvenance | None
+    bundle_files: dict[str, bytes]
+    """The files a method bundle ships that its steps read, which join the request's `resources`."""
 
 
 @contextmanager
@@ -817,6 +846,7 @@ def _run_source(run_request: RunRequest) -> Generator[_ResolvedRunSource, None,
                 methods_dirs=fetched.methods_dirs,
                 pipe_code=run_request.pipe_code or fetched.main_pipe,
                 method_provenance=fetched.provenance,
+                bundle_files={},
             )
         return
     with _bundle_run_source(run_request) as bundle_source:
@@ -826,6 +856,7 @@ def _run_source(run_request: RunRequest) -> Generator[_ResolvedRunSource, None,
             methods_dirs=bundle_source.methods_dirs,
             pipe_code=run_request.pipe_code,
             method_provenance=None,
+            bundle_files=bundle_source.bundle_files,
         )
 
 
@@ -878,6 +909,7 @@ async def execute(request: Request) -> JSONResponse:
             extras=extras.analytics_groups,
             library_dirs=source.library_dirs,
             methods_dirs=source.methods_dirs,
+            bundle_resources=decode_bundle_resources(resources=run_request.resources, bundle_files=source.bundle_files),
         )
         response = await runner.execute(
             pipe_code=source.pipe_code,
@@ -970,6 +1002,7 @@ async def start(
             extras=extras.analytics_groups,
             library_dirs=source.library_dirs,
             methods_dirs=source.methods_dirs,
+            bundle_resources=decode_bundle_resources(resources=run_request.resources, bundle_files=source.bundle_files),
         )
         start_result = await runner.start(
             pipe_code=source.pipe_code,

{pipelex_api-0.84.0 → pipelex_api-0.85.0}/pipelex_api/routes/pipelex/validate.py

@@ -12,13 +12,14 @@ from pipelex.pipeline.validation_report import PipelexValidationReport
 from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
 from pydantic import BaseModel, Field, model_validator
 
+from pipelex_api.bundle_resources import decode_bundle_resources
 from pipelex_api.errors import raise_validation_error
 from pipelex_api.exception_handlers import problem_response_from_error_report
 from pipelex_api.json_body import JsonBodyRoute
 from pipelex_api.method_source import fetched_method_source
 from pipelex_api.openapi_responses import PROBLEM_403_RUN_POLICY, PROBLEM_404_METHOD_PACKAGE
 from pipelex_api.routes.pipelex.pipeline import ApiRunner, get_request_user_id
-from pipelex_api.schemas.models import CallerAnalyticsGroupsMixin, MthdsContentsRequest
+from pipelex_api.schemas.models import RESOURCES_FIELD_DESCRIPTION, CallerAnalyticsGroupsMixin, MthdsContentsRequest
 
 router = APIRouter(tags=["validate"], route_class=JsonBodyRoute)
 
@@ -87,6 +88,7 @@ class ValidateRequest(MthdsContentsRequest, CallerAnalyticsGroupsMixin):
             "at the owning file. Must match `mthds_contents` in length when present."
         ),
     )
+    resources: dict[str, str] | None = Field(default=None, description=RESOURCES_FIELD_DESCRIPTION)
     render: list[str] = Field(
         default_factory=list,
         description=(
@@ -124,6 +126,10 @@ class ValidateRequest(MthdsContentsRequest, CallerAnalyticsGroupsMixin):
         if self.mthds_sources is not None and self.method_ref is not None:
             msg = "mthds_sources accompanies inline mthds_contents; a method_ref's sources are the package's own file names"
             raise ValueError(msg)
+        # `resources` are the files the inline contents' steps read; a `method_ref`'s files are the package's own.
+        if self.resources and self.method_ref is not None:
+            msg = "resources accompany inline mthds_contents; a method_ref's files are the package's own"
+            raise ValueError(msg)
         # A caller-supplied length mismatch is a request-shape bug → caught here as a 422.
         # Without this guard it reaches the runtime's `validate_bundle`, which treats the
         # mismatch as an internal host error (500) — the wrong status for caller input.
@@ -396,7 +402,11 @@ async def validate_mthds(request: Request, request_data: ValidateRequest) -> JSO
         validated_contents = request_data.mthds_contents
         # Inline contents carry no manifest, so the closure's own declaration is the whole chain.
         manifest_main_pipe = None
-        verdict = await ApiRunner(user_id=user_id, extras=request_data.analytics_groups).validate_verdict(
+        verdict = await ApiRunner(
+            user_id=user_id,
+            extras=request_data.analytics_groups,
+            bundle_resources=decode_bundle_resources(resources=request_data.resources),
+        ).validate_verdict(
             mthds_contents=request_data.mthds_contents,
             mthds_sources=request_data.mthds_sources,
             allow_signatures=request_data.allow_signatures,

{pipelex_api-0.84.0 → pipelex_api-0.85.0}/pipelex_api/schemas/models.py

@@ -15,6 +15,7 @@ from urllib.parse import urlparse
 from mthds.protocol.exceptions import PipelineRequestError
 from mthds.protocol.pipe_output import VariableMultiplicity
 from mthds.protocol.pipeline_inputs import PipelineInputs
+from pipelex.core.bundle_resources.bundle_resource import MAX_LOAD_STAGE_RESOURCES_BYTES
 from pipelex.core.pipes.pipe_output import PipeOutput
 from pipelex.methods.fetching import MethodProvenance
 from pipelex.pipeline.pipeline_response import PipelexRunResultExecute, PipelexRunResultStart
@@ -25,6 +26,9 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_valida
 from pydantic.functional_validators import SkipValidation
 
 from pipelex_api.limits import (
+    DEFAULT_MAX_RESOURCE_FILE_KIB,
+    DEFAULT_MAX_RESOURCES_PER_REQUEST,
+    DEFAULT_MAX_RESOURCES_TOTAL_KIB,
     MAX_CALLBACK_URL_LEN,
     MAX_CALLBACK_URLS,
     MAX_METHOD_REF_LEN,
@@ -41,6 +45,21 @@ def _ensure_mthds_file_within_bytes_limit(content: str) -> None:
         raise ValueError(msg)
 
 
+RESOURCES_FIELD_DESCRIPTION = (
+    "The files submitted with the bundle that its steps read, such as a PipeDocGen step's `template_file` (an "
+    "`.xlsx`, `.docx`, `.pptx` or `.html` template): a JSON object mapping each file's bundle-relative path, exactly "
+    "as the step names it, to the file's bytes in standard base64. Every bundle of the request is read as sitting in "
+    "one directory. A path must be relative and written with forward slashes, with no `..` segment, backslash or "
+    "colon. Every file must be named by a step, or the request is refused with a 422 `BundleResourceNotNamedError`. "
+    f"By default at most {DEFAULT_MAX_RESOURCES_PER_REQUEST} files, {DEFAULT_MAX_RESOURCE_FILE_KIB} KiB each and "
+    f"{DEFAULT_MAX_RESOURCES_TOTAL_KIB} KiB in all, decoded, counting the files a method bundle ships for its steps; a "
+    "deployment sets its own with `MAX_RESOURCES_PER_REQUEST`, `MAX_RESOURCE_FILE_KIB` and `MAX_RESOURCES_TOTAL_KIB`. A file "
+    "both sent here and shipped in a method bundle must be the same. The `.html` template files, which travel inline with the "
+    f"method wherever it runs, may total {MAX_LOAD_STAGE_RESOURCES_BYTES // 1024} KiB at most, a fixed bound, or the request "
+    "is refused with a 413 `BundleResourcesTooLargeError`."
+)
+
+
 class RunRequest(BaseModel):
     """Body of `POST /execute` — this server's typed request model.
 
@@ -83,6 +102,9 @@ class RunRequest(BaseModel):
             by manifest identity); mutually exclusive with `mthds_contents` and with a
             method bundle. `pipe_code` may accompany it to override the manifest's
             `main_pipe`.
+        resources: The files the submitted bundle's steps read, such as a PipeDocGen
+            step's `template_file`, as `{bundle_relative_path: base64}`. Accompanies inline
+            `mthds_contents` or a method bundle, never a `method_ref`.
     """
 
     model_config = ConfigDict(extra="allow")
@@ -127,6 +149,7 @@ class RunRequest(BaseModel):
             "MTHDS concepts and sandboxed PipeFuncs, not in-process Python."
         ),
     )
+    resources: dict[str, str] | None = Field(default=None, description=RESOURCES_FIELD_DESCRIPTION)
 
     @model_validator(mode="before")
     @classmethod
@@ -155,6 +178,15 @@ class RunRequest(BaseModel):
         if has_method_ref and has_bundle:
             msg = "method_ref and a method bundle (bundle_b64 / files) are mutually exclusive; send one or the other."
             raise PipelineRequestError(msg)
+        # `resources` are the files the submitted bundle's steps read, so they accompany a bundle the request
+        # carries: inline contents or a method bundle. A fetched package's files are its own.
+        if values.get("resources"):
+            if has_method_ref:
+                msg = "resources accompany inline mthds_contents or a method bundle; a method_ref's files are the package's own."
+                raise PipelineRequestError(msg)
+            if not values.get("mthds_contents") and not values.get("mthds_content") and not has_bundle:
+                msg = "resources accompany the bundle whose steps read them: send them with mthds_contents or a method bundle."
+                raise PipelineRequestError(msg)
         if values.get("pipe_code") is None and not values.get("mthds_contents") and not has_bundle and not has_method_ref:
             msg = (
                 "pipe_code and mthds_contents cannot both be empty. Either: both are provided, or if there are no mthds_contents, "
@@ -231,6 +263,7 @@ class RunRequest(BaseModel):
             bundle_b64=request_body.get("bundle_b64"),
             files=request_body.get("files"),
             method_ref=request_body.get("method_ref"),
+            resources=request_body.get("resources"),
         )
 
 
@@ -565,11 +598,16 @@ class MthdsFilesRequest(BaseModel):
         ),
     )
 
+    resources: dict[str, str] | None = Field(default=None, description=RESOURCES_FIELD_DESCRIPTION)
+
     @model_validator(mode="after")
     def _exactly_one_selector(self) -> Self:
         if (self.files is None) == (self.method_ref is None):
             msg = "provide exactly one of `files` or `method_ref`"
             raise ValueError(msg)
+        if self.resources and self.method_ref is not None:
+            msg = "resources accompany inline `files`; a method_ref's files are the package's own"
+            raise ValueError(msg)
         return self