pipelex-api 0.71.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pipelex_api/__init__.py +0 -0
- pipelex_api/api.toml +21 -0
- pipelex_api/api_config.py +144 -0
- pipelex_api/bundle.py +243 -0
- pipelex_api/disclosure.py +43 -0
- pipelex_api/error_types.py +80 -0
- pipelex_api/error_uri.py +45 -0
- pipelex_api/errors.py +130 -0
- pipelex_api/exception_handlers.py +693 -0
- pipelex_api/json_body.py +182 -0
- pipelex_api/limits.py +67 -0
- pipelex_api/main.py +221 -0
- pipelex_api/method_cache.py +241 -0
- pipelex_api/method_source.py +215 -0
- pipelex_api/middleware.py +209 -0
- pipelex_api/openapi_responses.py +186 -0
- pipelex_api/openapi_schema.py +83 -0
- pipelex_api/problem_document.py +134 -0
- pipelex_api/py.typed +0 -0
- pipelex_api/routes/__init__.py +23 -0
- pipelex_api/routes/health.py +24 -0
- pipelex_api/routes/pipelex/__init__.py +21 -0
- pipelex_api/routes/pipelex/agent/__init__.py +11 -0
- pipelex_api/routes/pipelex/agent/concept.py +60 -0
- pipelex_api/routes/pipelex/agent/models.py +49 -0
- pipelex_api/routes/pipelex/agent/pipe_spec.py +59 -0
- pipelex_api/routes/pipelex/build/__init__.py +11 -0
- pipelex_api/routes/pipelex/build/inputs.py +192 -0
- pipelex_api/routes/pipelex/build/output.py +163 -0
- pipelex_api/routes/pipelex/build/runner.py +236 -0
- pipelex_api/routes/pipelex/codegen.py +164 -0
- pipelex_api/routes/pipelex/crate_ops.py +331 -0
- pipelex_api/routes/pipelex/pipe_io.py +186 -0
- pipelex_api/routes/pipelex/pipeline.py +938 -0
- pipelex_api/routes/pipelex/resolve.py +81 -0
- pipelex_api/routes/pipelex/tools.py +111 -0
- pipelex_api/routes/pipelex/utils.py +6 -0
- pipelex_api/routes/pipelex/validate.py +473 -0
- pipelex_api/routes/version.py +51 -0
- pipelex_api/schemas/__init__.py +0 -0
- pipelex_api/schemas/models.py +653 -0
- pipelex_api/security.py +284 -0
- pipelex_api-0.71.0.dist-info/METADATA +188 -0
- pipelex_api-0.71.0.dist-info/RECORD +46 -0
- pipelex_api-0.71.0.dist-info/WHEEL +4 -0
- pipelex_api-0.71.0.dist-info/licenses/LICENSE +95 -0
pipelex_api/__init__.py
ADDED
|
File without changes
|
pipelex_api/api.toml
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Pipelex-API deployment config — the packaged default, loaded by `load_api_config()`
|
|
2
|
+
# (pipelex_api/api_config.py) via core's env-aware `load_plugin_config` and validated into `ApiConfig`.
|
|
3
|
+
# Keys live at the file root (no `[api]` wrapper) — `load_plugin_config` validates the whole merged
|
|
4
|
+
# document against the schema, exactly like `pipelex-temporal`'s `temporal.toml`. The packaged
|
|
5
|
+
# default here is deep-merged with an optional `api_{env}.toml` / `api_override.toml` at `~/.pipelex`
|
|
6
|
+
# then the project `.pipelex` (env selected by `PIPELEX_ENV`). This source-available base names NO
|
|
7
|
+
# orchestrator: it ships the in-process `direct` mode and refuses per-request override. A deployment
|
|
8
|
+
# flavor bakes its own `.pipelex/api_{env}.toml` to flip this (e.g. `pipelex-api-hosted` sets
|
|
9
|
+
# `orchestration_mode = "temporal"`).
|
|
10
|
+
|
|
11
|
+
# Which orchestrator a top-level run dispatches to, through the orchestrator registry.
|
|
12
|
+
# `orchestration_mode` is an OPEN string token: core owns "direct" (in-process); each orchestrator
|
|
13
|
+
# plugin owns its own ("temporal" from pipelex-temporal, "mistral-workflows" from
|
|
14
|
+
# pipelex-mistral-workflows). A token whose plugin is not installed fails loud at dispatch with the
|
|
15
|
+
# plugin's install hint. The delivery axis (blocking vs fire-and-forget) is NOT configured here — it
|
|
16
|
+
# is set by the endpoint (`/execute` and `/validate` block; `/start` is fire-and-forget).
|
|
17
|
+
orchestration_mode = "direct"
|
|
18
|
+
|
|
19
|
+
# Whether a caller may override `orchestration_mode` per request. Off on the base (and recommended off
|
|
20
|
+
# on hosted flavors): a locked-down distributed runner must not be coercible into `direct`.
|
|
21
|
+
allow_request_orchestration_mode_override = false
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""Pipelex-API deployment config: the top-level orchestration mode + override policy.
|
|
2
|
+
|
|
3
|
+
The runner is orchestrator-agnostic. WHICH orchestrator a top-level run dispatches
|
|
4
|
+
to — ``direct`` in-process (the base default), ``temporal``, ``mistral-workflows``,
|
|
5
|
+
… — is a *deployment* choice, never a property of this source-available base.
|
|
6
|
+
``orchestration_mode`` is an open string token (core owns ``"direct"``; each plugin
|
|
7
|
+
owns its own); the *delivery* axis (blocking vs fire-and-forget) is endpoint-set, not
|
|
8
|
+
configured here. It is read from a packaged ``api.toml`` (keys at the file root — no
|
|
9
|
+
``[api]`` wrapper, since :meth:`load_plugin_config` validates the whole document
|
|
10
|
+
against the schema, exactly like ``temporal.toml``), env-layered like the main pipelex
|
|
11
|
+
config and every plugin config (D2): the packaged default ``api.toml`` (shipped in this
|
|
12
|
+
wheel) is deep-merged with the env-selected ``api_{environment}.toml`` and
|
|
13
|
+
``api_override.toml`` from ``~/.pipelex`` then the project ``.pipelex``, with
|
|
14
|
+
``PIPELEX_ENV`` (``runtime_manager.environment``) choosing the env file. One image
|
|
15
|
+
bakes every env file; a deployment flavor (e.g. ``pipelex-api-hosted``) bakes
|
|
16
|
+
``.pipelex/api_{env}.toml`` to flip the default. The base names no orchestrator and
|
|
17
|
+
ships ``orchestration_mode = "direct"``.
|
|
18
|
+
|
|
19
|
+
Why a separate ``api.toml`` and not the core ``pipelex_{env}.toml``: core's
|
|
20
|
+
config is ``extra="forbid"``, so an ``[api]`` section there is rejected at load.
|
|
21
|
+
Loading it via core's reusable :meth:`load_plugin_config` keeps this a pure
|
|
22
|
+
pipelex-api concern while reusing the identical env-layering machinery — and
|
|
23
|
+
keeps it symmetric with how ``pipelex-temporal`` self-loads ``temporal.toml``.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from functools import cache
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
from pipelex.runtime_bridge.orchestration_mode import DIRECT_ORCHESTRATION_MODE
|
|
30
|
+
from pipelex.system.configuration.config_loader import config_manager
|
|
31
|
+
from pydantic import BaseModel, ConfigDict
|
|
32
|
+
|
|
33
|
+
from pipelex_api.error_types import ErrorType
|
|
34
|
+
from pipelex_api.errors import raise_forbidden
|
|
35
|
+
|
|
36
|
+
API_CONFIG_NAME = "api"
|
|
37
|
+
|
|
38
|
+
# The packaged default ``api.toml`` ships in the wheel alongside this module.
|
|
39
|
+
_PACKAGE_DIR = Path(__file__).resolve().parent
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class ApiConfig(BaseModel):
|
|
43
|
+
"""The ``[api]`` deployment config: default orchestration mode + override policy.
|
|
44
|
+
|
|
45
|
+
No field defaults — the packaged ``api.toml`` is the single source of the
|
|
46
|
+
base defaults (mirroring core's "defaults live in the TOML, never in the
|
|
47
|
+
model" discipline). ``extra="forbid"`` so a typo'd key in a baked override
|
|
48
|
+
fails loud at load instead of being silently ignored.
|
|
49
|
+
|
|
50
|
+
``orchestration_mode`` is an open string token (core owns ``"direct"``; each
|
|
51
|
+
plugin owns its own). It is NOT validated against a closed enum here — an
|
|
52
|
+
unregistered token is refused at dispatch by ``MissingOrchestratorError``,
|
|
53
|
+
the single validation point. The delivery axis (blocking vs fire-and-forget)
|
|
54
|
+
is endpoint-set, never configured, so nothing about wait-semantics lives here.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
model_config = ConfigDict(extra="forbid")
|
|
58
|
+
|
|
59
|
+
orchestration_mode: str
|
|
60
|
+
allow_request_orchestration_mode_override: bool
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def load_api_config() -> ApiConfig:
|
|
64
|
+
"""Load the ``[api]`` config from ``api.toml`` with env-aware layering (D2).
|
|
65
|
+
|
|
66
|
+
Delegates to core's reusable plugin-config loader: the packaged ``api.toml``
|
|
67
|
+
is deep-merged with the env-selected overrides. The packaged default alone is
|
|
68
|
+
a valid, fully-resolved config — every override tier is optional. Requires
|
|
69
|
+
Pipelex to be booted (``runtime_manager.environment`` must be resolved), so
|
|
70
|
+
it is called only after ``Pipelex.make`` — never at import.
|
|
71
|
+
"""
|
|
72
|
+
return config_manager.load_plugin_config(name=API_CONFIG_NAME, package_dir=_PACKAGE_DIR, schema=ApiConfig)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@cache
|
|
76
|
+
def get_api_config() -> ApiConfig:
|
|
77
|
+
"""Process-cached :class:`ApiConfig`.
|
|
78
|
+
|
|
79
|
+
The config is immutable for the life of the process (``PIPELEX_ENV`` is fixed
|
|
80
|
+
at boot), so it is loaded once and cached. ``pipelex_api.main`` warms this at startup
|
|
81
|
+
so a malformed ``api.toml`` / baked override fails the app fast — the same
|
|
82
|
+
fail-fast posture as ``ERROR_DISCLOSURE``. Tests that need a different mode
|
|
83
|
+
patch this getter (or call :func:`resolve_orchestration_mode` with a hand-built
|
|
84
|
+
config) rather than mutating the cache.
|
|
85
|
+
"""
|
|
86
|
+
return load_api_config()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def resolve_orchestration_mode(requested: str | None, *, config: ApiConfig) -> str:
|
|
90
|
+
"""Resolve the effective orchestration mode for a top-level run, applying policy.
|
|
91
|
+
|
|
92
|
+
The deployment default (``config.orchestration_mode``) wins unless the caller
|
|
93
|
+
supplied a *different* token AND the deployment opted into per-request
|
|
94
|
+
override (``allow_request_orchestration_mode_override``). A caller-supplied
|
|
95
|
+
token equal to the default is always honored (it changes nothing). A caller
|
|
96
|
+
trying to FORCE a different mode on a runner whose policy forbids it is refused
|
|
97
|
+
with a 403 — so a locked-down Temporal runner can never be coerced into
|
|
98
|
+
``direct`` (whose whole point would be to bypass distributed execution), and
|
|
99
|
+
vice versa. The token is a plain string compare; an *unregistered* token is
|
|
100
|
+
not rejected here — that surfaces at dispatch as ``MissingOrchestratorError``.
|
|
101
|
+
"""
|
|
102
|
+
if requested is None or requested == config.orchestration_mode:
|
|
103
|
+
return config.orchestration_mode
|
|
104
|
+
if config.allow_request_orchestration_mode_override:
|
|
105
|
+
return requested
|
|
106
|
+
msg = (
|
|
107
|
+
f"This deployment does not allow overriding orchestration_mode per request "
|
|
108
|
+
f"(configured mode '{config.orchestration_mode}', requested '{requested}')."
|
|
109
|
+
)
|
|
110
|
+
raise_forbidden(msg, error_type=ErrorType.ORCHESTRATION_MODE_OVERRIDE_FORBIDDEN)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class ApiBootConfigError(ValueError):
|
|
114
|
+
"""Raised at startup when the deployment's orchestration config cannot boot coherently."""
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def resolve_boot_orchestrator(config: ApiConfig) -> str | None:
|
|
118
|
+
"""The orchestrator plugin this process boots under, derived from the deployment config.
|
|
119
|
+
|
|
120
|
+
The base ``direct`` mode names no orchestrator and boots in-process (``None``); any other
|
|
121
|
+
mode (a plugin token like ``"temporal"``) boots the process under that orchestrator so its
|
|
122
|
+
execution-hub slots are claimed and async dispatch is enabled.
|
|
123
|
+
|
|
124
|
+
A process boots under exactly one orchestrator, so a ``direct`` default that ALSO enables
|
|
125
|
+
per-request override is incoherent: no async hub is claimed at boot, yet
|
|
126
|
+
:func:`resolve_orchestration_mode` would honor a request overriding to a non-direct mode and
|
|
127
|
+
resolve that orchestrator's dispatch arm — which then fails at dispatch with
|
|
128
|
+
``AsyncExecutionNotEnabledError``. Refuse it here: fail loud at boot, where the operator sees
|
|
129
|
+
it, rather than on the first overriding request. The mirror case (a non-direct default with
|
|
130
|
+
override on) IS coherent — the async hub is claimed at boot, and a per-request ``direct``
|
|
131
|
+
override still runs in-process — so it boots normally under that orchestrator.
|
|
132
|
+
"""
|
|
133
|
+
mode = config.orchestration_mode
|
|
134
|
+
if config.allow_request_orchestration_mode_override and mode == DIRECT_ORCHESTRATION_MODE:
|
|
135
|
+
msg = (
|
|
136
|
+
"allow_request_orchestration_mode_override=true with a 'direct' orchestration_mode default "
|
|
137
|
+
"cannot service a non-direct per-request override: no async execution hub is claimed at boot, "
|
|
138
|
+
"so such a request would fail at dispatch. Set a non-direct orchestration_mode default or "
|
|
139
|
+
"disable per-request override."
|
|
140
|
+
)
|
|
141
|
+
raise ApiBootConfigError(msg)
|
|
142
|
+
if mode == DIRECT_ORCHESTRATION_MODE:
|
|
143
|
+
return None
|
|
144
|
+
return mode
|
pipelex_api/bundle.py
ADDED
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
"""Materialize a caller-supplied method bundle into a temporary library directory.
|
|
2
|
+
|
|
3
|
+
A run request may carry the whole method — the `.mthds` bundle plus its PipeFunc
|
|
4
|
+
Python (`pipe_func.py`) and a `requirements.txt` — instead of only
|
|
5
|
+
the inline `mthds_contents` text. Two transport forms are accepted, exactly one
|
|
6
|
+
per request:
|
|
7
|
+
|
|
8
|
+
- `bundle_b64`: a base64-encoded zip archive of the bundle directory.
|
|
9
|
+
- `files`: a `{relative_path: text_content}` map (the zip's contents, unzipped).
|
|
10
|
+
|
|
11
|
+
The two are equivalent: `files` ≡ the zip's entries. This module decodes either
|
|
12
|
+
form, enforces the ingest guards (both transport forms are refused together; a
|
|
13
|
+
hard file-count and total-size ceiling; per-entry path-safety against absolute
|
|
14
|
+
paths and `..` traversal; a zip-bomb guard that bounds actual decompression),
|
|
15
|
+
writes the surviving files into a fresh temp directory, and hands that directory
|
|
16
|
+
back so the runner can load it via `library_dirs`. In a sandbox-hosted
|
|
17
|
+
deployment the load path reads every `.py` as source text, never importing it:
|
|
18
|
+
it refuses a bundle whose Python declares a structure class
|
|
19
|
+
(`MethodStructuresRefusedError`, a 403) and captures the rest onto the crate for
|
|
20
|
+
the sandbox. The caller is responsible for the hosted-mode gate.
|
|
21
|
+
|
|
22
|
+
Nothing here imports or executes the bundle's Python — it only writes bytes to
|
|
23
|
+
disk. The caller cleans the directory up via the `materialized_bundle` context
|
|
24
|
+
manager's guaranteed teardown.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import base64
|
|
30
|
+
import binascii
|
|
31
|
+
import shutil
|
|
32
|
+
import tempfile
|
|
33
|
+
import zipfile
|
|
34
|
+
from contextlib import contextmanager
|
|
35
|
+
from io import BytesIO
|
|
36
|
+
from pathlib import Path, PurePosixPath
|
|
37
|
+
from typing import TYPE_CHECKING, NamedTuple
|
|
38
|
+
|
|
39
|
+
from pipelex import log
|
|
40
|
+
from pydantic import ConfigDict
|
|
41
|
+
from pydantic.dataclasses import dataclass
|
|
42
|
+
|
|
43
|
+
from pipelex_api.error_types import ErrorType
|
|
44
|
+
from pipelex_api.errors import raise_bad_request, raise_payload_too_large, raise_validation_error
|
|
45
|
+
from pipelex_api.limits import MAX_BUNDLE_FILES, MAX_BUNDLE_TOTAL_BYTES
|
|
46
|
+
|
|
47
|
+
if TYPE_CHECKING:
|
|
48
|
+
from collections.abc import Generator
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(frozen=True, config=ConfigDict(arbitrary_types_allowed=True))
|
|
52
|
+
class MaterializedBundle:
|
|
53
|
+
"""A bundle written to disk: the directory to load and the relpaths written."""
|
|
54
|
+
|
|
55
|
+
directory: Path
|
|
56
|
+
relpaths: tuple[str, ...]
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def has_python_sources(self) -> bool:
|
|
60
|
+
"""True when the bundle ships any `.py` — the trigger for the hosted-mode gate."""
|
|
61
|
+
return any(relpath.endswith(".py") for relpath in self.relpaths)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _safe_relpath(name: str) -> PurePosixPath:
|
|
65
|
+
"""Validate one bundle entry name and return it as a normalized relative POSIX path.
|
|
66
|
+
|
|
67
|
+
Rejects anything that could escape the destination directory: absolute paths,
|
|
68
|
+
Windows drive/backslash forms (a bare drive prefix like `C:foo` has no slash or
|
|
69
|
+
backslash yet is drive-relative on Windows, so `:` is rejected outright), and any
|
|
70
|
+
`..` component. Directory-only entries (trailing slash) return an empty path and
|
|
71
|
+
are filtered by the caller.
|
|
72
|
+
"""
|
|
73
|
+
if not name or name in {".", "./"}:
|
|
74
|
+
return PurePosixPath()
|
|
75
|
+
if "\\" in name:
|
|
76
|
+
msg = f"Bundle entry {name!r} uses backslashes; use forward-slash relative paths only"
|
|
77
|
+
raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
|
|
78
|
+
if ":" in name:
|
|
79
|
+
msg = f"Bundle entry {name!r} contains ':' (a Windows drive/stream form); use plain relative paths only"
|
|
80
|
+
raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
|
|
81
|
+
pure = PurePosixPath(name)
|
|
82
|
+
if pure.is_absolute():
|
|
83
|
+
msg = f"Bundle entry {name!r} is an absolute path; only relative paths are allowed"
|
|
84
|
+
raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
|
|
85
|
+
if any(part == ".." for part in pure.parts):
|
|
86
|
+
msg = f"Bundle entry {name!r} escapes the bundle root via '..'; not allowed"
|
|
87
|
+
raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
|
|
88
|
+
return pure
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _guard_count(count: int) -> None:
|
|
92
|
+
if count == 0:
|
|
93
|
+
raise_validation_error(message="Bundle is empty (no files)", error_type=ErrorType.INVALID_BUNDLE)
|
|
94
|
+
if count > MAX_BUNDLE_FILES:
|
|
95
|
+
raise_payload_too_large(message=f"Bundle exceeds the {MAX_BUNDLE_FILES}-file limit (got {count})")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _guard_running_total(total_bytes: int) -> None:
|
|
99
|
+
if total_bytes > MAX_BUNDLE_TOTAL_BYTES:
|
|
100
|
+
raise_payload_too_large(message=f"Bundle exceeds the {MAX_BUNDLE_TOTAL_BYTES // 1024} KiB decompressed-size limit")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# Base64 inflates by 4/3; a zip is compressed, so a bundle whose DECOMPRESSED content is within
|
|
104
|
+
# the ceiling encodes to well under this. Bounding the base64 string BEFORE decoding stops a
|
|
105
|
+
# ~100 MiB request-body (the only other bound) from being expanded into ~75 MiB of heap just to
|
|
106
|
+
# be rejected later by the decompressed-size guard. Slack (+4) covers padding.
|
|
107
|
+
_MAX_BUNDLE_B64_CHARS = MAX_BUNDLE_TOTAL_BYTES * 4 // 3 + 4
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _entries_from_zip(bundle_b64: str) -> list[tuple[PurePosixPath, bytes]]:
|
|
111
|
+
"""Decode a base64 zip and return its (safe relpath, bytes) file entries.
|
|
112
|
+
|
|
113
|
+
Zip-bomb guard: each member is read through a bounded stream so a lying
|
|
114
|
+
uncompressed-size header cannot force unbounded decompression — the running
|
|
115
|
+
total is checked against `MAX_BUNDLE_TOTAL_BYTES` as bytes are pulled. A cheap
|
|
116
|
+
length check on the still-encoded string runs FIRST, so an oversized payload is
|
|
117
|
+
refused before it is buffered into memory as decoded bytes.
|
|
118
|
+
"""
|
|
119
|
+
if len(bundle_b64) > _MAX_BUNDLE_B64_CHARS:
|
|
120
|
+
raise_payload_too_large(message=f"bundle_b64 exceeds the {MAX_BUNDLE_TOTAL_BYTES // 1024} KiB compressed-size limit")
|
|
121
|
+
try:
|
|
122
|
+
raw = base64.b64decode(bundle_b64, validate=True)
|
|
123
|
+
except (binascii.Error, ValueError) as decode_error:
|
|
124
|
+
log.warning(f"bundle: invalid base64 ({decode_error})")
|
|
125
|
+
raise_bad_request(message="bundle_b64 is not valid base64", error_type=ErrorType.INVALID_BASE64)
|
|
126
|
+
|
|
127
|
+
try:
|
|
128
|
+
archive = zipfile.ZipFile(BytesIO(raw))
|
|
129
|
+
except zipfile.BadZipFile as zip_error:
|
|
130
|
+
log.warning(f"bundle: corrupt zip ({zip_error})")
|
|
131
|
+
raise_validation_error(message="bundle_b64 is not a valid zip archive", error_type=ErrorType.INVALID_BUNDLE)
|
|
132
|
+
|
|
133
|
+
entries: list[tuple[PurePosixPath, bytes]] = []
|
|
134
|
+
total_bytes = 0
|
|
135
|
+
with archive:
|
|
136
|
+
members = [info for info in archive.infolist() if not info.is_dir()]
|
|
137
|
+
_guard_count(len(members))
|
|
138
|
+
budget = MAX_BUNDLE_TOTAL_BYTES
|
|
139
|
+
for info in members:
|
|
140
|
+
relpath = _safe_relpath(info.filename)
|
|
141
|
+
if not relpath.parts:
|
|
142
|
+
continue
|
|
143
|
+
# Bounded read: pull at most (remaining budget + 1) bytes so a zip bomb whose header
|
|
144
|
+
# under-reports its size still cannot decompress past the ceiling.
|
|
145
|
+
with archive.open(info) as member:
|
|
146
|
+
data = member.read(budget + 1)
|
|
147
|
+
total_bytes += len(data)
|
|
148
|
+
_guard_running_total(total_bytes)
|
|
149
|
+
budget = MAX_BUNDLE_TOTAL_BYTES - total_bytes
|
|
150
|
+
entries.append((relpath, data))
|
|
151
|
+
return entries
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _entries_from_files(files: dict[str, str]) -> list[tuple[PurePosixPath, bytes]]:
|
|
155
|
+
"""Validate a {relpath: text} map and return its (safe relpath, bytes) entries."""
|
|
156
|
+
_guard_count(len(files))
|
|
157
|
+
entries: list[tuple[PurePosixPath, bytes]] = []
|
|
158
|
+
total_bytes = 0
|
|
159
|
+
for name, content in files.items():
|
|
160
|
+
relpath = _safe_relpath(name)
|
|
161
|
+
if not relpath.parts:
|
|
162
|
+
msg = f"Bundle entry {name!r} has no filename"
|
|
163
|
+
raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
|
|
164
|
+
data = content.encode("utf-8")
|
|
165
|
+
total_bytes += len(data)
|
|
166
|
+
_guard_running_total(total_bytes)
|
|
167
|
+
entries.append((relpath, data))
|
|
168
|
+
return entries
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class ParsedBundle(NamedTuple):
|
|
172
|
+
"""A decoded-and-validated bundle held in memory, NOT yet written to disk.
|
|
173
|
+
|
|
174
|
+
Splitting parse from materialize lets the caller apply the sandbox-hosted gate
|
|
175
|
+
(which keys on `has_python_sources`) BEFORE any disk write — so a bundle destined
|
|
176
|
+
for a 403 never touches the filesystem. Entries are `(safe relpath, bytes)`.
|
|
177
|
+
"""
|
|
178
|
+
|
|
179
|
+
entries: tuple[tuple[PurePosixPath, bytes], ...]
|
|
180
|
+
|
|
181
|
+
@property
|
|
182
|
+
def has_python_sources(self) -> bool:
|
|
183
|
+
"""True when the bundle ships any `.py` — the trigger for the hosted-mode gate."""
|
|
184
|
+
return any(str(relpath).endswith(".py") for relpath, _ in self.entries)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def parse_bundle(*, bundle_b64: str | None, files: dict[str, str] | None) -> ParsedBundle:
|
|
188
|
+
"""Decode + guard a bundle into an in-memory `ParsedBundle` (no disk writes).
|
|
189
|
+
|
|
190
|
+
Exactly one of `bundle_b64` / `files` must be supplied; supplying both is a
|
|
191
|
+
caller mistake (they are the same content in two forms) and is refused. All
|
|
192
|
+
ingest guards (base64, size, count, path-safety, zip-bomb) run here, so the
|
|
193
|
+
caller can inspect `has_python_sources` and reject BEFORE materializing to disk.
|
|
194
|
+
"""
|
|
195
|
+
if bundle_b64 is not None and files is not None:
|
|
196
|
+
msg = "Provide either bundle_b64 or files, not both"
|
|
197
|
+
raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
|
|
198
|
+
if bundle_b64 is not None:
|
|
199
|
+
entries = _entries_from_zip(bundle_b64)
|
|
200
|
+
elif files is not None:
|
|
201
|
+
entries = _entries_from_files(files)
|
|
202
|
+
else:
|
|
203
|
+
msg = "No bundle supplied (bundle_b64 and files are both absent)"
|
|
204
|
+
raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
|
|
205
|
+
return ParsedBundle(entries=tuple(entries))
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@contextmanager
|
|
209
|
+
def materialize_parsed(parsed: ParsedBundle) -> Generator[MaterializedBundle, None, None]:
|
|
210
|
+
"""Write an already-parsed bundle into a fresh temp directory, cleaned up on exit.
|
|
211
|
+
|
|
212
|
+
The yielded `MaterializedBundle.directory` is safe to pass as a `library_dirs`
|
|
213
|
+
entry; it is removed when the context exits, on both the happy and error path.
|
|
214
|
+
"""
|
|
215
|
+
directory = Path(tempfile.mkdtemp(prefix="pipelex-bundle-"))
|
|
216
|
+
try:
|
|
217
|
+
root = directory.resolve()
|
|
218
|
+
relpaths: list[str] = []
|
|
219
|
+
for relpath, data in parsed.entries:
|
|
220
|
+
target = (directory / relpath).resolve()
|
|
221
|
+
# Defense-in-depth: even after per-part validation, confirm the resolved target stays
|
|
222
|
+
# under the temp root before writing (guards against symlink/edge normalization surprises).
|
|
223
|
+
if root != target and root not in target.parents:
|
|
224
|
+
msg = f"Bundle entry {str(relpath)!r} resolves outside the bundle root"
|
|
225
|
+
raise_validation_error(message=msg, error_type=ErrorType.INVALID_BUNDLE)
|
|
226
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
227
|
+
target.write_bytes(data)
|
|
228
|
+
relpaths.append(relpath.as_posix())
|
|
229
|
+
yield MaterializedBundle(directory=directory, relpaths=tuple(relpaths))
|
|
230
|
+
finally:
|
|
231
|
+
shutil.rmtree(directory, ignore_errors=True)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@contextmanager
|
|
235
|
+
def materialized_bundle(*, bundle_b64: str | None, files: dict[str, str] | None) -> Generator[MaterializedBundle, None, None]:
|
|
236
|
+
"""Parse AND materialize a bundle in one step (convenience for callers that don't gate).
|
|
237
|
+
|
|
238
|
+
Equivalent to `parse_bundle(...)` followed by `materialize_parsed(...)`; a caller
|
|
239
|
+
that must apply the sandbox-hosted gate before disk writes should use the two
|
|
240
|
+
steps directly instead.
|
|
241
|
+
"""
|
|
242
|
+
with materialize_parsed(parse_bundle(bundle_b64=bundle_b64, files=files)) as bundle:
|
|
243
|
+
yield bundle
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Resolution of the `ERROR_DISCLOSURE` environment variable.
|
|
2
|
+
|
|
3
|
+
`ERROR_DISCLOSURE` selects how much of an error report reaches the client:
|
|
4
|
+
|
|
5
|
+
- `verbose` (default) — the full report, including the human-readable message.
|
|
6
|
+
The intended default for a self-hosted server, where the operator and the
|
|
7
|
+
debugging developer are usually the same person.
|
|
8
|
+
- `strict` — the lossy projection pipelex's `DisclosureMode.STRICT` produces,
|
|
9
|
+
for hosted multi-tenant deployments.
|
|
10
|
+
|
|
11
|
+
The value is resolved once, at app startup (`pipelex_api.main`). An unrecognized value
|
|
12
|
+
fails the app at boot rather than silently degrading — a misconfigured
|
|
13
|
+
disclosure mode is a security-relevant mistake the operator must see.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from pipelex.base_exceptions import DisclosureMode
|
|
17
|
+
from pipelex.system.environment import get_optional_env
|
|
18
|
+
|
|
19
|
+
ERROR_DISCLOSURE_ENV_VAR = "ERROR_DISCLOSURE"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class InvalidErrorDisclosureError(ValueError):
|
|
23
|
+
"""Raised at startup when `ERROR_DISCLOSURE` holds an unrecognized value."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def resolve_disclosure_mode() -> DisclosureMode:
|
|
27
|
+
"""Resolve `ERROR_DISCLOSURE` to a `DisclosureMode`.
|
|
28
|
+
|
|
29
|
+
Returns `DisclosureMode.VERBOSE` when the variable is unset, empty, or
|
|
30
|
+
whitespace-only. Raises `InvalidErrorDisclosureError` for any value other
|
|
31
|
+
than `verbose` or `strict` (matched case-insensitively, surrounding
|
|
32
|
+
whitespace ignored).
|
|
33
|
+
"""
|
|
34
|
+
raw = get_optional_env(ERROR_DISCLOSURE_ENV_VAR)
|
|
35
|
+
normalized = (raw or "").strip().lower()
|
|
36
|
+
if not normalized:
|
|
37
|
+
return DisclosureMode.VERBOSE
|
|
38
|
+
try:
|
|
39
|
+
return DisclosureMode(normalized)
|
|
40
|
+
except ValueError as exc:
|
|
41
|
+
valid = ", ".join(f"'{mode}'" for mode in DisclosureMode)
|
|
42
|
+
msg = f"{ERROR_DISCLOSURE_ENV_VAR}={raw!r} is not a valid disclosure mode. Valid values: {valid}."
|
|
43
|
+
raise InvalidErrorDisclosureError(msg) from exc
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Centralized `error_type` symbols for API-authored error responses.
|
|
2
|
+
|
|
3
|
+
Every fixed (non-dynamic) `error_type` an API-authored error emits lives here
|
|
4
|
+
so call sites reference symbols, not literals — easier to grep, rename, and
|
|
5
|
+
document. The `pipelex_api.errors` helpers stamp one of these onto the RFC 7807
|
|
6
|
+
problem document as the `error_type` extension member.
|
|
7
|
+
|
|
8
|
+
Pipelex domain errors carry their own open-ended `error_type` (the exception
|
|
9
|
+
class name, from the `ErrorReport`) and are NOT enumerated here — they are
|
|
10
|
+
deliberately not a fixed set.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from enum import StrEnum
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ErrorType(StrEnum):
|
|
17
|
+
# Authentication / authorization
|
|
18
|
+
UNAUTHENTICATED = "Unauthenticated"
|
|
19
|
+
FORBIDDEN = "Forbidden"
|
|
20
|
+
# A caller asked to run in an orchestration_mode this deployment forbids overriding
|
|
21
|
+
# (per-request override is off — see `allow_request_orchestration_mode_override` in api.toml).
|
|
22
|
+
# A 403: the deployment policy refuses to honor the requested mode.
|
|
23
|
+
ORCHESTRATION_MODE_OVERRIDE_FORBIDDEN = "OrchestrationModeOverrideForbidden"
|
|
24
|
+
INVALID_TOKEN = "InvalidToken"
|
|
25
|
+
TOKEN_EXPIRED = "TokenExpired"
|
|
26
|
+
SERVER_MISCONFIGURED = "ServerMisconfigured"
|
|
27
|
+
|
|
28
|
+
# A caller hit `/start` on a deployment whose resolved orchestration mode cannot do genuine
|
|
29
|
+
# async (its orchestrator's `supports_fire_and_forget` is False — e.g. the in-process `direct`
|
|
30
|
+
# base). `/start` is fire-and-forget by nature, so rather than silently running blocking and
|
|
31
|
+
# acking, it refuses HONESTLY with a 400: use `/execute` (synchronous) instead. Checked AFTER
|
|
32
|
+
# the override policy, so a forbidden per-request override still 403s first.
|
|
33
|
+
START_REQUIRES_ASYNC_ORCHESTRATION = "StartRequiresAsyncOrchestration"
|
|
34
|
+
|
|
35
|
+
# Request validation
|
|
36
|
+
BAD_REQUEST = "BadRequest"
|
|
37
|
+
VALIDATION_ERROR = "ValidationError"
|
|
38
|
+
INVALID_JSON = "InvalidJSON"
|
|
39
|
+
# A run request's body carries, at any depth, an object key a kajson decoder reads as a class
|
|
40
|
+
# marker: `__class__`, `__module__`, or anything starting with `__kajson`. A 422, not data: the
|
|
41
|
+
# run body is plain JSON, and such a key would make a later kajson round trip of the inputs
|
|
42
|
+
# import and instantiate whatever class the caller named.
|
|
43
|
+
RESERVED_OBJECT_KEY = "ReservedObjectKey"
|
|
44
|
+
INVALID_CALLBACK_URLS = "InvalidCallbackUrls"
|
|
45
|
+
# A run request's `storage_scope` extra is not one to three path-safe segments. A 422, not a
|
|
46
|
+
# 500 from deep in the run: the value becomes a storage key prefix, so a `..` escapes the tenant.
|
|
47
|
+
INVALID_STORAGE_SCOPE = "InvalidStorageScope"
|
|
48
|
+
# A run request's `read_scope` extra is not one to three path-safe segments, or the run's storage
|
|
49
|
+
# scope does not lie under it, so the run could not read back what it writes.
|
|
50
|
+
INVALID_READ_SCOPE = "InvalidReadScope"
|
|
51
|
+
# A run request's `analytics_groups` extra is not a mapping of group type to group key within
|
|
52
|
+
# the runtime's charset and entry cap (`pipelex.system.run_extras`).
|
|
53
|
+
INVALID_ANALYTICS_GROUPS = "InvalidAnalyticsGroups"
|
|
54
|
+
INVALID_MODEL_CATEGORY = "InvalidModelCategory"
|
|
55
|
+
INVALID_BASE64 = "InvalidBase64"
|
|
56
|
+
PAYLOAD_TOO_LARGE = "PayloadTooLarge"
|
|
57
|
+
# A run request carried a malformed method bundle (`bundle_b64` / `files`): a corrupt zip, an
|
|
58
|
+
# unsafe entry name (absolute path or `..` traversal), or both transport forms supplied at once.
|
|
59
|
+
INVALID_BUNDLE = "InvalidBundle"
|
|
60
|
+
# A method carrying custom Python (`.py` in a bundle, or in a fetched `method_ref` package)
|
|
61
|
+
# reached a deployment that is NOT sandbox-hosted. Running customer code in-process is
|
|
62
|
+
# refused (403): custom code is a sandbox-hosted capability only. Use a sandbox-hosted
|
|
63
|
+
# deployment. (On a sandbox-hosted deployment, a bundle or a fetched package declaring Python
|
|
64
|
+
# STRUCTURE classes is still refused — that is pipelex's `MethodStructuresRefusedError`, not this.)
|
|
65
|
+
CUSTOM_CODE_REQUIRES_SANDBOX = "CustomCodeRequiresSandbox"
|
|
66
|
+
|
|
67
|
+
# A caller selected a closure by a REGISTRY-FORM `method_ref` (not a `github.com/...`
|
|
68
|
+
# address) on a tooling route. Address-form refs resolve server-side (fetch at tag, package
|
|
69
|
+
# located by manifest identity); the registry form is the packaging program's closing phase,
|
|
70
|
+
# so it keeps an honest 501 until a method registry exists — never a silent empty verdict.
|
|
71
|
+
METHOD_REF_NOT_SUPPORTED = "MethodRefNotSupported"
|
|
72
|
+
|
|
73
|
+
# Misc
|
|
74
|
+
PACKAGE_NOT_FOUND = "PackageNotFound"
|
|
75
|
+
# The `error_type` for the catch-all 500 emitted by `handle_unexpected_error`
|
|
76
|
+
# (any failure matched by no more-specific handler — not an `ApiError`, a
|
|
77
|
+
# `RequestValidationError`, a `PipelexError`, or an orchestrator plugin's mapped
|
|
78
|
+
# transport exception). Stays in this enum so the same `build_problem_document_from_api_error`
|
|
79
|
+
# builder renders it — same shape as every other API-authored 500.
|
|
80
|
+
INTERNAL_SERVER_ERROR = "InternalServerError"
|
pipelex_api/error_uri.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""RFC 7807 `type` URI and `title` for API-authored errors.
|
|
2
|
+
|
|
3
|
+
A pipelex `ErrorReport` always carries its own `type_uri` and `title` (set
|
|
4
|
+
upstream by `PipelexError.type_uri()` / `PipelexError.title()`, both required
|
|
5
|
+
fields), so the global exception handler never needs these helpers for a
|
|
6
|
+
`PipelexError`.
|
|
7
|
+
|
|
8
|
+
They exist for the *other* error source: the API's own 4xx responses
|
|
9
|
+
(`raise_validation_error` and friends in `pipelex_api.errors`), authored from the
|
|
10
|
+
static `pipelex_api.error_types.ErrorType` enum with no `ErrorReport` behind them.
|
|
11
|
+
`build_problem_document_from_api_error` uses these to give an API-authored
|
|
12
|
+
error the same RFC 7807 `type` / `title` shape as a pipelex one. They also
|
|
13
|
+
serve as a defensive backstop should a future pipelex ever hand back a report
|
|
14
|
+
with an absent identity pair.
|
|
15
|
+
|
|
16
|
+
The URI namespace and the kebab / humanize transforms are deliberately the
|
|
17
|
+
same ones pipelex uses (`URLs.error_docs_base`, `pascal_case_to_kebab`,
|
|
18
|
+
`pascal_case_to_sentence`), so an API-authored error and a pipelex error are
|
|
19
|
+
indistinguishable in shape on the wire. Both functions are pure.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from pipelex.tools.misc.string_utils import pascal_case_to_kebab, pascal_case_to_sentence
|
|
23
|
+
from pipelex.urls import URLs
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def error_type_uri(error_type: str) -> str:
|
|
27
|
+
"""Return the RFC 7807 `type` URI for an error type name.
|
|
28
|
+
|
|
29
|
+
`EnvVarNotFoundError` -> `https://docs.pipelex.com/latest/errors/env-var-not-found-error/`.
|
|
30
|
+
The trailing slash matches the canonical docs URL form, exactly as
|
|
31
|
+
`PipelexError.type_uri()` produces it upstream.
|
|
32
|
+
"""
|
|
33
|
+
return f"{URLs.error_docs_base}/{pascal_case_to_kebab(error_type)}/"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def error_type_title(error_type: str) -> str:
|
|
37
|
+
"""Return a human-readable RFC 7807 `title` for an error type name.
|
|
38
|
+
|
|
39
|
+
`ValidationError` -> `Validation error`; `BadRequest` -> `Bad request`.
|
|
40
|
+
Sentence case via the same `pascal_case_to_sentence` transform pipelex uses
|
|
41
|
+
in `PipelexError.title()`. Unlike `PipelexError.title()` it does NOT strip a
|
|
42
|
+
trailing `Error`: an API `ErrorType` such as `ValidationError` must keep the
|
|
43
|
+
suffix — `"Validation"` alone would be a worse label than `"Validation error"`.
|
|
44
|
+
"""
|
|
45
|
+
return pascal_case_to_sentence(error_type)
|