weft-kernel 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- weft_kernel/__init__.py +178 -0
- weft_kernel/blocking.py +370 -0
- weft_kernel/context.py +250 -0
- weft_kernel/discovery.py +1181 -0
- weft_kernel/errors.py +73 -0
- weft_kernel/fallback.py +159 -0
- weft_kernel/payload/__init__.py +43 -0
- weft_kernel/payload/applicability.py +298 -0
- weft_kernel/payload/ext.py +172 -0
- weft_kernel/payload/ids.py +22 -0
- weft_kernel/payload/lineage.py +90 -0
- weft_kernel/payload/media_type.py +18 -0
- weft_kernel/payload/node.py +260 -0
- weft_kernel/payload/outcome.py +40 -0
- weft_kernel/payload/property.py +60 -0
- weft_kernel/payload/vector.py +28 -0
- weft_kernel/pipeline.py +748 -0
- weft_kernel/py.typed +0 -0
- weft_kernel/registry.py +692 -0
- weft_kernel/resolution.py +1582 -0
- weft_kernel/runner.py +1436 -0
- weft_kernel/seam.py +725 -0
- weft_kernel-0.1.0.dist-info/METADATA +88 -0
- weft_kernel-0.1.0.dist-info/RECORD +27 -0
- weft_kernel-0.1.0.dist-info/WHEEL +4 -0
- weft_kernel-0.1.0.dist-info/licenses/LICENSE +21 -0
- weft_kernel-0.1.0.dist-info/licenses/NOTICE +77 -0
|
@@ -0,0 +1,1582 @@
|
|
|
1
|
+
"""`resolve()` — a pipeline document reduced to a frozen, fully-explicit form. Task 1.3.
|
|
2
|
+
|
|
3
|
+
Specified in `docs/02-extension-model.md` §3 → *Derivation* and *When resolution
|
|
4
|
+
fails*. `weft_kernel.pipeline.Pipeline` is what an author wrote: `extends` unfollowed,
|
|
5
|
+
`vars` unsubstituted, no plugin looked up. `weft_kernel.runner.RunnablePipeline` is
|
|
6
|
+
what a `Runner` executes: constructed plugin instances, cached per `Lifetime`, scoped
|
|
7
|
+
to a tenant. Neither is what `02` §3 means by "the resolved form" — a **data** value in
|
|
8
|
+
between the two, produced here: "Resolution produces a frozen, fully-explicit
|
|
9
|
+
pipeline: every stage, plugin, version and configuration value named, with no
|
|
10
|
+
inheritance left to interpret. That resolved form is what runs, what gets logged, and
|
|
11
|
+
what evaluation compares." `ResolvedPipeline` is a `BaseModel`, not a `dataclass`
|
|
12
|
+
wrapping live objects the way `RunnablePipeline` does — printable, diffable, loggable,
|
|
13
|
+
comparable by `==` across two separate calls to `resolve()`, which is exactly what a
|
|
14
|
+
constructed plugin instance can never promise. Building a `RunnablePipeline` from a
|
|
15
|
+
`ResolvedPipeline` — instantiating every `StageSpec` this module names — is a later
|
|
16
|
+
step's job, not this one's; nothing here calls a factory.
|
|
17
|
+
|
|
18
|
+
**Everything a resolved pipeline can be wrong about is wrong before it runs.** `02` §3
|
|
19
|
+
lists five checks; `weft_kernel.registry.Registry.entry` already does the first
|
|
20
|
+
(`UnknownPluginError`, reused unchanged, never re-implemented) and the other four are
|
|
21
|
+
this module's own: `requires` produced by an earlier stage, consecutive stages compose
|
|
22
|
+
by `Stage[In, Out]`, `intact` not already destroyed (task 1.2's ordering constraints),
|
|
23
|
+
and every `${var:NAME}` reference defined somewhere in the `extends` chain. Two more
|
|
24
|
+
belong to `extends` itself and have no equivalent in `weft_kernel.runner.Runner.resolve`,
|
|
25
|
+
because `Runner` was never handed a document with a parent to follow: an `extends`
|
|
26
|
+
target absent from the caller's `parents` lookup, and a cycle, named as the whole
|
|
27
|
+
chain rather than the one link that happened to close it. **Task 1.5 adds one more,
|
|
28
|
+
against the last thing a stage carries once vars are substituted:** a stage's `with:`
|
|
29
|
+
block is validated against its plugin's own declared `config_model`, and the *validated
|
|
30
|
+
object* — never the raw mapping — is what a `ResolvedStage.config` actually holds.
|
|
31
|
+
|
|
32
|
+
**Why this check exists — `02` §1's contract rule, and a repeated failure pattern.**
|
|
33
|
+
`02` §1: "a contract's registration API carries a typed configuration model,
|
|
34
|
+
or the extension point is decorative." `02` §3's own extended note names the evidence
|
|
35
|
+
that rule is written against: the identical failure recurring *independently* in two
|
|
36
|
+
unrelated subsystems — a parameterisation field commented out, so no per-instance
|
|
37
|
+
configuration could ever reach the instance it was meant to shape, and a factory
|
|
38
|
+
function taking only fixed keyword arguments with an unresolved TODO for the case
|
|
39
|
+
its author already knew was coming, because there was nowhere typed to put it. Both
|
|
40
|
+
defects are the same shape: a `with:`-style block
|
|
41
|
+
with nowhere checked to land, so it was either silently dropped or never offered at
|
|
42
|
+
all. This module is where that shape stops being possible for a pipeline document —
|
|
43
|
+
`InvalidStageConfigError` and `StageNotConfigurableError` below are its two ways of
|
|
44
|
+
refusing rather than repeating it.
|
|
45
|
+
|
|
46
|
+
**`requires`/`provides`/`intact`/`destroys`/`config_model` are read off the registered
|
|
47
|
+
factory *class*, never an instance this module builds.** `weft_kernel.registry`'s own
|
|
48
|
+
`_require_destroys_if_governed` already establishes the precedent — "reads the class,
|
|
49
|
+
not an instance... no config exists yet to build an instance from at registration
|
|
50
|
+
time" — and the same is true here for the identical reason: a resolved *form* is data,
|
|
51
|
+
so nothing here ever calls a factory. `config_model` follows the same defensive-`getattr`
|
|
52
|
+
grain those four already do, on purpose: G1 keeps the kernel from naming any capability,
|
|
53
|
+
so there is no `Chunker`-shaped or `Extractor`-shaped hook to hang a config check on —
|
|
54
|
+
the only thing every contract's plugin has in common is that it is *a class a factory
|
|
55
|
+
constructs*, and that is exactly what `requires`/`provides`/`intact`/`destroys` already
|
|
56
|
+
read off generically, with no ClassVar declared on `Stage` itself for `typing.Protocol`
|
|
57
|
+
reasons the runner's own module docstring explains. `config_model` costs a plugin one
|
|
58
|
+
optional class attribute — `getattr(declared, "config_model", None)` supplies `None`
|
|
59
|
+
for a plugin that never declares one, exactly as `Lifetime.RUN` / `()` / `()` are
|
|
60
|
+
supplied for the others — never a required base class or a name the kernel has to know
|
|
61
|
+
in advance. Every stand-in plugin class this module's own tests declare states these as
|
|
62
|
+
class attributes, exactly as `weft_chunk.fixed_size.FixedSizeChunker` does for real
|
|
63
|
+
(`destroys = (WordBoundaries,)` at its own class body). Every read goes through
|
|
64
|
+
`weft_kernel.registry.unwrap_factory` first, for the same reason
|
|
65
|
+
`_require_destroys_if_governed` does: `entry.factory` is sometimes a
|
|
66
|
+
`functools.partial` binding pack settings — `weft_store`'s own registration —
|
|
67
|
+
and reading `getattr` straight off a `partial` silently sees `()` (or `None`) regardless
|
|
68
|
+
of what the wrapped class actually declares. Skipping the unwrap here would be worse
|
|
69
|
+
than in `registry`, because there is no refusal to catch it: a governed contract's
|
|
70
|
+
`intact` or `destroys` would just vanish, and `weft_kernel.runner.Runner.resolve` —
|
|
71
|
+
which reads a *constructed instance*, immune to this — would disagree about the same
|
|
72
|
+
pipeline with nothing loud to say why.
|
|
73
|
+
|
|
74
|
+
**A stage's contract is supplied by the caller, not guessed from a bare `use:` name.**
|
|
75
|
+
A `StageDeclaration` carries no `contract:` key — G1 keeps the kernel from naming any
|
|
76
|
+
capability, so nothing here could recognise "Extractor" or "Chunker" if it saw one, and
|
|
77
|
+
searching every registered contract for a name that happens to match would make two
|
|
78
|
+
unrelated packs registering the same bare name under two different contracts
|
|
79
|
+
(`Registry`'s own docstring: "two names collide only if they share both the contract
|
|
80
|
+
*and* the string name... unrelated registrations") silently ambiguous rather than
|
|
81
|
+
loudly refused, which is precisely the guessing `weft_kernel.pipeline`'s own docstring
|
|
82
|
+
says a pre-resolution converter would have had to do. `resolve()` instead takes
|
|
83
|
+
`contracts: Mapping[str, type[object]]`, keyed by **stage id** — the same externally
|
|
84
|
+
supplied dependency shape `parents` already is: `02` §3 says "the kernel opens no
|
|
85
|
+
file... resolution takes its parent lookup as an argument", and a contract lookup is
|
|
86
|
+
the same kind of thing, supplied the same way. Once a stage's contract is known,
|
|
87
|
+
`registry.entry(contract, stage.use)` is the real, checked lookup `02` §3 means by "the
|
|
88
|
+
lookup resolution does against a registry" — a check, not a guess, run against
|
|
89
|
+
information resolution was actually given.
|
|
90
|
+
|
|
91
|
+
**Only the pipeline with no `extends` may carry stages — every other pipeline in the
|
|
92
|
+
chain carries only operators.**
|
|
93
|
+
`weft_kernel.pipeline.Pipeline._extends_and_stages_are_mutually_exclusive_with_operators`
|
|
94
|
+
refuses `stages:` alongside `extends`, and refuses an operator with no `extends`, both in
|
|
95
|
+
the authored form, before a registry or a parent lookup exists to resolve against — so
|
|
96
|
+
every stage this module ever *starts* from belongs to the *root* of an `extends` chain.
|
|
97
|
+
Task 1.4 is what a non-root pipeline in that chain is *for*: "resolve the parent
|
|
98
|
+
completely, then apply this pipeline's own operators to that result" (`02` §3 →
|
|
99
|
+
*Derivation*), root to leaf, one ancestor at a time — `_apply_ancestry_operators` below.
|
|
100
|
+
A stage's `provenance` is therefore no longer always the root: it is the root's name for
|
|
101
|
+
every stage the root wrote, and the name of whichever descendant's `insert` or `replace`
|
|
102
|
+
most recently put a *different* stage — or a different plugin at an existing id — there,
|
|
103
|
+
which is what `02` §3 means by "every stage in the resolved form records which pipeline
|
|
104
|
+
or pack put it there, so depth stays forensically readable." `set` never moves
|
|
105
|
+
provenance: it changes configuration, never which plugin answers for the id, so the
|
|
106
|
+
question `provenance` answers — who is responsible for *this stage existing, running
|
|
107
|
+
this plugin* — has the same answer before and after a `set`. Depth still matters for
|
|
108
|
+
`vars` exactly as it did before 1.4: each ancestor's `vars:` block is merged root-to-leaf,
|
|
109
|
+
so a leaf's override reaches a var referenced in a `with:` block the root itself wrote —
|
|
110
|
+
`02` §3: "a child's override re-resolves every inherited stage that references it."
|
|
111
|
+
|
|
112
|
+
**Operators apply against the *running* stage list, never against the original root.**
|
|
113
|
+
`02` §3: "Operators apply in written order, each validated against the running result."
|
|
114
|
+
That is what makes `remove` followed by `insert` on one id a move rather than a
|
|
115
|
+
collision — task 1.4 settles the question `docs/build-ledger.md` left open after 1.1:
|
|
116
|
+
"the order in which those keys appear in the document is the order they apply."
|
|
117
|
+
`weft_kernel.pipeline.Pipeline.operator_order` is read off the document (or the call)
|
|
118
|
+
that built the pipeline, never assumed from field order, and `_apply_operators` below
|
|
119
|
+
walks it literally, one block at a time. **Every operator is strict** — a target id
|
|
120
|
+
absent from the running result is `StaleOperatorTargetError`, naming the id, the
|
|
121
|
+
pipeline that wrote the operator, the parent it extends, and the ids that do exist;
|
|
122
|
+
`remove` gets no exemption, because a `remove` matching nothing is evidence the parent
|
|
123
|
+
moved under the child, not something to shrug past. `insert` additionally refuses a new
|
|
124
|
+
id that already exists — `OperatorIdCollisionError` — because inserting it would
|
|
125
|
+
silently shadow a stage the parent chain already has one of.
|
|
126
|
+
|
|
127
|
+
**`${var:NAME}` mirrors `weft_kernel.discovery.interpolate_env`'s `${env:VAR}` on
|
|
128
|
+
purpose**, not by coincidence: `02` §3 gives `vars:` scalar values substituted into
|
|
129
|
+
`with:`, the same shape `${env:VAR}` already substitutes environment values into pack
|
|
130
|
+
settings, and neither document specifies a template *language* — `interpolate_env`'s
|
|
131
|
+
own docstring is explicit that partial substitution inside a longer string "is a
|
|
132
|
+
template engine this project does not have and does not need". A value must be
|
|
133
|
+
*exactly* `${var:NAME}` to substitute; a string that merely contains the token passes
|
|
134
|
+
through untouched, and an undefined reference is `UndefinedVarError`, naming the var,
|
|
135
|
+
the pipeline and the stage whose `with:` block held the reference, exactly as `02` §3
|
|
136
|
+
requires: "An undefined var is a resolution error naming the var and the pipeline." The
|
|
137
|
+
stage id is not `02` §3's own wording, but it is task 1.13's own rule for the family
|
|
138
|
+
this class belongs to: `stages` is populated "wherever a failure genuinely has none to
|
|
139
|
+
name" is the only excuse for leaving it empty, and the stage holding the bad reference
|
|
140
|
+
is right there in scope at the call site below — there is nothing genuine about leaving
|
|
141
|
+
it out.
|
|
142
|
+
|
|
143
|
+
**Config validation runs last, per stage, after vars have already been substituted** —
|
|
144
|
+
task 1.5. A stage's `with:` block is a document fragment until this point: raw values,
|
|
145
|
+
possibly still carrying `${var:NAME}` tokens. `_substitute_vars` resolves those first,
|
|
146
|
+
so a bad var reference is still `UndefinedVarError` rather than a confusing validation
|
|
147
|
+
failure against a token pydantic was never going to accept as, say, an `int`. Only once
|
|
148
|
+
the block is fully literal does `_validate_stage_config` check it against
|
|
149
|
+
`declared.config_model` (read the same defensive way as `requires`/`intact`/`destroys`
|
|
150
|
+
above): `InvalidStageConfigError` if a model is declared and the block fails it —
|
|
151
|
+
naming the stage, the plugin, every field pydantic rejected and what the model accepts
|
|
152
|
+
— or `StageNotConfigurableError` if no model is declared and the block is non-empty
|
|
153
|
+
anyway. A plugin that declares no `config_model` at all still resolves, exactly as
|
|
154
|
+
before this task, provided its `with:` block is empty; the two are the same rule seen
|
|
155
|
+
from either side; see `02` §1: "a contract's registration API carries a typed
|
|
156
|
+
configuration model, or the extension point is decorative." The **object**
|
|
157
|
+
`config_model.model_validate(...)` returns — never the raw mapping it validated — is
|
|
158
|
+
what `ResolvedStage.config` holds from here on, which is the whole point: nothing that
|
|
159
|
+
reads a `ResolvedStage` downstream ever sees an untyped `dict`.
|
|
160
|
+
|
|
161
|
+
**`unapplied_operators` stays empty even now; `unplaced_contributions` still has nothing
|
|
162
|
+
to describe.** Task 1.4 gives every operator no exemption from failing loudly — a stale
|
|
163
|
+
target is `StaleOperatorTargetError`, not a recorded no-op — so there is no such thing as
|
|
164
|
+
an *unapplied* `insert`/`replace`/`remove`/`set` yet. `02` §3 → *Slots* is what first
|
|
165
|
+
gives an operator a legitimate reason to land nowhere: "Installation-dependent targets
|
|
166
|
+
are recorded, never fatal" for a contribution targeting a slot the running pack does not
|
|
167
|
+
provide — a distinction task 1.11 draws, not this one. `tuple[str, ...]` holds a short
|
|
168
|
+
description per entry until that task gives the concept its own shape; what matters for
|
|
169
|
+
this task is that the *field* already exists, so widening its element type in 1.11 is a
|
|
170
|
+
smaller, more contained change than discovering the field absent altogether after
|
|
171
|
+
evaluation has started comparing resolved forms by equality.
|
|
172
|
+
|
|
173
|
+
**Task 1.13 — the audit `02` §3 → *When resolution fails* asks for.** Two things changed
|
|
174
|
+
here, neither a reversal. First, `UnmetRequiresError`, `StageCompositionError` and
|
|
175
|
+
`IntactViolationError` are no longer declared in this module — they moved to
|
|
176
|
+
`weft_kernel.runner`, re-exported here (`from weft_kernel.runner import X as X`, the
|
|
177
|
+
explicit form a type checker accepts as a real export rather than an unused import) so
|
|
178
|
+
every existing `resolution.UnmetRequiresError` reference keeps working unchanged. The
|
|
179
|
+
reason is the defect the sweep found: `weft_kernel.runner.Runner.resolve` performed the
|
|
180
|
+
identical three checks against a `StageSpec` list and raised the bare
|
|
181
|
+
`PipelineResolutionError` family base directly for all three, told apart only by reading
|
|
182
|
+
the message — exactly the fat-class shape `02` §3 rules out, one module over from where
|
|
183
|
+
this module had already solved it correctly for a pipeline *document*. Reusing these three
|
|
184
|
+
names is not a new decision; it is applying the one this module already made to the other
|
|
185
|
+
mechanism that needed it, which is also why `manual/troubleshooting.md`'s own entry for
|
|
186
|
+
each already said "this is the same check `weft_kernel.runner.PipelineResolutionError`
|
|
187
|
+
performs for an explicit `StageSpec` list" before this task made it the same *class*.
|
|
188
|
+
|
|
189
|
+
Second, every subclass below now passes real, structured values for the four fields `02`
|
|
190
|
+
§3 requires on the family base — `pipeline`, `stages`, `distributions`, `remedy` — not only
|
|
191
|
+
a formatted sentence containing the same facts. `pipeline` and `distributions` are `None`
|
|
192
|
+
or `()` wherever this module genuinely has none to name (a `PipelineCycleError` has no
|
|
193
|
+
distribution in conflict; `_stage_signature`'s "contract does not declare `Stage[In,
|
|
194
|
+
Out]`" case has no pipeline in scope at all), the identical honest-absence reasoning `02`
|
|
195
|
+
§3 already gives `UnknownParentPipelineError`'s "no stage ids and no distribution to
|
|
196
|
+
name" — never a fabricated placeholder a caller could mistake for real data.
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
from __future__ import annotations
|
|
200
|
+
|
|
201
|
+
import hashlib
|
|
202
|
+
import json
|
|
203
|
+
import re
|
|
204
|
+
import typing
|
|
205
|
+
from collections.abc import Mapping
|
|
206
|
+
from types import MappingProxyType
|
|
207
|
+
from typing import Final, cast
|
|
208
|
+
|
|
209
|
+
from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, ValidationError, field_validator
|
|
210
|
+
|
|
211
|
+
from weft_kernel.errors import UnresolvedNameError
|
|
212
|
+
from weft_kernel.payload import Applies, ExtModel
|
|
213
|
+
from weft_kernel.pipeline import (
|
|
214
|
+
Pipeline,
|
|
215
|
+
Scalar,
|
|
216
|
+
SetOperator,
|
|
217
|
+
SlotDeclaration,
|
|
218
|
+
StageDeclaration,
|
|
219
|
+
VarBlock,
|
|
220
|
+
)
|
|
221
|
+
from weft_kernel.registry import Registry, unwrap_factory
|
|
222
|
+
from weft_kernel.runner import IntactViolationError as IntactViolationError
|
|
223
|
+
from weft_kernel.runner import (
|
|
224
|
+
PipelineResolutionError,
|
|
225
|
+
Stage,
|
|
226
|
+
UnresolvedNameInPipelineResolutionError,
|
|
227
|
+
)
|
|
228
|
+
from weft_kernel.runner import StageCompositionError as StageCompositionError
|
|
229
|
+
from weft_kernel.runner import UnmetRequiresError as UnmetRequiresError
|
|
230
|
+
|
|
231
|
+
_VAR_TOKEN: Final[re.Pattern[str]] = re.compile(r"^\$\{var:([^}]+)\}$")
|
|
232
|
+
"""On the same footing as `weft_kernel.discovery._ENV_TOKEN` — see the module docstring."""
|
|
233
|
+
|
|
234
|
+
_QUALIFIER: Final[str] = ":"
|
|
235
|
+
"""A deliberate duplicate of `weft_kernel.pipeline._QUALIFIER`, private to that module —
|
|
236
|
+
see `weft_kernel.pipeline._read_only`'s own docstring for why this module duplicates
|
|
237
|
+
rather than reaches across a private name: `_stage_signature` below does the identical
|
|
238
|
+
thing for the identical reason. This is what `_qualify` below stitches a contribution's
|
|
239
|
+
`distribution` and its own local `stage.id` together with, and what the deferred-`set`
|
|
240
|
+
split in `_apply_sets` reads to tell a plain stage id from a pack's contributed one.
|
|
241
|
+
"""
|
|
242
|
+
|
|
243
|
+
_NO_PARENTS: Final[Mapping[str, Pipeline]] = MappingProxyType({})
|
|
244
|
+
"""The default `parents` lookup for a pipeline that does not `extends` anything.
|
|
245
|
+
|
|
246
|
+
A `MappingProxyType`, not a plain `{}`, on the same reasoning
|
|
247
|
+
`weft_kernel.pipeline._NO_CONFIG` already documents: it is shared rather than
|
|
248
|
+
rebuilt per call, safe only because nothing can write through it, so a mutable-default
|
|
249
|
+
hazard never gets the chance to exist.
|
|
250
|
+
"""
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
class Contribution(BaseModel):
|
|
254
|
+
"""One pack's stage, offered into a named slot rather than claimed by a stage id.
|
|
255
|
+
|
|
256
|
+
`02` §3 → *Slots*: "A contribution targets a named slot, never a stage id." `stage`
|
|
257
|
+
reuses `StageDeclaration` for exactly that reason — the plugin name and its own
|
|
258
|
+
`with:` block are the same shape a pipeline's own stages already have — but
|
|
259
|
+
`stage.id` here is the pack's own **local**, unqualified name (`entities`, not
|
|
260
|
+
`weft-kg:entities`): `StageDeclaration.id`'s own reserved-qualifier check already
|
|
261
|
+
refuses a colon there, and there is no reason to give the same thing a second name.
|
|
262
|
+
`_qualify` below is what prefixes it with `distribution` once — and only once — a
|
|
263
|
+
contribution actually gets placed: an id only needs to be globally unique from the
|
|
264
|
+
moment it exists in a resolved stage list, and an unplaced contribution never reaches
|
|
265
|
+
one.
|
|
266
|
+
|
|
267
|
+
Never authored: a pipeline document has no syntax that builds one of these, and
|
|
268
|
+
resolution never builds one either. A caller supplies a tuple of these to
|
|
269
|
+
`resolve()`, on the identical footing `contracts` and `parents` already are — an
|
|
270
|
+
externally supplied dependency the kernel neither opens nor discovers, per `02` §3's
|
|
271
|
+
own G1 reasoning for `parents`. In practice that caller is whatever assembled the
|
|
272
|
+
`Registry` from every installed pack's own registration, since a contribution is
|
|
273
|
+
only ever real once a pack that offers one is actually on the machine.
|
|
274
|
+
|
|
275
|
+
**Task 5.3a (`S8`) is where that caller stopped being hypothetical.** A pack offers one
|
|
276
|
+
through its own `register()`, via `weft_kernel.discovery.PackRegistrar.add_contribution`
|
|
277
|
+
— `distribution` filled in there, never stated by the pack — and `weft_cli.
|
|
278
|
+
registry_bootstrap.build_dependencies` is the "whatever assembled the `Registry`" this
|
|
279
|
+
docstring already named: it concatenates every `weft_kernel.discovery.PackReport.
|
|
280
|
+
contributions` tuple `discover()` returned into the one `contributions=` argument every
|
|
281
|
+
`weft_kernel.resolution.resolve` call site in `weft-cli` now passes.
|
|
282
|
+
"""
|
|
283
|
+
|
|
284
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
285
|
+
|
|
286
|
+
slot: str = Field(min_length=1)
|
|
287
|
+
distribution: str = Field(min_length=1)
|
|
288
|
+
stage: StageDeclaration
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _qualify(contribution: Contribution) -> str:
|
|
292
|
+
"""The id a placed contribution wears in the resolved stage list — `02` §3 → *Slots*:
|
|
293
|
+
"Contributed stage ids are qualified by distribution (`weft-kg:entities`)."
|
|
294
|
+
"""
|
|
295
|
+
return f"{contribution.distribution}{_QUALIFIER}{contribution.stage.id}"
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _qualified_stage(
|
|
299
|
+
*, id: str, use: str, config: Mapping[str, object], fallback: tuple[str, ...]
|
|
300
|
+
) -> StageDeclaration:
|
|
301
|
+
"""Build a `StageDeclaration` bypassing validation — for an id this module computed itself.
|
|
302
|
+
|
|
303
|
+
`model_construct`, not the ordinary constructor, deliberately: `StageDeclaration.id`'s
|
|
304
|
+
own field validator (`_id_is_not_a_pack_s_to_give`) exists to refuse a `:`-qualified
|
|
305
|
+
id everywhere an *author* could type one — exactly the spelling `id` carries every
|
|
306
|
+
time this function is called, since both of this module's two call sites
|
|
307
|
+
(`_placed_stage`, placing a fresh contribution; `_apply_deferred_sets`, merging a
|
|
308
|
+
`set` into one already placed) only ever pass a *pack's* id, never an author's.
|
|
309
|
+
Re-validating it would refuse the one shape `02` §3 reserves the qualifier to
|
|
310
|
+
produce. `config` is trusted to already be the read-only view ordinary construction
|
|
311
|
+
gives it — a fresh dict merge (`{**current.config, **op.config}`) or a contribution's
|
|
312
|
+
own already-validated `stage.config` — so nothing here re-wraps it.
|
|
313
|
+
"""
|
|
314
|
+
return StageDeclaration.model_construct(id=id, use=use, config=config, fallback=fallback)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _placed_stage(qualified_id: str, stage: StageDeclaration) -> StageDeclaration:
|
|
318
|
+
"""A contribution's own `StageDeclaration`, wearing its placed, qualified id."""
|
|
319
|
+
return _qualified_stage(
|
|
320
|
+
id=qualified_id, use=stage.use, config=stage.config, fallback=stage.fallback
|
|
321
|
+
)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
_NO_CONTRIBUTIONS: Final[tuple[Contribution, ...]] = ()
|
|
325
|
+
"""The default `contributions` a `resolve()` call is given — a pipeline with slots that
|
|
326
|
+
no installed pack contributes into resolves exactly as it would with none declared at
|
|
327
|
+
all, which is what task 1.11 means by "installed and doing nothing must be visible
|
|
328
|
+
without breaking every pipeline lacking that slot" in the other direction: a pipeline
|
|
329
|
+
with a slot and *no* contributions must not need a caller to supply anything special
|
|
330
|
+
either.
|
|
331
|
+
"""
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
class UnknownParentPipelineError(
|
|
335
|
+
UnresolvedNameInPipelineResolutionError, PipelineResolutionError, UnresolvedNameError
|
|
336
|
+
):
|
|
337
|
+
"""`extends` names a pipeline the caller's `parents` mapping does not contain.
|
|
338
|
+
|
|
339
|
+
`02` §3 → *When resolution fails*: every failure names "the pipeline, the stage
|
|
340
|
+
ids, the distributions in conflict and the remedy" where those apply — a missing
|
|
341
|
+
parent has no stage ids and no distribution to name, so the message states the
|
|
342
|
+
child, the parent name it wrote, and the remedy: supply that pipeline in `parents`,
|
|
343
|
+
or fix the typo. It also names every pipeline `parents` *does* contain, the same
|
|
344
|
+
"what was wanted, why it is unavailable, and what the valid options are" shape
|
|
345
|
+
`weft_kernel.errors`' own module docstring requires of every kernel-raised error —
|
|
346
|
+
without it a one-character typo is unfixable from the message alone, when the name
|
|
347
|
+
that would have worked was in the caller's own argument the whole time.
|
|
348
|
+
`weft_kernel.runner`'s own `PipelineResolutionError` has no equivalent, because
|
|
349
|
+
`Runner.resolve` is never handed a document with a parent to follow at all.
|
|
350
|
+
|
|
351
|
+
Fitness function 12's family: `valid_options` is every pipeline name `parents`
|
|
352
|
+
*does* contain.
|
|
353
|
+
|
|
354
|
+
No `__init__` of its own — task 2.36's repair collapsed this class's own 19-line
|
|
355
|
+
forwarding body into `weft_kernel.runner.UnresolvedNameInPipelineResolutionError`, which
|
|
356
|
+
it now inherits unmodified; see that class's own docstring for why the shared body
|
|
357
|
+
lives in `runner` rather than here, and why `valid_options` staying required and
|
|
358
|
+
keyword-only, with no default, is unaffected by the collapse.
|
|
359
|
+
"""
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
class PipelineCycleError(PipelineResolutionError):
|
|
363
|
+
"""`extends` walks back to a pipeline already in the chain being resolved.
|
|
364
|
+
|
|
365
|
+
Named as the **whole chain**, not the one link that happened to close it — `02` §3:
|
|
366
|
+
"A cycle is a resolution error naming the whole chain." Reporting only the
|
|
367
|
+
repeated name would tell an author *that* something loops without telling them
|
|
368
|
+
*which* edit to undo; the full chain, in the order it was walked, does.
|
|
369
|
+
"""
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
class InvalidStageConfigError(PipelineResolutionError):
|
|
373
|
+
"""A stage's `with:` block does not validate against its plugin's own `config_model`.
|
|
374
|
+
|
|
375
|
+
Task 1.5, `02` §1: "a contract's registration API carries a typed configuration
|
|
376
|
+
model, or the extension point is decorative." Raised once `${var:NAME}` substitution
|
|
377
|
+
has already happened (see the module docstring, *"Config validation runs last..."*),
|
|
378
|
+
so this is always a genuine mismatch between the literal `with:` block and what the
|
|
379
|
+
plugin's model accepts — never a var reference read as a stray string. Names the
|
|
380
|
+
stage id, the plugin (`contract:name`), every field pydantic's own `ValidationError`
|
|
381
|
+
rejected with its reason, and what the model accepts, so a typo'd field reads as a
|
|
382
|
+
typo and a wrong type reads as a wrong type, from the message alone.
|
|
383
|
+
"""
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
class StageNotConfigurableError(PipelineResolutionError):
|
|
387
|
+
"""A stage sets a non-empty `with:` block for a plugin that publishes no `config_model`.
|
|
388
|
+
|
|
389
|
+
Task 1.5, the other half of `02` §1's rule: an extension point with no typed model
|
|
390
|
+
is decorative, and a `with:` block written against a decorative extension point
|
|
391
|
+
cannot be silently accepted and dropped — that is exactly the failure pattern the
|
|
392
|
+
module docstring describes: a parameterisation field commented out, a per-instance
|
|
393
|
+
configuration path with nowhere typed to go, both swallowed with no error at all.
|
|
394
|
+
An absent `config_model` is not a refusal to be configured
|
|
395
|
+
forever, only *today*; the remedy this names is either dropping the `with:` block,
|
|
396
|
+
or having the plugin declare a `config_model` so it has somewhere to land.
|
|
397
|
+
"""
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
class UndefinedVarError(
|
|
401
|
+
UnresolvedNameInPipelineResolutionError, PipelineResolutionError, UnresolvedNameError
|
|
402
|
+
):
|
|
403
|
+
"""A `with:` value references `${var:NAME}` and no pipeline in the chain defines it.
|
|
404
|
+
|
|
405
|
+
`02` §3 → *Language, and what a var is for*: "An undefined var is a resolution
|
|
406
|
+
error naming the var and the pipeline." Checked against every ancestor's `vars:`
|
|
407
|
+
merged root to leaf, so a var the root never set but a child later supplies is
|
|
408
|
+
still found — the reference is resolved against the *final* chain, not the level
|
|
409
|
+
that wrote it. Also names every var `merged_vars` *does* define at that point —
|
|
410
|
+
the chain's own final answer, already in scope where this raises — so a reader can
|
|
411
|
+
tell a misspelling from a var genuinely missing from every ancestor, rather than
|
|
412
|
+
grepping the whole chain by hand to find out. `stages` names the one stage whose
|
|
413
|
+
`with:` block held the reference — task 1.13's own field, populated here because
|
|
414
|
+
the stage is never absent from scope, unlike a cycle's missing distribution.
|
|
415
|
+
|
|
416
|
+
Fitness function 12's family: `valid_options` is every var `merged_vars` defines.
|
|
417
|
+
|
|
418
|
+
No `__init__` of its own — task 2.36's repair collapsed this class's own 19-line
|
|
419
|
+
forwarding body into `weft_kernel.runner.UnresolvedNameInPipelineResolutionError`, which
|
|
420
|
+
it now inherits unmodified; see that class's own docstring for why the shared body
|
|
421
|
+
lives in `runner` rather than here, and why `valid_options` staying required and
|
|
422
|
+
keyword-only, with no default, is unaffected by the collapse.
|
|
423
|
+
"""
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
class StaleOperatorTargetError(
|
|
427
|
+
UnresolvedNameInPipelineResolutionError, PipelineResolutionError, UnresolvedNameError
|
|
428
|
+
):
|
|
429
|
+
"""Task 1.4: an operator names a stage id the running result does not have.
|
|
430
|
+
|
|
431
|
+
`02` §3 → the operator table's edge rules: "Every operator is strict. A target id
|
|
432
|
+
absent from the resolved parent fails resolution, naming the id, the pipeline that
|
|
433
|
+
wrote the operator, the parent it resolved against and the ids that do exist." One
|
|
434
|
+
class for `insert`'s `after:`/`before:`, `replace`'s id, `remove`'s id and `set`'s id
|
|
435
|
+
alike — all four are the same failure *kind*, a reference into a parent that turned
|
|
436
|
+
out not to have what it named, which is why they share one name rather than four.
|
|
437
|
+
`remove` gets no exemption: "a `remove` line matching nothing is evidence the parent
|
|
438
|
+
moved under you," the same words `02` §3 uses, not a softer case of this error.
|
|
439
|
+
|
|
440
|
+
Task 1.11 widens `remove`'s own half two ways, without adding a fifth name: its
|
|
441
|
+
target may now be a *slot* id as well as a stage id (`_apply_removes` below), and a
|
|
442
|
+
slot's own `after:`/`before:` anchor going missing — because a descendant's `remove`
|
|
443
|
+
took the stage it pointed at, never because the slot itself moved — is the identical
|
|
444
|
+
*kind* of reference-that-turned-out-missing `_slot_anchor_index` raises this for, not
|
|
445
|
+
a sixth error class.
|
|
446
|
+
|
|
447
|
+
Fitness function 12's family: `valid_options` is every id that does exist at this
|
|
448
|
+
point in the chain — stage ids and slot ids alike, since either may be a legal
|
|
449
|
+
target depending on the operator.
|
|
450
|
+
|
|
451
|
+
No `__init__` of its own — task 2.36's repair collapsed this class's own 19-line
|
|
452
|
+
forwarding body into `weft_kernel.runner.UnresolvedNameInPipelineResolutionError`, which
|
|
453
|
+
it now inherits unmodified; see that class's own docstring for why the shared body
|
|
454
|
+
lives in `runner` rather than here, and why `valid_options` staying required and
|
|
455
|
+
keyword-only, with no default, is unaffected by the collapse.
|
|
456
|
+
"""
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
class OperatorIdCollisionError(PipelineResolutionError):
|
|
460
|
+
"""Task 1.4: an `insert` operator's new stage id already exists in the running result.
|
|
461
|
+
|
|
462
|
+
`02` §3 → the operator table's edge rules: "`insert` fails equally when its *new* id
|
|
463
|
+
collides with an existing one, or a child would silently shadow a parent's stage." A
|
|
464
|
+
distinct failure *kind* from `StaleOperatorTargetError` above — that one is a
|
|
465
|
+
reference to something missing, this one is a name that is not free to take — so it
|
|
466
|
+
gets its own class rather than a shared one with a reason field, per `02` §3 → *When
|
|
467
|
+
resolution fails*'s "one class per kind rather than one class with a `kind` field."
|
|
468
|
+
"""
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
class DuplicateContributionError(PipelineResolutionError):
|
|
472
|
+
"""Task 1.11: two contributions resolve to the identical qualified id.
|
|
473
|
+
|
|
474
|
+
`_qualify` stitches a contribution's `distribution` and its own local `stage.id`
|
|
475
|
+
together into the id it wears once placed (`02` §3 → *Slots*: "Contributed stage
|
|
476
|
+
ids are qualified by distribution... so they cannot collide with the author's").
|
|
477
|
+
That guarantee is about a contribution colliding with something the *document*
|
|
478
|
+
wrote; it says nothing about two contributions colliding with *each other* — nothing
|
|
479
|
+
forces a distribution's own `register()` to keep the local ids of the contributions
|
|
480
|
+
it offers unique among themselves. Without this check, `_order_contributions`'s own
|
|
481
|
+
`remaining = {_qualify(c): c for c in contributions}` would silently collapse two
|
|
482
|
+
such contributions to one dict entry — the second one built wins, the first is gone:
|
|
483
|
+
not placed, not refused, and never counted in `unplaced_contributions` either, since
|
|
484
|
+
it never survives long enough to be checked against a declared slot. That is the
|
|
485
|
+
same silently-overwriting registration defect `weft_kernel.registry` refuses, one
|
|
486
|
+
seam further in, and `02` §3's *Slots* section rules out exactly this shape of
|
|
487
|
+
collision for an
|
|
488
|
+
author's own stages; this is the same rule applied where two packs, not a pack and
|
|
489
|
+
an author, are the ones that can collide.
|
|
490
|
+
"""
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
class SlotOrderConflictError(PipelineResolutionError):
|
|
494
|
+
"""Task 1.11: two or more contributions to one slot cannot be ordered at all.
|
|
495
|
+
|
|
496
|
+
`02` §3 → *Slots*: "two contributions in one slot are ordered by the declared
|
|
497
|
+
[`intact`/`destroys`] relations... genuine ties break by distribution name." That
|
|
498
|
+
tie-break only ever runs among contributions with *no* relation between them —
|
|
499
|
+
`_order_contributions` below still has to know what to do when the relations
|
|
500
|
+
themselves cannot be satisfied by any order at all, the same way a cycle in `extends`
|
|
501
|
+
cannot be resolved by trying harder. A distinct failure *kind* from
|
|
502
|
+
`IntactViolationError`: that one checks a single, already-fixed order against a
|
|
503
|
+
constraint; this one is what happens when *no* order would satisfy every declared
|
|
504
|
+
constraint among a slot's contributions, which needs its own name rather than
|
|
505
|
+
borrowing one that presumes an order already exists to be wrong about.
|
|
506
|
+
"""
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
_EMPTY_CONFIG: Final[Mapping[str, object]] = MappingProxyType({})
|
|
510
|
+
"""What `ResolvedStage.config` holds for a plugin that declares no `config_model` at all.
|
|
511
|
+
|
|
512
|
+
Shared rather than rebuilt per stage, on the same reasoning `weft_kernel.pipeline.
|
|
513
|
+
_NO_CONFIG` already documents: safe only because nothing can write through a
|
|
514
|
+
`MappingProxyType`, so there is nothing a stage's stored default could do to another
|
|
515
|
+
stage's if they happened to be the exact same (empty) object.
|
|
516
|
+
"""
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def _dump_stage_config(value: object) -> object:
|
|
520
|
+
"""Serialise `ResolvedStage.config`'s two possible shapes back into something JSON holds.
|
|
521
|
+
|
|
522
|
+
A `config_model` instance dumps through its own `model_dump(mode="json")` — pydantic
|
|
523
|
+
already knows how to turn any nested field of its own into JSON-safe data, which a
|
|
524
|
+
third-party plugin's `config_model` gets for free by being a `BaseModel` subclass at
|
|
525
|
+
all. The no-model case is a plain (read-only) mapping already, so `dict(...)` is the
|
|
526
|
+
whole job — the same unwrap `weft_kernel.pipeline._read_only`'s own `PlainSerializer`
|
|
527
|
+
performs for exactly the same reason: pydantic knows how to write a `dict` back to a
|
|
528
|
+
document and refuses a `MappingProxyType` outright.
|
|
529
|
+
"""
|
|
530
|
+
if isinstance(value, BaseModel):
|
|
531
|
+
return value.model_dump(mode="json")
|
|
532
|
+
if isinstance(value, Mapping):
|
|
533
|
+
return dict(cast("Mapping[str, object]", value))
|
|
534
|
+
return value
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
type StageConfig = typing.Annotated[
|
|
538
|
+
object, PlainSerializer(_dump_stage_config, return_type=object, when_used="always")
|
|
539
|
+
]
|
|
540
|
+
"""What `ResolvedStage.config` actually holds — task 1.5.
|
|
541
|
+
|
|
542
|
+
Either a `config_model` instance the plugin declared and this module validated the
|
|
543
|
+
stage's `with:` block against, or an empty read-only mapping for a plugin that declares
|
|
544
|
+
none. `object`, not `ConfigBlock` (`weft_kernel.pipeline`'s own `Mapping[str, object]`
|
|
545
|
+
alias): the two shapes this field can hold share no common `Mapping` base once one of
|
|
546
|
+
them is an arbitrary `BaseModel` subclass a third-party plugin defines, and pydantic
|
|
547
|
+
validates an `object`-typed field by accepting whatever `_validate_stage_config` below
|
|
548
|
+
already decided to build, rather than trying to coerce it into some narrower shape.
|
|
549
|
+
`_dump_stage_config`, defined just above so it exists before this alias's value is ever
|
|
550
|
+
evaluated, is the corresponding write side: `model_dump()` needs to know how to turn
|
|
551
|
+
either shape back into something JSON can hold.
|
|
552
|
+
"""
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def _validate_stage_config(
|
|
556
|
+
config: Mapping[str, object],
|
|
557
|
+
*,
|
|
558
|
+
declared: object,
|
|
559
|
+
stage_id: str,
|
|
560
|
+
contract: type[object],
|
|
561
|
+
use: str,
|
|
562
|
+
pipeline_name: str,
|
|
563
|
+
) -> object:
|
|
564
|
+
"""Validate a stage's (already var-substituted) `with:` block against its plugin's model.
|
|
565
|
+
|
|
566
|
+
See the module docstring, *"Config validation runs last, per stage..."*, for why this
|
|
567
|
+
runs after `_substitute_vars` rather than before, and `02` §1's rule this exists to
|
|
568
|
+
make real: "a contract's registration API carries a typed configuration model, or the
|
|
569
|
+
extension point is decorative." `config_model` is read the same defensive way
|
|
570
|
+
`requires`/`intact`/`destroys` already are, off `declared` — the caller's
|
|
571
|
+
`unwrap_factory(entry.factory)` — so a plugin that never declares one answers `None`,
|
|
572
|
+
not an `AttributeError`.
|
|
573
|
+
|
|
574
|
+
No model declared and an empty block: nothing to check, returns `_EMPTY_CONFIG`. No
|
|
575
|
+
model declared and a non-empty block: `StageNotConfigurableError`, naming the stage
|
|
576
|
+
and the plugin — a `with:` block with nowhere checked to land must never be silently
|
|
577
|
+
accepted and dropped, which is exactly the defect this task exists to close. A
|
|
578
|
+
model declared: `config_model.model_validate(config)`, and any `pydantic.
|
|
579
|
+
ValidationError` it raises becomes `InvalidStageConfigError`, naming the stage, the
|
|
580
|
+
plugin, every field pydantic rejected with its own reason, and — read off `config_model.
|
|
581
|
+
model_fields` rather than hand-maintained — what the model actually accepts.
|
|
582
|
+
"""
|
|
583
|
+
config_model = cast("type[BaseModel] | None", getattr(declared, "config_model", None))
|
|
584
|
+
if config_model is None:
|
|
585
|
+
if config:
|
|
586
|
+
raise StageNotConfigurableError(
|
|
587
|
+
f"stage '{stage_id}' ({contract.__name__}:{use}) in pipeline '{pipeline_name}' "
|
|
588
|
+
f"sets a 'with:' block ({dict(config)!r}), but {contract.__name__}:{use} "
|
|
589
|
+
f"publishes no configuration model — it cannot be parameterised at all. Drop "
|
|
590
|
+
f"'with:', or have the plugin declare `config_model`.",
|
|
591
|
+
pipeline=pipeline_name,
|
|
592
|
+
stages=(stage_id,),
|
|
593
|
+
remedy=(
|
|
594
|
+
f"drop the 'with:' block on '{stage_id}', or have {contract.__name__}:"
|
|
595
|
+
f"{use} declare a `config_model`."
|
|
596
|
+
),
|
|
597
|
+
)
|
|
598
|
+
return _EMPTY_CONFIG
|
|
599
|
+
try:
|
|
600
|
+
return config_model.model_validate(config)
|
|
601
|
+
except ValidationError as exc:
|
|
602
|
+
problems = "; ".join(
|
|
603
|
+
f"field '{'.'.join(str(part) for part in error['loc']) or '(with block itself)'}': "
|
|
604
|
+
f"{error['msg']}"
|
|
605
|
+
for error in exc.errors()
|
|
606
|
+
)
|
|
607
|
+
accepted = ", ".join(sorted(config_model.model_fields)) or "(no fields)"
|
|
608
|
+
raise InvalidStageConfigError(
|
|
609
|
+
f"stage '{stage_id}' ({contract.__name__}:{use}) in pipeline '{pipeline_name}' has "
|
|
610
|
+
f"an invalid 'with:' block for {config_model.__name__}: {problems}. "
|
|
611
|
+
f"{config_model.__name__} accepts: {accepted}.",
|
|
612
|
+
pipeline=pipeline_name,
|
|
613
|
+
stages=(stage_id,),
|
|
614
|
+
remedy=(
|
|
615
|
+
f"fix the 'with:' fields the message rejected. "
|
|
616
|
+
f"{config_model.__name__} accepts: {accepted}."
|
|
617
|
+
),
|
|
618
|
+
) from exc
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
class ResolvedStage(BaseModel):
|
|
622
|
+
"""One stage, checked and fully explicit — no plugin instance, only what building one needs.
|
|
623
|
+
|
|
624
|
+
Not `weft_kernel.runner.StageSpec`: that one is handed *by a caller* to
|
|
625
|
+
`Runner.resolve`, already carrying a concrete contract type and a config object a
|
|
626
|
+
caller assembled by hand. This one is what `resolve()` *produces* from a
|
|
627
|
+
`StageDeclaration` — `contract` is document-shaped, printable form (a name) rather
|
|
628
|
+
than a caller-assembled Python type, and `config` — task 1.5 — is the plugin's own
|
|
629
|
+
`config_model` already validated against the stage's `with:` block, or an empty
|
|
630
|
+
mapping for a plugin that declares none. Either way it is data built fresh by this
|
|
631
|
+
call, from what `resolve()` was given, never a caller-assembled object handed in —
|
|
632
|
+
which is what makes two `ResolvedStage`s from two separate `resolve()` calls
|
|
633
|
+
comparable by `==`.
|
|
634
|
+
|
|
635
|
+
`provenance` is which pipeline is responsible for this stage existing and running the
|
|
636
|
+
plugin it runs — `02` §3: "every stage in the resolved form records which pipeline or
|
|
637
|
+
pack put it there, so depth stays forensically readable." The root's name for every
|
|
638
|
+
stage the root wrote and no descendant's `insert`/`replace` has touched since; the
|
|
639
|
+
name of whichever descendant's `insert` introduced the stage, or whose `replace` most
|
|
640
|
+
recently swapped its plugin, otherwise. `set` never changes it — overriding
|
|
641
|
+
configuration answers a different question than "who put this stage's plugin here."
|
|
642
|
+
Task 1.11 is what first lets a *pack* be the answer here instead of a pipeline.
|
|
643
|
+
|
|
644
|
+
`contract_version` is the *contract's* declared version — `getattr(contract,
|
|
645
|
+
"version", None)`, the same `ClassVar` `weft_extract.contract.Extractor.version`
|
|
646
|
+
and `weft_chunk.contract.Chunker.version` both carry — never defaulted to a
|
|
647
|
+
plausible-looking string when a caller hands `resolve()` a contract that
|
|
648
|
+
declares none: `None` recorded is visible in a diff, a fabricated version is
|
|
649
|
+
not. `02` §3 names "every stage, plugin, version and configuration value" as
|
|
650
|
+
what a resolved pipeline makes explicit; this is that version. It is not the
|
|
651
|
+
*distribution's* version (what a pack upgrade actually bumps, and what
|
|
652
|
+
evaluation would need to diff a retrieval regression across one) — the kernel
|
|
653
|
+
reads no package metadata to know that, and nothing here pretends otherwise
|
|
654
|
+
by inventing a field it cannot fill honestly.
|
|
655
|
+
|
|
656
|
+
`applies_to` — task 1.6, `02` §3 → *Applicability* — is the one exception to this
|
|
657
|
+
module's general rule that a check's inputs (`requires`/`provides`/`intact`/
|
|
658
|
+
`destroys`) are read off `declared` and then discarded once they have done their
|
|
659
|
+
job: applicability is not a load-time *check* at all, nothing here evaluates it, so
|
|
660
|
+
there is nothing for it to be discarded after. It is read off `declared` the
|
|
661
|
+
identical defensive way and simply carried onto the record, because `02` §3 states
|
|
662
|
+
it as its own requirement: "The resolved form must print each stage's applicability,
|
|
663
|
+
since a predicate is data." A reader of a `ResolvedPipeline` — a person running
|
|
664
|
+
`weft pipeline show`, eventually, or a test comparing two resolutions — has no other
|
|
665
|
+
way to see what a stage will route around once `weft_kernel.runner` actually runs it.
|
|
666
|
+
|
|
667
|
+
**`fallback` is the other field this module deliberately never looks up, and that is
|
|
668
|
+
a second, different exception from `applies_to`'s.** `use` is checked against
|
|
669
|
+
`registry` by `Registry.entry` below, and refuses an unregistered name loudly, by
|
|
670
|
+
design — `01`'s own rule against a silent fallback. `fallback` is not: a document's
|
|
671
|
+
`fallback:` list round-trips from `StageDeclaration` onto this field unchecked,
|
|
672
|
+
exactly as authored, string for string. This is not an oversight this module carries
|
|
673
|
+
quietly — `manual/user-manual.md` §1 and `02` §3's own worked example say so where a
|
|
674
|
+
reader meets the field. Looking a fallback name up *here* would force every fallback
|
|
675
|
+
entry to name a plugin already installed alongside the stage that names it, which
|
|
676
|
+
forecloses exactly the case `fallback:` exists for: a document naming `ocr` as the
|
|
677
|
+
fallback for `text` before any pack ships one, so the document is already correct the
|
|
678
|
+
day a pack providing it is installed, with no edit. `fallback:` is intentionally out
|
|
679
|
+
of `tests/architecture/test_ff11_pipeline_integrity.py`'s check for the identical
|
|
680
|
+
reason: that check exists to catch a stage whose plugin will not run *today*, and a
|
|
681
|
+
fallback naming a plugin nobody has written yet is not that mistake.
|
|
682
|
+
|
|
683
|
+
**Phase 2 task 2.28 answered the question this docstring used to leave open**, and the
|
|
684
|
+
answer preserves everything above. `weft_kernel.fallback.try_in_order` now walks the
|
|
685
|
+
list, and `weft_kernel.runner.Runner.resolve` — a *later* step than this one — refuses
|
|
686
|
+
an unregistered fallback name with `UnknownFallbackError`. Not at try time, which
|
|
687
|
+
would make the failure depend on encountering a document the primary cannot read, and
|
|
688
|
+
never by skipping, which is `01` rule 5's silent fallback with extra steps. So the
|
|
689
|
+
promise this module protects is intact and is now visibly a *document*-level one: a
|
|
690
|
+
pipeline may be authored, stored, diffed and derived while naming a plugin nobody has
|
|
691
|
+
shipped; what is refused is making that pipeline runnable.
|
|
692
|
+
"""
|
|
693
|
+
|
|
694
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
695
|
+
|
|
696
|
+
id: str
|
|
697
|
+
contract: str
|
|
698
|
+
contract_version: str | None = None
|
|
699
|
+
use: str
|
|
700
|
+
config: StageConfig = Field(default_factory=lambda: _EMPTY_CONFIG)
|
|
701
|
+
fallback: tuple[str, ...] = ()
|
|
702
|
+
applies_to: tuple[Applies, ...] = ()
|
|
703
|
+
distribution: str
|
|
704
|
+
provenance: str
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
class ResolvedPipeline(BaseModel):
|
|
708
|
+
"""A pipeline document reduced to its frozen, fully-explicit form. See the module docstring.
|
|
709
|
+
|
|
710
|
+
`vars` holds the **final** merged value of every var in the chain, substituted into
|
|
711
|
+
every stage's `config` already — nothing about a `ResolvedPipeline` requires
|
|
712
|
+
walking `extends` again to understand what it means. `stages` is the root's own list
|
|
713
|
+
with every descendant's operators already folded in, in the order they applied, not
|
|
714
|
+
the order the root originally wrote them. `unapplied_operators` and
|
|
715
|
+
`unplaced_contributions` are carried empty until task 1.11 gives them something to
|
|
716
|
+
hold — task 1.4's operators are strict, never recorded-and-skipped; see the module
|
|
717
|
+
docstring for why the fields exist before their content can.
|
|
718
|
+
"""
|
|
719
|
+
|
|
720
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
721
|
+
|
|
722
|
+
name: str
|
|
723
|
+
vars: VarBlock = Field(default_factory=dict)
|
|
724
|
+
stages: tuple[ResolvedStage, ...] = ()
|
|
725
|
+
unapplied_operators: tuple[str, ...] = ()
|
|
726
|
+
unplaced_contributions: tuple[str, ...] = ()
|
|
727
|
+
|
|
728
|
+
@field_validator("vars", mode="after")
|
|
729
|
+
@classmethod
|
|
730
|
+
def _vars_are_read_only(cls, value: Mapping[str, Scalar]) -> Mapping[str, Scalar]:
|
|
731
|
+
return _read_only(value)
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def _read_only[K, V](value: Mapping[K, V]) -> Mapping[K, V]:
|
|
735
|
+
"""Make a resolved mapping field as frozen as the model that carries it.
|
|
736
|
+
|
|
737
|
+
A deliberate duplicate of `weft_kernel.pipeline._read_only`, private to that
|
|
738
|
+
module, for the identical reason `_stage_signature` above is duplicated rather
|
|
739
|
+
than imported — see that function's own docstring. Same shape, same reasoning:
|
|
740
|
+
`frozen=True` blocks rebinding a field, never mutating what it holds, and
|
|
741
|
+
`ResolvedStage.config`/`ResolvedPipeline.vars` are shared, never copied, across
|
|
742
|
+
every stage that came from the same ancestor — see the module docstring's note on
|
|
743
|
+
`extends` reading live parents.
|
|
744
|
+
"""
|
|
745
|
+
return MappingProxyType(dict(value))
|
|
746
|
+
|
|
747
|
+
|
|
748
|
+
#: How much of the sha256 hex digest `pipeline_identity` keeps. Thirty-two hex characters is 128
|
|
749
|
+
#: bits — far past any collision an operator's set of pipelines could reach, and short enough that
|
|
750
|
+
#: the value stays readable in a `weft index` line and in a `weft_sources` column an operator
|
|
751
|
+
#: greps. The full 64 buys nothing here and costs legibility at exactly the moment it is read.
|
|
752
|
+
_IDENTITY_WIDTH: Final[int] = 32
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
def pipeline_identity(pipeline: ResolvedPipeline) -> str:
|
|
756
|
+
"""A stable digest of what a resolved pipeline actually *runs* — ledger task **9.17**.
|
|
757
|
+
|
|
758
|
+
`SourceRecord.pipeline` records a document's **name**, and a name does not move when the
|
|
759
|
+
plugin behind a stage does: swap `pdf-text` for `pdf-layout` inside `index-text`, or change an
|
|
760
|
+
embedder's `with: model:`, and the record still says `index-text` while the corpus was built
|
|
761
|
+
two different ways. This is the comparable thing — `weft_cli.pipeline_diff`'s field-by-field
|
|
762
|
+
`==` over two `ResolvedPipeline` values, reduced to one string a store column can carry and an
|
|
763
|
+
operator can read.
|
|
764
|
+
|
|
765
|
+
**What moves it.** Every resolved stage in order, and for each: `id`, `contract`,
|
|
766
|
+
`contract_version`, `use`, `distribution`, and `config` with its keys sorted. Plus the
|
|
767
|
+
pipeline's `vars`, keys sorted — a var reaches a stage's config at resolution, but a var the
|
|
768
|
+
document declares and no stage reads is still part of what an operator wrote, and excluding it
|
|
769
|
+
would need an argument nobody has made.
|
|
770
|
+
|
|
771
|
+
**What does not, and why each is deliberate.** `name` is excluded: renaming a document does not
|
|
772
|
+
re-parse a corpus, and an identity that moved on a rename would report a reparse that did not
|
|
773
|
+
happen — a false positive in a change detector is worse than no detector, because it teaches
|
|
774
|
+
people to ignore it. `unapplied_operators` and `unplaced_contributions` are excluded: they
|
|
775
|
+
record what resolution *could not* do, which is a diagnostic about the document rather than part
|
|
776
|
+
of what ran, so a corpus built with one is the same corpus.
|
|
777
|
+
|
|
778
|
+
**Every part is length-prefixed before hashing**, exactly as
|
|
779
|
+
`weft_kernel.payload.node._content_digest` does and for its reason: without it no concatenation
|
|
780
|
+
of variable-length strings is safe against a different split of the same bytes, and
|
|
781
|
+
`("ab", "c")` would collide with `("a", "bc")`.
|
|
782
|
+
|
|
783
|
+
Pure and deterministic — no clock, no environment, no registry. It names no capability either:
|
|
784
|
+
it reads `ResolvedStage`'s own fields and knows nothing about what a `contract` string means.
|
|
785
|
+
"""
|
|
786
|
+
digest = hashlib.sha256()
|
|
787
|
+
for part in _identity_parts(pipeline):
|
|
788
|
+
encoded = part.encode("utf-8")
|
|
789
|
+
digest.update(len(encoded).to_bytes(8, byteorder="big"))
|
|
790
|
+
digest.update(encoded)
|
|
791
|
+
return digest.hexdigest()[:_IDENTITY_WIDTH]
|
|
792
|
+
|
|
793
|
+
|
|
794
|
+
def _identity_parts(pipeline: ResolvedPipeline) -> tuple[str, ...]:
|
|
795
|
+
"""Everything `pipeline_identity` hashes, in order — see its docstring for what is left out.
|
|
796
|
+
|
|
797
|
+
`default=str` on the config dump is deliberate rather than lax: a `config` value pydantic
|
|
798
|
+
validated but `json` cannot encode would otherwise make this function *raise*, and an identity
|
|
799
|
+
function able to fail a run that was otherwise fine is a worse outcome than a value rendered
|
|
800
|
+
through `str`. The digest stays stable either way, which is the only property asked of it.
|
|
801
|
+
"""
|
|
802
|
+
parts: list[str] = [
|
|
803
|
+
json.dumps(dict(sorted(pipeline.vars.items())), sort_keys=True, default=str)
|
|
804
|
+
]
|
|
805
|
+
for stage in pipeline.stages:
|
|
806
|
+
parts.extend(
|
|
807
|
+
(
|
|
808
|
+
stage.id,
|
|
809
|
+
stage.contract,
|
|
810
|
+
stage.contract_version or "",
|
|
811
|
+
stage.use,
|
|
812
|
+
stage.distribution,
|
|
813
|
+
json.dumps(cast("Mapping[str, object]", stage.config), sort_keys=True, default=str),
|
|
814
|
+
)
|
|
815
|
+
)
|
|
816
|
+
return tuple(parts)
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
def resolve(
|
|
820
|
+
pipeline: Pipeline,
|
|
821
|
+
*,
|
|
822
|
+
registry: Registry,
|
|
823
|
+
contracts: Mapping[str, type[object]],
|
|
824
|
+
parents: Mapping[str, Pipeline] = _NO_PARENTS,
|
|
825
|
+
contributions: tuple[Contribution, ...] = _NO_CONTRIBUTIONS,
|
|
826
|
+
) -> ResolvedPipeline:
|
|
827
|
+
"""Reduce `pipeline` to a frozen, fully-explicit `ResolvedPipeline`. See the module docstring.
|
|
828
|
+
|
|
829
|
+
`contracts` must carry an entry for every stage id `pipeline` resolves to,
|
|
830
|
+
including every stage inherited through `extends` **and every contribution that
|
|
831
|
+
might be placed** — task 1.11 widens the same requirement the module docstring
|
|
832
|
+
already states for `extends`, since a contributed id is unknown to the caller's
|
|
833
|
+
`parents` chain the same way an inherited one already was. A missing entry is a
|
|
834
|
+
programming error in the caller, not a pipeline-authoring mistake, so it surfaces
|
|
835
|
+
as a plain `KeyError` naming the stage rather than a `WeftError`: no pipeline
|
|
836
|
+
author can fix a caller's own omission by editing a document.
|
|
837
|
+
|
|
838
|
+
Raises `UnknownParentPipelineError` or `PipelineCycleError` while walking
|
|
839
|
+
`extends`; then `StaleOperatorTargetError` or `OperatorIdCollisionError` while
|
|
840
|
+
folding operators onto the root's stage list (task 1.4); then, task 1.11,
|
|
841
|
+
`StaleOperatorTargetError` again for a slot whose own anchor went missing, or
|
|
842
|
+
`SlotOrderConflictError` if two contributions to one slot cannot be ordered at all;
|
|
843
|
+
then, once the merged stage list and the merged `vars` are known, per stage in
|
|
844
|
+
order: `UnknownPluginError` (`Registry.entry`, reused), `StageCompositionError`,
|
|
845
|
+
`UnmetRequiresError`, `IntactViolationError`, `UndefinedVarError` — the five checks
|
|
846
|
+
`02` §3 names for a stage list — and finally, task 1.5, `InvalidStageConfigError` or
|
|
847
|
+
`StageNotConfigurableError` once that stage's `with:` block is fully literal. Run
|
|
848
|
+
once against the list operators and slot-fill have already finished changing, so a
|
|
849
|
+
pipeline with more than one problem always reports the cheapest one first.
|
|
850
|
+
"""
|
|
851
|
+
ancestry, merged_vars = _walk_extends(pipeline, parents)
|
|
852
|
+
working_stages, working_slots, provenance, deferred_sets = _apply_ancestry_operators(ancestry)
|
|
853
|
+
unplaced = _fill_slots(
|
|
854
|
+
working_stages,
|
|
855
|
+
working_slots,
|
|
856
|
+
provenance,
|
|
857
|
+
contributions,
|
|
858
|
+
registry=registry,
|
|
859
|
+
contracts=contracts,
|
|
860
|
+
pipeline_name=pipeline.name,
|
|
861
|
+
)
|
|
862
|
+
unapplied = _apply_deferred_sets(working_stages, deferred_sets)
|
|
863
|
+
_check_composition(tuple(working_stages), contracts, pipeline_name=pipeline.name)
|
|
864
|
+
|
|
865
|
+
resolved_stages: list[ResolvedStage] = []
|
|
866
|
+
provided: set[type[object]] = set()
|
|
867
|
+
destroyed_by: dict[type[object], str] = {}
|
|
868
|
+
for stage in working_stages:
|
|
869
|
+
contract = contracts[stage.id]
|
|
870
|
+
entry = registry.entry(contract, stage.use)
|
|
871
|
+
declared = unwrap_factory(entry.factory)
|
|
872
|
+
|
|
873
|
+
for required in cast("tuple[type[ExtModel], ...]", getattr(declared, "requires", ())):
|
|
874
|
+
if required not in provided:
|
|
875
|
+
available = ", ".join(sorted(model.__name__ for model in provided)) or "(none)"
|
|
876
|
+
raise UnmetRequiresError(
|
|
877
|
+
f"stage '{stage.id}' ({contract.__name__}:{stage.use}) requires "
|
|
878
|
+
f"'{required.__name__}' but no earlier stage in pipeline "
|
|
879
|
+
f"'{pipeline.name}' provides it. Provided so far: {available}.",
|
|
880
|
+
pipeline=pipeline.name,
|
|
881
|
+
stages=(stage.id,),
|
|
882
|
+
distributions=(required.__namespace__,),
|
|
883
|
+
remedy=(
|
|
884
|
+
f"add an earlier stage that provides '{required.__name__}', or "
|
|
885
|
+
f"reorder '{pipeline.name}' so one already does."
|
|
886
|
+
),
|
|
887
|
+
)
|
|
888
|
+
for needed_intact in cast("tuple[type[object], ...]", getattr(declared, "intact", ())):
|
|
889
|
+
destroyer = destroyed_by.get(needed_intact)
|
|
890
|
+
if destroyer is not None:
|
|
891
|
+
raise IntactViolationError(
|
|
892
|
+
f"stage '{stage.id}' ({contract.__name__}:{stage.use}) needs "
|
|
893
|
+
f"'{needed_intact.__name__}' intact, but stage '{destroyer}' earlier in "
|
|
894
|
+
f"pipeline '{pipeline.name}' already destroys it. The only legal "
|
|
895
|
+
f"positions for '{stage.id}' are before '{destroyer}', never after.",
|
|
896
|
+
pipeline=pipeline.name,
|
|
897
|
+
stages=(stage.id, destroyer),
|
|
898
|
+
remedy=f"move '{stage.id}' to before '{destroyer}', never after.",
|
|
899
|
+
)
|
|
900
|
+
provided.update(cast("tuple[type[object], ...]", getattr(declared, "provides", ())))
|
|
901
|
+
for destroyed in cast("tuple[type[object], ...]", getattr(declared, "destroys", ())):
|
|
902
|
+
destroyed_by.setdefault(destroyed, stage.id)
|
|
903
|
+
|
|
904
|
+
substituted_config = _substitute_vars(
|
|
905
|
+
stage.config, merged_vars, pipeline_name=pipeline.name, stage_id=stage.id
|
|
906
|
+
)
|
|
907
|
+
validated_config = _validate_stage_config(
|
|
908
|
+
substituted_config,
|
|
909
|
+
declared=declared,
|
|
910
|
+
stage_id=stage.id,
|
|
911
|
+
contract=contract,
|
|
912
|
+
use=stage.use,
|
|
913
|
+
pipeline_name=pipeline.name,
|
|
914
|
+
)
|
|
915
|
+
|
|
916
|
+
resolved_stages.append(
|
|
917
|
+
ResolvedStage(
|
|
918
|
+
id=stage.id,
|
|
919
|
+
contract=contract.__name__,
|
|
920
|
+
contract_version=getattr(contract, "version", None),
|
|
921
|
+
use=stage.use,
|
|
922
|
+
config=validated_config,
|
|
923
|
+
fallback=stage.fallback,
|
|
924
|
+
applies_to=cast("tuple[Applies, ...]", getattr(declared, "applies_to", ())),
|
|
925
|
+
distribution=entry.distribution,
|
|
926
|
+
provenance=provenance[stage.id],
|
|
927
|
+
)
|
|
928
|
+
)
|
|
929
|
+
|
|
930
|
+
return ResolvedPipeline(
|
|
931
|
+
name=pipeline.name,
|
|
932
|
+
vars=merged_vars,
|
|
933
|
+
stages=tuple(resolved_stages),
|
|
934
|
+
unapplied_operators=tuple(unapplied),
|
|
935
|
+
unplaced_contributions=tuple(unplaced),
|
|
936
|
+
)
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
def _walk_extends(
|
|
940
|
+
pipeline: Pipeline, parents: Mapping[str, Pipeline]
|
|
941
|
+
) -> tuple[tuple[Pipeline, ...], dict[str, Scalar]]:
|
|
942
|
+
"""Follow `extends` to its root, returning the whole chain root-first, and merged vars.
|
|
943
|
+
|
|
944
|
+
`chain` records every pipeline name visited, in order, purely to make
|
|
945
|
+
`PipelineCycleError` name the whole loop rather than the one repeated name. Vars
|
|
946
|
+
merge root to leaf — each ancestor visited updates the accumulator in the order it
|
|
947
|
+
was walked away from the root, so a leaf's own `vars:` wins last, which is what
|
|
948
|
+
lets `test_a_child_that_only_retargets_a_var_...` see the child's override rather
|
|
949
|
+
than the root's original value. `parents` is read fresh on every call and never
|
|
950
|
+
copied — `02` §3 → *Derivation*: "the parent is referenced, never copied: resolution
|
|
951
|
+
reads live parents" — so editing the `Pipeline` a caller's `parents` mapping points
|
|
952
|
+
at, or handing `resolve()` a different mapping entirely, changes what the very next
|
|
953
|
+
call sees with no cache anywhere in between to go stale.
|
|
954
|
+
"""
|
|
955
|
+
chain: list[str] = []
|
|
956
|
+
ancestry: list[Pipeline] = []
|
|
957
|
+
current = pipeline
|
|
958
|
+
while True:
|
|
959
|
+
if current.name in chain:
|
|
960
|
+
raise PipelineCycleError(
|
|
961
|
+
f"pipeline '{pipeline.name}' has a cycle in its 'extends' chain: "
|
|
962
|
+
f"{' -> '.join([*chain, current.name])}. A pipeline cannot extend itself, "
|
|
963
|
+
f"directly or through any number of intermediate parents.",
|
|
964
|
+
pipeline=pipeline.name,
|
|
965
|
+
remedy=(
|
|
966
|
+
f"break the cycle: {' -> '.join([*chain, current.name])} — remove or "
|
|
967
|
+
f"retarget one 'extends' link in that chain."
|
|
968
|
+
),
|
|
969
|
+
)
|
|
970
|
+
chain.append(current.name)
|
|
971
|
+
ancestry.append(current)
|
|
972
|
+
if current.extends is None:
|
|
973
|
+
break
|
|
974
|
+
parent = parents.get(current.extends)
|
|
975
|
+
if parent is None:
|
|
976
|
+
options = tuple(sorted(parents))
|
|
977
|
+
available = ", ".join(repr(candidate) for candidate in options) or "(none)"
|
|
978
|
+
raise UnknownParentPipelineError(
|
|
979
|
+
f"pipeline '{current.name}' extends '{current.extends}', but the parent "
|
|
980
|
+
f"lookup this resolve() call was given has no pipeline named that. Supply it "
|
|
981
|
+
f"in 'parents', or fix the name if it was mistyped. Pipelines available in "
|
|
982
|
+
f"'parents': {available}.",
|
|
983
|
+
valid_options=options,
|
|
984
|
+
pipeline=current.name,
|
|
985
|
+
remedy=(
|
|
986
|
+
f"add '{current.extends}' to 'parents', or fix '{current.name}'s "
|
|
987
|
+
f"'extends:' if it was mistyped."
|
|
988
|
+
),
|
|
989
|
+
)
|
|
990
|
+
current = parent
|
|
991
|
+
|
|
992
|
+
ancestry.reverse() # root first
|
|
993
|
+
merged_vars: dict[str, Scalar] = {}
|
|
994
|
+
for ancestor in ancestry:
|
|
995
|
+
merged_vars.update(ancestor.vars)
|
|
996
|
+
|
|
997
|
+
return tuple(ancestry), merged_vars
|
|
998
|
+
|
|
999
|
+
|
|
1000
|
+
def _apply_ancestry_operators(
|
|
1001
|
+
ancestry: tuple[Pipeline, ...],
|
|
1002
|
+
) -> tuple[
|
|
1003
|
+
list[StageDeclaration], list[SlotDeclaration], dict[str, str], list[tuple[SetOperator, str]]
|
|
1004
|
+
]:
|
|
1005
|
+
"""Fold every descendant's operators onto the root's stage list, root to leaf.
|
|
1006
|
+
|
|
1007
|
+
`02` §3 → *Derivation*: "resolve the parent completely, then apply this pipeline's
|
|
1008
|
+
operators to that result" — "the same operation at depth one and depth five". The
|
|
1009
|
+
root (`ancestry[0]`) is the only pipeline that may carry `stages:` or `slots:`
|
|
1010
|
+
(`Pipeline._extends_and_stages_are_mutually_exclusive_with_operators`); every
|
|
1011
|
+
pipeline after it in the chain contributes only operators, applied by
|
|
1012
|
+
`_apply_operators` against the result the ancestor *before* it left — never against
|
|
1013
|
+
the original root — which is what lets a grandchild's `remove` see an id its own
|
|
1014
|
+
parent inserted, and what makes depth "the same operation" rather than a special
|
|
1015
|
+
case.
|
|
1016
|
+
|
|
1017
|
+
Task 1.11 gives this two more things to carry through, alongside `stages` and
|
|
1018
|
+
`provenance`: `slots`, so a descendant's `remove: <slot-id>` can drop one before slot
|
|
1019
|
+
fill ever runs (`02` §3: "operators address the author's own stages and slots"); and
|
|
1020
|
+
`deferred_sets`, every `set` operator whose target already carries a pack's qualifier
|
|
1021
|
+
(`_apply_sets` below) — those cannot apply here, because a contributed id does not
|
|
1022
|
+
exist until slot fill runs *after* every ancestor's operators have, so `resolve()`
|
|
1023
|
+
applies them separately, once slot fill is done (`_apply_deferred_sets`).
|
|
1024
|
+
"""
|
|
1025
|
+
root = ancestry[0]
|
|
1026
|
+
stages: list[StageDeclaration] = list(root.stages)
|
|
1027
|
+
slots: list[SlotDeclaration] = list(root.slots)
|
|
1028
|
+
provenance: dict[str, str] = {stage.id: root.name for stage in stages}
|
|
1029
|
+
deferred_sets: list[tuple[SetOperator, str]] = []
|
|
1030
|
+
for descendant in ancestry[1:]:
|
|
1031
|
+
_apply_operators(stages, slots, provenance, deferred_sets, descendant)
|
|
1032
|
+
return stages, slots, provenance, deferred_sets
|
|
1033
|
+
|
|
1034
|
+
|
|
1035
|
+
def _apply_operators(
|
|
1036
|
+
stages: list[StageDeclaration],
|
|
1037
|
+
slots: list[SlotDeclaration],
|
|
1038
|
+
provenance: dict[str, str],
|
|
1039
|
+
deferred_sets: list[tuple[SetOperator, str]],
|
|
1040
|
+
pipeline: Pipeline,
|
|
1041
|
+
) -> None:
|
|
1042
|
+
"""Apply one pipeline's own operator blocks, in the order its document wrote them.
|
|
1043
|
+
|
|
1044
|
+
`02` §3, settled by task 1.4: "the order in which those keys appear in the document
|
|
1045
|
+
is the order they apply." `pipeline.operator_order` is read off the mapping or the
|
|
1046
|
+
keyword arguments that built `pipeline`, never assumed from field declaration order
|
|
1047
|
+
— see `weft_kernel.pipeline`'s own module docstring for why assuming it would make
|
|
1048
|
+
one of *remove-then-insert* and *insert-then-remove* permanently unwritable. `stages`,
|
|
1049
|
+
`slots` and `provenance` are mutated in place: they *are* "the running result" `02`
|
|
1050
|
+
§3 means by "each validated against the running result", carried from block to block
|
|
1051
|
+
within one pipeline and then on to the next descendant in `_apply_ancestry_operators`.
|
|
1052
|
+
"""
|
|
1053
|
+
for key in pipeline.operator_order:
|
|
1054
|
+
if key == "insert":
|
|
1055
|
+
_apply_inserts(stages, provenance, pipeline)
|
|
1056
|
+
elif key == "replace":
|
|
1057
|
+
_apply_replaces(stages, provenance, pipeline)
|
|
1058
|
+
elif key == "remove":
|
|
1059
|
+
_apply_removes(stages, slots, provenance, pipeline)
|
|
1060
|
+
elif key == "set":
|
|
1061
|
+
_apply_sets(stages, provenance, deferred_sets, pipeline)
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
def _apply_inserts(
|
|
1065
|
+
stages: list[StageDeclaration], provenance: dict[str, str], pipeline: Pipeline
|
|
1066
|
+
) -> None:
|
|
1067
|
+
"""`insert`: add `op.stage`, positioned `after:`/`before:` an id already in `stages`.
|
|
1068
|
+
|
|
1069
|
+
Refuses a colliding new id before it refuses a missing target, so two problems in one
|
|
1070
|
+
`insert` entry always report the collision — the cheaper, purely-local check.
|
|
1071
|
+
"""
|
|
1072
|
+
for op in pipeline.insert:
|
|
1073
|
+
_refuse_id_collision(stages, op.stage.id, pipeline=pipeline)
|
|
1074
|
+
target = op.after if op.after is not None else op.before
|
|
1075
|
+
index = _index_of(stages, cast("str", target), pipeline=pipeline, operator="insert")
|
|
1076
|
+
position = index + 1 if op.after is not None else index
|
|
1077
|
+
stages.insert(position, op.stage)
|
|
1078
|
+
provenance[op.stage.id] = pipeline.name
|
|
1079
|
+
|
|
1080
|
+
|
|
1081
|
+
def _apply_replaces(
|
|
1082
|
+
stages: list[StageDeclaration], provenance: dict[str, str], pipeline: Pipeline
|
|
1083
|
+
) -> None:
|
|
1084
|
+
"""`replace`: swap the `StageDeclaration` at an existing id, keeping its position.
|
|
1085
|
+
|
|
1086
|
+
`pipeline.replace` reuses `StageDeclaration` itself rather than a bespoke operator
|
|
1087
|
+
model — its own `id` field *is* the target, so "keeping its position" is not a rule
|
|
1088
|
+
this function enforces so much as a consequence of `id` staying fixed: `list.
|
|
1089
|
+
__setitem__` overwrites the slot `_index_of` found without moving anything else.
|
|
1090
|
+
"""
|
|
1091
|
+
for replacement in pipeline.replace:
|
|
1092
|
+
index = _index_of(stages, replacement.id, pipeline=pipeline, operator="replace")
|
|
1093
|
+
stages[index] = replacement
|
|
1094
|
+
provenance[replacement.id] = pipeline.name
|
|
1095
|
+
|
|
1096
|
+
|
|
1097
|
+
def _apply_removes(
|
|
1098
|
+
stages: list[StageDeclaration],
|
|
1099
|
+
slots: list[SlotDeclaration],
|
|
1100
|
+
provenance: dict[str, str],
|
|
1101
|
+
pipeline: Pipeline,
|
|
1102
|
+
) -> None:
|
|
1103
|
+
"""`remove`: drop a stage **or a slot** by id. No exemption from strictness — `02` §3.
|
|
1104
|
+
|
|
1105
|
+
Task 1.11, `02` §3 → *Slots*: "`remove: enrich` drops the slot itself, which is how a
|
|
1106
|
+
pipeline refuses contributions without naming any pack." A slot id is checked first —
|
|
1107
|
+
`Pipeline._slot_ids_are_unique_and_free` already keeps the two vocabularies disjoint,
|
|
1108
|
+
so this is never an ambiguous choice, only an ordering of which lookup to try. Neither
|
|
1109
|
+
lookup finding `target` gets no softer treatment than task 1.4's own `remove` did:
|
|
1110
|
+
still `StaleOperatorTargetError`, now naming both the stage ids and the slot ids that
|
|
1111
|
+
do exist, since a typo could plausibly have meant either.
|
|
1112
|
+
"""
|
|
1113
|
+
for target in pipeline.remove:
|
|
1114
|
+
slot_index = next((i for i, slot in enumerate(slots) if slot.id == target), None)
|
|
1115
|
+
if slot_index is not None:
|
|
1116
|
+
del slots[slot_index]
|
|
1117
|
+
continue
|
|
1118
|
+
stage_index = next((i for i, stage in enumerate(stages) if stage.id == target), None)
|
|
1119
|
+
if stage_index is None:
|
|
1120
|
+
stage_ids = tuple(stage.id for stage in stages)
|
|
1121
|
+
slot_ids = tuple(slot.id for slot in slots)
|
|
1122
|
+
existing_stages = ", ".join(repr(stage_id) for stage_id in stage_ids) or "(none)"
|
|
1123
|
+
existing_slots = ", ".join(repr(slot_id) for slot_id in slot_ids) or "(none)"
|
|
1124
|
+
raise StaleOperatorTargetError(
|
|
1125
|
+
f"pipeline '{pipeline.name}' extends '{pipeline.extends}' and its 'remove' "
|
|
1126
|
+
f"operator targets id '{target}', but no stage or slot with that id exists "
|
|
1127
|
+
f"in the parent it resolved against at this point in the chain. Stage ids "
|
|
1128
|
+
f"that do exist: {existing_stages}. Slot ids that do exist: {existing_slots}.",
|
|
1129
|
+
valid_options=stage_ids + slot_ids,
|
|
1130
|
+
pipeline=pipeline.name,
|
|
1131
|
+
stages=(target,),
|
|
1132
|
+
remedy=(
|
|
1133
|
+
f"fix the 'remove' target — stage ids that do exist: {existing_stages}; "
|
|
1134
|
+
f"slot ids that do exist: {existing_slots}."
|
|
1135
|
+
),
|
|
1136
|
+
)
|
|
1137
|
+
del stages[stage_index]
|
|
1138
|
+
del provenance[target]
|
|
1139
|
+
|
|
1140
|
+
|
|
1141
|
+
def _apply_sets(
|
|
1142
|
+
stages: list[StageDeclaration],
|
|
1143
|
+
provenance: dict[str, str],
|
|
1144
|
+
deferred_sets: list[tuple[SetOperator, str]],
|
|
1145
|
+
pipeline: Pipeline,
|
|
1146
|
+
) -> None:
|
|
1147
|
+
"""`set`: override configuration at an id, plugin/position/provenance untouched.
|
|
1148
|
+
|
|
1149
|
+
`{**current.config, **op.config}` builds a **new** plain `dict` rather than writing
|
|
1150
|
+
through `current.config` — `weft_kernel.pipeline._read_only` wraps every `with:`
|
|
1151
|
+
block in a `MappingProxyType` precisely so an in-place update raises `TypeError`
|
|
1152
|
+
instead of silently mutating a mapping the parent, and every *other* child of that
|
|
1153
|
+
parent, still shares. `provenance` is deliberately left alone: `02` §3's phrase is
|
|
1154
|
+
"who put this stage's plugin here", and `set` never touches which plugin that is —
|
|
1155
|
+
see `ResolvedStage.provenance`'s own docstring for the definition this obeys. An
|
|
1156
|
+
earlier version of this function moved `provenance` on every `set`, on the reasoning
|
|
1157
|
+
that "current behaviour" was the more forensically useful thing to name; that read
|
|
1158
|
+
contradicted the definition this module and `ResolvedStage` both state, so it was the
|
|
1159
|
+
function that was wrong, not the two docstrings.
|
|
1160
|
+
|
|
1161
|
+
Task 1.11 splits this operator in two. A target carrying no pack qualifier is an
|
|
1162
|
+
ordinary stage id, unchanged from task 1.4, applied here and now — `_index_of`
|
|
1163
|
+
strict as ever. A target carrying one (`_QUALIFIER in op.id`) names a stage that,
|
|
1164
|
+
at this point in resolution, cannot possibly exist yet: slot fill runs only *after*
|
|
1165
|
+
every ancestor's operators have (`02` §3: "slots fill after the extends chain
|
|
1166
|
+
resolves"), so applying it here would always be `StaleOperatorTargetError`,
|
|
1167
|
+
whether or not the pack it names is actually installed. `Pipeline.
|
|
1168
|
+
_remove_targets_are_not_a_packs_to_name` already refuses this shape for `remove`; a
|
|
1169
|
+
`set` is different precisely because `02` §3 grants it the one exception — "may be
|
|
1170
|
+
`set` but never `replaced` or `removed`" — so instead of applying or refusing it
|
|
1171
|
+
now, it is appended to `deferred_sets` and settled by `_apply_deferred_sets`, once
|
|
1172
|
+
slot fill has had its chance to make the target real.
|
|
1173
|
+
"""
|
|
1174
|
+
for op in pipeline.set:
|
|
1175
|
+
if _QUALIFIER in op.id:
|
|
1176
|
+
deferred_sets.append((op, pipeline.name))
|
|
1177
|
+
continue
|
|
1178
|
+
index = _index_of(stages, op.id, pipeline=pipeline, operator="set")
|
|
1179
|
+
current = stages[index]
|
|
1180
|
+
merged_config = {**current.config, **op.config}
|
|
1181
|
+
stages[index] = StageDeclaration(
|
|
1182
|
+
id=current.id, use=current.use, config=merged_config, fallback=current.fallback
|
|
1183
|
+
)
|
|
1184
|
+
|
|
1185
|
+
|
|
1186
|
+
def _fill_slots(
|
|
1187
|
+
stages: list[StageDeclaration],
|
|
1188
|
+
slots: list[SlotDeclaration],
|
|
1189
|
+
provenance: dict[str, str],
|
|
1190
|
+
contributions: tuple[Contribution, ...],
|
|
1191
|
+
*,
|
|
1192
|
+
registry: Registry,
|
|
1193
|
+
contracts: Mapping[str, type[object]],
|
|
1194
|
+
pipeline_name: str,
|
|
1195
|
+
) -> list[str]:
|
|
1196
|
+
"""Fill every declared slot with its ordered contributions, in place. Task 1.11.
|
|
1197
|
+
|
|
1198
|
+
`02` §3 → *Slots*: "slots fill after the extends chain resolves" — every ancestor's
|
|
1199
|
+
operators have already run by the time `resolve()` calls this, so `stages`/`slots`
|
|
1200
|
+
are the same "running result" they were the whole way through, just with nothing
|
|
1201
|
+
left to change it but this. Every declared slot's own anchor is checked here, via
|
|
1202
|
+
`_slot_anchor_index`, **whether or not anything contributes to it** — a slot whose
|
|
1203
|
+
`after:`/`before:` target a descendant's own `remove` quietly took out from under it
|
|
1204
|
+
is a genuine document defect independent of any pack, and `weft pipeline validate`
|
|
1205
|
+
with no packs installed at all must still be able to see it (`StaleOperatorTargetError`,
|
|
1206
|
+
see its own docstring's task 1.11 addendum) — never only once someone happens to try
|
|
1207
|
+
filling it.
|
|
1208
|
+
|
|
1209
|
+
Refuses a duplicate qualified id across every contribution before any of it is
|
|
1210
|
+
grouped or ordered — see `_refuse_duplicate_contributions`.
|
|
1211
|
+
|
|
1212
|
+
Returns every contribution that named a slot this pipeline does not have — `02` §3:
|
|
1213
|
+
"a contribution with no matching slot is a recorded no-op". Multiple slots fill from
|
|
1214
|
+
a snapshot of every anchor position taken before any insertion, applied highest
|
|
1215
|
+
position first so an earlier slot's insertion never shifts a later slot's already-
|
|
1216
|
+
computed index — the identical reasoning `_apply_inserts` already relies on for a
|
|
1217
|
+
single stage, just batched — **and, among slots sharing one anchor, latest-declared
|
|
1218
|
+
first**: `sorted(..., reverse=True)` is stable, so without a declaration-position
|
|
1219
|
+
tie-break two same-anchor slots would keep list order going *in*, and inserting both
|
|
1220
|
+
at one position then reverses them coming *out* (`stages[index:index] = placed`
|
|
1221
|
+
pushes each new batch in front of the one already there). Carrying each slot's own
|
|
1222
|
+
`enumerate` position as the tie-break and reversing it too means the *later*-declared
|
|
1223
|
+
slot is inserted first and gets pushed in front by the *earlier*-declared one's
|
|
1224
|
+
insertion right after it — restoring the declared order rather than reversing it.
|
|
1225
|
+
`02` §3's own rule for the document itself, "the written order is the pipeline,"
|
|
1226
|
+
applies here without an exception for two slots naming the same anchor.
|
|
1227
|
+
"""
|
|
1228
|
+
_refuse_duplicate_contributions(contributions, pipeline_name=pipeline_name)
|
|
1229
|
+
|
|
1230
|
+
by_slot: dict[str, list[Contribution]] = {}
|
|
1231
|
+
for contribution in contributions:
|
|
1232
|
+
by_slot.setdefault(contribution.slot, []).append(contribution)
|
|
1233
|
+
|
|
1234
|
+
declared_ids = {slot.id for slot in slots}
|
|
1235
|
+
unplaced = [
|
|
1236
|
+
f"{_qualify(contribution)} -> slot '{contribution.slot}' (pipeline '{pipeline_name}' "
|
|
1237
|
+
f"declares no such slot)"
|
|
1238
|
+
for contribution in contributions
|
|
1239
|
+
if contribution.slot not in declared_ids
|
|
1240
|
+
]
|
|
1241
|
+
|
|
1242
|
+
fills: list[tuple[int, int, list[StageDeclaration]]] = []
|
|
1243
|
+
for declaration_position, slot in enumerate(slots):
|
|
1244
|
+
index = _slot_anchor_index(stages, slot, pipeline_name=pipeline_name)
|
|
1245
|
+
entries = by_slot.get(slot.id, [])
|
|
1246
|
+
if not entries:
|
|
1247
|
+
continue
|
|
1248
|
+
ordered = _order_contributions(
|
|
1249
|
+
entries, registry=registry, contracts=contracts, pipeline_name=pipeline_name
|
|
1250
|
+
)
|
|
1251
|
+
placed: list[StageDeclaration] = []
|
|
1252
|
+
for contribution in ordered:
|
|
1253
|
+
qualified_id = _qualify(contribution)
|
|
1254
|
+
placed.append(_placed_stage(qualified_id, contribution.stage))
|
|
1255
|
+
provenance[qualified_id] = contribution.distribution
|
|
1256
|
+
fills.append((index, declaration_position, placed))
|
|
1257
|
+
|
|
1258
|
+
for index, _, placed in sorted(fills, key=lambda item: (item[0], item[1]), reverse=True):
|
|
1259
|
+
stages[index:index] = placed
|
|
1260
|
+
|
|
1261
|
+
return unplaced
|
|
1262
|
+
|
|
1263
|
+
|
|
1264
|
+
def _slot_anchor_index(
|
|
1265
|
+
stages: list[StageDeclaration], slot: SlotDeclaration, *, pipeline_name: str
|
|
1266
|
+
) -> int:
|
|
1267
|
+
"""Where `slot` inserts into `stages` — `after:`/`before:`, on `_apply_inserts`'s own terms."""
|
|
1268
|
+
target = slot.after if slot.after is not None else slot.before
|
|
1269
|
+
for position, stage in enumerate(stages):
|
|
1270
|
+
if stage.id == target:
|
|
1271
|
+
return position + 1 if slot.after is not None else position
|
|
1272
|
+
options = tuple(stage.id for stage in stages)
|
|
1273
|
+
existing = ", ".join(repr(stage_id) for stage_id in options) or "(none)"
|
|
1274
|
+
raise StaleOperatorTargetError(
|
|
1275
|
+
f"pipeline '{pipeline_name}' declares slot '{slot.id}' positioned against stage id "
|
|
1276
|
+
f"'{target}', but no stage with that id exists in the fully resolved chain — an "
|
|
1277
|
+
f"ancestor's own operator likely removed or renamed it. The ids that do exist: "
|
|
1278
|
+
f"{existing}.",
|
|
1279
|
+
valid_options=options,
|
|
1280
|
+
pipeline=pipeline_name,
|
|
1281
|
+
stages=(cast("str", target),),
|
|
1282
|
+
remedy=(
|
|
1283
|
+
f"restore stage '{target}', or move slot '{slot.id}'s after:/before: to one "
|
|
1284
|
+
f"of the ids that still exist: {existing}."
|
|
1285
|
+
),
|
|
1286
|
+
)
|
|
1287
|
+
|
|
1288
|
+
|
|
1289
|
+
def _refuse_duplicate_contributions(
|
|
1290
|
+
contributions: tuple[Contribution, ...], *, pipeline_name: str
|
|
1291
|
+
) -> None:
|
|
1292
|
+
"""Every contribution's qualified id must be unique before slot-fill groups or orders any.
|
|
1293
|
+
|
|
1294
|
+
Checked globally, across every slot at once, rather than one slot's own entries at a
|
|
1295
|
+
time: two contributions naming the same qualified id are exactly as dangerous whether
|
|
1296
|
+
they target the same slot or two different ones, since either way both would try to
|
|
1297
|
+
wear the identical id once placed into the merged stage list. See
|
|
1298
|
+
`DuplicateContributionError` for why this cannot be left to `_order_contributions`'s
|
|
1299
|
+
own dict-building to catch — by the time that runs, one of the two is already gone.
|
|
1300
|
+
"""
|
|
1301
|
+
seen: dict[str, Contribution] = {}
|
|
1302
|
+
for contribution in contributions:
|
|
1303
|
+
qualified = _qualify(contribution)
|
|
1304
|
+
earlier = seen.get(qualified)
|
|
1305
|
+
if earlier is not None:
|
|
1306
|
+
raise DuplicateContributionError(
|
|
1307
|
+
f"pipeline '{pipeline_name}': distribution '{contribution.distribution}' "
|
|
1308
|
+
f"offers stage id '{contribution.stage.id}' more than once — once for slot "
|
|
1309
|
+
f"'{earlier.slot}' and again for slot '{contribution.slot}' — and both "
|
|
1310
|
+
f"would resolve to the identical qualified id '{qualified}'. Give each "
|
|
1311
|
+
f"contribution its own local stage id.",
|
|
1312
|
+
pipeline=pipeline_name,
|
|
1313
|
+
stages=(qualified,),
|
|
1314
|
+
distributions=(contribution.distribution,),
|
|
1315
|
+
remedy=(
|
|
1316
|
+
f"give the contribution offered for slot '{contribution.slot}' its own "
|
|
1317
|
+
f"local stage id, distinct from the one already offered for slot "
|
|
1318
|
+
f"'{earlier.slot}'."
|
|
1319
|
+
),
|
|
1320
|
+
)
|
|
1321
|
+
seen[qualified] = contribution
|
|
1322
|
+
|
|
1323
|
+
|
|
1324
|
+
def _order_contributions(
|
|
1325
|
+
contributions: list[Contribution],
|
|
1326
|
+
*,
|
|
1327
|
+
registry: Registry,
|
|
1328
|
+
contracts: Mapping[str, type[object]],
|
|
1329
|
+
pipeline_name: str,
|
|
1330
|
+
) -> list[Contribution]:
|
|
1331
|
+
"""One slot's contributions, ordered by declared `intact`/`destroys`, ties by name.
|
|
1332
|
+
|
|
1333
|
+
`02` §3 → *Slots*: "two contributions in one slot are ordered by the declared
|
|
1334
|
+
[ordering] relations... genuine ties break by distribution name, so two machines
|
|
1335
|
+
with the same installs resolve identically." Mirrors the deterministic-topological-
|
|
1336
|
+
sort shape `resolve()`'s own main loop already uses for a whole pipeline — a
|
|
1337
|
+
contribution that needs a property intact can never be placed while one that
|
|
1338
|
+
destroys it is still unplaced — except tie-broken explicitly rather than falling out
|
|
1339
|
+
of list order, because there is no author-written order among contributions to sort
|
|
1340
|
+
by: they arrive from however many packs happen to be installed, in no order anyone
|
|
1341
|
+
chose.
|
|
1342
|
+
|
|
1343
|
+
A single contribution needs no registry lookup at all — the common case, and the one
|
|
1344
|
+
every existing test *without* task 1.11's own ordering fixtures exercises.
|
|
1345
|
+
`pipeline_name` — task 1.13 — exists solely so `SlotOrderConflictError` can name the
|
|
1346
|
+
pipeline on its own `pipeline` field; nothing here reads the value otherwise.
|
|
1347
|
+
"""
|
|
1348
|
+
if len(contributions) <= 1:
|
|
1349
|
+
return list(contributions)
|
|
1350
|
+
|
|
1351
|
+
declared: dict[str, object] = {}
|
|
1352
|
+
for contribution in contributions:
|
|
1353
|
+
qualified = _qualify(contribution)
|
|
1354
|
+
contract = contracts[qualified]
|
|
1355
|
+
entry = registry.entry(contract, contribution.stage.use)
|
|
1356
|
+
declared[qualified] = unwrap_factory(entry.factory)
|
|
1357
|
+
|
|
1358
|
+
remaining = {_qualify(c): c for c in contributions}
|
|
1359
|
+
destroys_of = {
|
|
1360
|
+
qid: set(cast("tuple[type[object], ...]", getattr(declared[qid], "destroys", ())))
|
|
1361
|
+
for qid in remaining
|
|
1362
|
+
}
|
|
1363
|
+
intact_of = {
|
|
1364
|
+
qid: set(cast("tuple[type[object], ...]", getattr(declared[qid], "intact", ())))
|
|
1365
|
+
for qid in remaining
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
ordered: list[Contribution] = []
|
|
1369
|
+
while remaining:
|
|
1370
|
+
ready = [
|
|
1371
|
+
qid
|
|
1372
|
+
for qid in remaining
|
|
1373
|
+
if not any(intact_of[other] & destroys_of[qid] for other in remaining if other != qid)
|
|
1374
|
+
]
|
|
1375
|
+
if not ready:
|
|
1376
|
+
names = ", ".join(sorted(remaining))
|
|
1377
|
+
distributions = tuple(sorted({remaining[qid].distribution for qid in remaining}))
|
|
1378
|
+
raise SlotOrderConflictError(
|
|
1379
|
+
f"contributions to slot '{contributions[0].slot}' cannot be ordered: "
|
|
1380
|
+
f"{names} each need a property intact that another destroys, with no legal "
|
|
1381
|
+
f"order between them. Fix the ordering declarations on the plugins involved.",
|
|
1382
|
+
pipeline=pipeline_name,
|
|
1383
|
+
stages=tuple(sorted(remaining)),
|
|
1384
|
+
distributions=distributions,
|
|
1385
|
+
remedy=(
|
|
1386
|
+
f"fix the intact/destroys declarations on the plugins behind {names} so "
|
|
1387
|
+
f"some order satisfies every constraint."
|
|
1388
|
+
),
|
|
1389
|
+
)
|
|
1390
|
+
chosen = min(ready, key=lambda qid: remaining[qid].distribution)
|
|
1391
|
+
ordered.append(remaining.pop(chosen))
|
|
1392
|
+
return ordered
|
|
1393
|
+
|
|
1394
|
+
|
|
1395
|
+
def _apply_deferred_sets(
|
|
1396
|
+
stages: list[StageDeclaration], deferred: list[tuple[SetOperator, str]]
|
|
1397
|
+
) -> list[str]:
|
|
1398
|
+
"""Apply every `set` operator that targeted a pack's qualified id, once slots are filled.
|
|
1399
|
+
|
|
1400
|
+
`02` §3 → *Slots*: "Installation-dependent targets are recorded, never fatal... `set:
|
|
1401
|
+
weft-kg:entities` where that pack is absent is an unapplied operator in the
|
|
1402
|
+
resolved form, not a resolution failure." `_apply_sets` deferred these rather than
|
|
1403
|
+
applying or refusing them, because a qualified id cannot exist until `_fill_slots`
|
|
1404
|
+
has had its chance to place it — checked here, against the *filled* `stages`, so a
|
|
1405
|
+
target that genuinely does not exist (its pack is not installed) is never confused
|
|
1406
|
+
with one that merely had not been placed yet.
|
|
1407
|
+
"""
|
|
1408
|
+
unapplied: list[str] = []
|
|
1409
|
+
for op, pipeline_name in deferred:
|
|
1410
|
+
index = next((i for i, stage in enumerate(stages) if stage.id == op.id), None)
|
|
1411
|
+
if index is None:
|
|
1412
|
+
unapplied.append(
|
|
1413
|
+
f"pipeline '{pipeline_name}' sets configuration for '{op.id}', but no pack "
|
|
1414
|
+
f"contributed that stage — the pack providing it is not installed. The set "
|
|
1415
|
+
f"is recorded, not applied."
|
|
1416
|
+
)
|
|
1417
|
+
continue
|
|
1418
|
+
current = stages[index]
|
|
1419
|
+
merged_config = {**current.config, **op.config}
|
|
1420
|
+
stages[index] = _qualified_stage(
|
|
1421
|
+
id=current.id, use=current.use, config=merged_config, fallback=current.fallback
|
|
1422
|
+
)
|
|
1423
|
+
return unapplied
|
|
1424
|
+
|
|
1425
|
+
|
|
1426
|
+
def _index_of(
|
|
1427
|
+
stages: list[StageDeclaration], target: str, *, pipeline: Pipeline, operator: str
|
|
1428
|
+
) -> int:
|
|
1429
|
+
"""The position of `target` in `stages`, or `StaleOperatorTargetError` naming everything."""
|
|
1430
|
+
for position, stage in enumerate(stages):
|
|
1431
|
+
if stage.id == target:
|
|
1432
|
+
return position
|
|
1433
|
+
options = tuple(stage.id for stage in stages)
|
|
1434
|
+
existing = ", ".join(repr(stage_id) for stage_id in options) or "(none)"
|
|
1435
|
+
raise StaleOperatorTargetError(
|
|
1436
|
+
f"pipeline '{pipeline.name}' extends '{pipeline.extends}' and its '{operator}' "
|
|
1437
|
+
f"operator targets stage id '{target}', but no stage with that id exists in the "
|
|
1438
|
+
f"parent it resolved against at this point in the chain. The ids that do exist: "
|
|
1439
|
+
f"{existing}.",
|
|
1440
|
+
valid_options=options,
|
|
1441
|
+
pipeline=pipeline.name,
|
|
1442
|
+
stages=(target,),
|
|
1443
|
+
remedy=f"fix the '{operator}' target — ids that do exist: {existing}.",
|
|
1444
|
+
)
|
|
1445
|
+
|
|
1446
|
+
|
|
1447
|
+
def _refuse_id_collision(
|
|
1448
|
+
stages: list[StageDeclaration], new_id: str, *, pipeline: Pipeline
|
|
1449
|
+
) -> None:
|
|
1450
|
+
"""`insert`'s new id must be free — `02` §3: it "would silently shadow a parent's stage"."""
|
|
1451
|
+
if any(stage.id == new_id for stage in stages):
|
|
1452
|
+
raise OperatorIdCollisionError(
|
|
1453
|
+
f"pipeline '{pipeline.name}' extends '{pipeline.extends}' and its 'insert' "
|
|
1454
|
+
f"operator adds stage id '{new_id}', but a stage with that id already exists "
|
|
1455
|
+
f"in the parent it resolved against — inserting it would silently shadow the "
|
|
1456
|
+
f"existing stage. Pick a different id, or use 'replace'/'set' if the intent "
|
|
1457
|
+
f"is to change the existing stage.",
|
|
1458
|
+
pipeline=pipeline.name,
|
|
1459
|
+
stages=(new_id,),
|
|
1460
|
+
remedy=f"pick a stage id other than '{new_id}', or use 'replace'/'set' instead.",
|
|
1461
|
+
)
|
|
1462
|
+
|
|
1463
|
+
|
|
1464
|
+
def _stage_signature(contract: type[object]) -> tuple[object, object]:
|
|
1465
|
+
"""The `(In, Out)` `contract` declared via `Stage[In, Out]` as one of its own bases.
|
|
1466
|
+
|
|
1467
|
+
A deliberate duplicate of `weft_kernel.runner._stage_signature`, not an import of
|
|
1468
|
+
it: that name is private to `runner`, and this module's own composition check runs
|
|
1469
|
+
over `StageDeclaration`s and a caller-supplied `contracts` mapping rather than
|
|
1470
|
+
`runner.StageSpec`s, so the two call sites read `__orig_bases__` off the same kind
|
|
1471
|
+
of object for two genuinely different resolution mechanisms. Ten lines duplicated
|
|
1472
|
+
on purpose reads better a year from now than a private cross-module import would.
|
|
1473
|
+
"""
|
|
1474
|
+
for base in getattr(contract, "__orig_bases__", ()):
|
|
1475
|
+
if typing.get_origin(base) is Stage:
|
|
1476
|
+
args = typing.get_args(base)
|
|
1477
|
+
if len(args) == 2: # noqa: PLR2004 - Stage is fixed at two type parameters
|
|
1478
|
+
return args[0], args[1]
|
|
1479
|
+
raise StageCompositionError(
|
|
1480
|
+
f"'{contract.__name__}' does not declare Stage[In, Out] as a base — every contract "
|
|
1481
|
+
f"used in a pipeline states what it consumes and produces, e.g. "
|
|
1482
|
+
f"class YourContract(Stage[list[In], list[Out]], Protocol).",
|
|
1483
|
+
remedy=(
|
|
1484
|
+
f"declare `class {contract.__name__}(Stage[In, Out], Protocol)` on the contract itself."
|
|
1485
|
+
),
|
|
1486
|
+
)
|
|
1487
|
+
|
|
1488
|
+
|
|
1489
|
+
def _check_composition(
|
|
1490
|
+
stages: tuple[StageDeclaration, ...],
|
|
1491
|
+
contracts: Mapping[str, type[object]],
|
|
1492
|
+
*,
|
|
1493
|
+
pipeline_name: str,
|
|
1494
|
+
) -> None:
|
|
1495
|
+
"""Every consecutive pair of `stages` composes, checked purely against `contracts`.
|
|
1496
|
+
|
|
1497
|
+
Mirrors `weft_kernel.runner._check_composition` exactly, one payload type pair per
|
|
1498
|
+
contract read off `Stage[In, Out]`'s own `__orig_bases__` — the same reading `02`
|
|
1499
|
+
§3's own narrowing note describes. Run before any registry lookup, so a
|
|
1500
|
+
mis-ordered pipeline fails on the cheapest check first. `pipeline_name` — task
|
|
1501
|
+
1.13 — exists solely so `StageCompositionError` can name the pipeline.
|
|
1502
|
+
"""
|
|
1503
|
+
previous: tuple[str, object] | None = None
|
|
1504
|
+
for stage in stages:
|
|
1505
|
+
contract = contracts[stage.id]
|
|
1506
|
+
payload_type, produced_type = _stage_signature(contract)
|
|
1507
|
+
if previous is not None:
|
|
1508
|
+
previous_id, previous_produced = previous
|
|
1509
|
+
if previous_produced != payload_type:
|
|
1510
|
+
raise StageCompositionError(
|
|
1511
|
+
f"stage '{stage.id}' ({contract.__name__}:{stage.use}) expects "
|
|
1512
|
+
f"{payload_type!r}, but the previous stage '{previous_id}' produces "
|
|
1513
|
+
f"{previous_produced!r}. Consecutive stages must compose by type.",
|
|
1514
|
+
pipeline=pipeline_name,
|
|
1515
|
+
stages=(previous_id, stage.id),
|
|
1516
|
+
remedy=(
|
|
1517
|
+
f"reorder pipeline '{pipeline_name}' so '{previous_id}' precedes a "
|
|
1518
|
+
f"stage expecting {previous_produced!r}, or so '{stage.id}' follows "
|
|
1519
|
+
f"one producing {payload_type!r}."
|
|
1520
|
+
),
|
|
1521
|
+
)
|
|
1522
|
+
previous = (stage.id, produced_type)
|
|
1523
|
+
|
|
1524
|
+
|
|
1525
|
+
def _substitute_vars(
|
|
1526
|
+
config: Mapping[str, object],
|
|
1527
|
+
merged_vars: Mapping[str, Scalar],
|
|
1528
|
+
*,
|
|
1529
|
+
pipeline_name: str,
|
|
1530
|
+
stage_id: str,
|
|
1531
|
+
) -> Mapping[str, object]:
|
|
1532
|
+
"""Recursively resolve every `${var:NAME}` string in `config` against `merged_vars`.
|
|
1533
|
+
|
|
1534
|
+
Mirrors `weft_kernel.discovery.interpolate_env` exactly — see the module docstring
|
|
1535
|
+
for why. A string that is *exactly* `${var:NAME}` becomes that var's value; a
|
|
1536
|
+
string merely containing the token passes through untouched; `dict` and `list`
|
|
1537
|
+
recurse so a whole `with:` block substitutes in one call. `stage_id` is plumbed
|
|
1538
|
+
through for the identical reason `pipeline_name` already was — task 1.13's repair:
|
|
1539
|
+
an `UndefinedVarError` raised deep inside a nested `with:` block still needs to name
|
|
1540
|
+
which stage's block it came from, and the caller below is the only frame that knows.
|
|
1541
|
+
"""
|
|
1542
|
+
return cast(
|
|
1543
|
+
"Mapping[str, object]",
|
|
1544
|
+
_substitute(config, merged_vars, pipeline_name=pipeline_name, stage_id=stage_id),
|
|
1545
|
+
)
|
|
1546
|
+
|
|
1547
|
+
|
|
1548
|
+
def _substitute(
|
|
1549
|
+
value: object, merged_vars: Mapping[str, Scalar], *, pipeline_name: str, stage_id: str
|
|
1550
|
+
) -> object:
|
|
1551
|
+
if isinstance(value, str):
|
|
1552
|
+
match = _VAR_TOKEN.match(value)
|
|
1553
|
+
if match is None:
|
|
1554
|
+
return value
|
|
1555
|
+
name = match.group(1)
|
|
1556
|
+
if name not in merged_vars:
|
|
1557
|
+
options = tuple(sorted(merged_vars))
|
|
1558
|
+
defined = ", ".join(repr(key) for key in options) or "(none)"
|
|
1559
|
+
raise UndefinedVarError(
|
|
1560
|
+
f"'{value}' references var '{name}' in stage '{stage_id}', but pipeline "
|
|
1561
|
+
f"'{pipeline_name}' defines no such var — not directly, and none of its "
|
|
1562
|
+
f"ancestors do either. Add it to a 'vars:' block somewhere in the chain, or "
|
|
1563
|
+
f"fix the reference. Vars defined in this chain: {defined}.",
|
|
1564
|
+
valid_options=options,
|
|
1565
|
+
pipeline=pipeline_name,
|
|
1566
|
+
stages=(stage_id,),
|
|
1567
|
+
remedy=f"add 'vars: {{{name}: ...}}' somewhere in the chain, or fix the reference.",
|
|
1568
|
+
)
|
|
1569
|
+
return merged_vars[name]
|
|
1570
|
+
if isinstance(value, Mapping):
|
|
1571
|
+
items = cast("Mapping[str, object]", value)
|
|
1572
|
+
return {
|
|
1573
|
+
key: _substitute(item, merged_vars, pipeline_name=pipeline_name, stage_id=stage_id)
|
|
1574
|
+
for key, item in items.items()
|
|
1575
|
+
}
|
|
1576
|
+
if isinstance(value, list):
|
|
1577
|
+
entries = cast("list[object]", value)
|
|
1578
|
+
return [
|
|
1579
|
+
_substitute(item, merged_vars, pipeline_name=pipeline_name, stage_id=stage_id)
|
|
1580
|
+
for item in entries
|
|
1581
|
+
]
|
|
1582
|
+
return value
|