dirigent-plugin 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dirigent_plugin/__init__.py +115 -0
- dirigent_plugin/blocks.py +719 -0
- dirigent_plugin/markers.py +30 -0
- dirigent_plugin/py.typed +0 -0
- dirigent_plugin/transforms.py +540 -0
- dirigent_plugin-0.9.0.dist-info/METADATA +16 -0
- dirigent_plugin-0.9.0.dist-info/RECORD +9 -0
- dirigent_plugin-0.9.0.dist-info/WHEEL +4 -0
- dirigent_plugin-0.9.0.dist-info/licenses/LICENSE +18 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Plugin markers and the collecting extension points a dirigent plugin implements."""
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from pluginkit import Extension, ExtensionPoint
|
|
6
|
+
|
|
7
|
+
from dirigent_common import Formatter
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from dirigent_plugin.blocks import Contribution
|
|
11
|
+
|
|
12
|
+
PROJECT_NAME = "dirigent"
|
|
13
|
+
|
|
14
|
+
#: The version is part of the group name, so an incompatible contract ships as a new group.
|
|
15
|
+
ENTRY_POINT_GROUP = "dirigent.plugins.v1"
|
|
16
|
+
|
|
17
|
+
extension_point = ExtensionPoint(PROJECT_NAME)
|
|
18
|
+
extension = Extension(PROJECT_NAME)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@extension_point
|
|
22
|
+
def contribute() -> "Contribution":
|
|
23
|
+
"""Collect everything a plugin adds across the five surfaces, once at host startup."""
|
|
24
|
+
raise NotImplementedError("an extension point is a declaration; call it via PluginManager.caller(...)")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@extension_point
|
|
28
|
+
def formatters() -> list[Formatter]:
|
|
29
|
+
"""Collect the formatters a plugin adds to ``dg format``, once at CLI startup."""
|
|
30
|
+
raise NotImplementedError("an extension point is a declaration; call it via PluginManager.caller(...)")
|
dirigent_plugin/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,540 @@
|
|
|
1
|
+
"""The transform verb frames: one contract per verb, one engine per kind.
|
|
2
|
+
|
|
3
|
+
A transform block's id is ``<verb>.<kind>``. The verb names the contract and the semantic
|
|
4
|
+
promise the frame enforces; the kind names the engine that keeps it. Four verbs live here:
|
|
5
|
+
``transform``, an arbitrary whole-value reshape driven by a program; ``convert``, a
|
|
6
|
+
content-preserving re-encoding from one format to another; ``map``, whose output has the
|
|
7
|
+
same length as its input; and ``filter``, whose output is a subset of its input with the
|
|
8
|
+
elements unmodified.
|
|
9
|
+
|
|
10
|
+
The frame owns everything an engine would otherwise repeat: where the input comes from and
|
|
11
|
+
how much of it may be read, where the result goes, the catalog entry, and the apply-time
|
|
12
|
+
check that refuses a bad program or an unsupported format pair before a document is stored.
|
|
13
|
+
An engine supplies only the part that is specific to it.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
from abc import ABC, abstractmethod
|
|
18
|
+
from typing import Any, ClassVar, Final, cast
|
|
19
|
+
|
|
20
|
+
from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator
|
|
21
|
+
|
|
22
|
+
from dirigent_common import BlockModel, Size, StorageUri
|
|
23
|
+
from dirigent_plugin.blocks import (
|
|
24
|
+
BlockFailure,
|
|
25
|
+
ErrorClass,
|
|
26
|
+
Operator,
|
|
27
|
+
OperatorSpec,
|
|
28
|
+
RemoteHandle,
|
|
29
|
+
StepContext,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
#: The same bound ``http.request`` puts on a body it holds in memory.
|
|
33
|
+
MAX_INPUT_DEFAULT: Final = 32 * 1024 * 1024
|
|
34
|
+
|
|
35
|
+
#: How much is handed to a storage sink at a time.
|
|
36
|
+
CHUNK_BYTES: Final = 64 * 1024
|
|
37
|
+
|
|
38
|
+
#: What the ``map`` frame says it does, in the refusal of an input that is not a list.
|
|
39
|
+
MAP_PROMISE: Final = "replaces every element of a list"
|
|
40
|
+
|
|
41
|
+
#: What the ``filter`` frame says it does, in the same refusal.
|
|
42
|
+
FILTER_PROMISE: Final = "keeps some of the elements of a list"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class TransformError(Exception):
|
|
46
|
+
"""An engine refused what it was given: a program it cannot compile, or a value it cannot reshape.
|
|
47
|
+
|
|
48
|
+
Raised by an engine and turned into a rejected block failure by the frame, so an engine
|
|
49
|
+
never classifies a failure or constructs one itself.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class TransformConfig(BlockModel):
|
|
54
|
+
"""The half of a transform's config the frame owns, whatever the verb or the engine."""
|
|
55
|
+
|
|
56
|
+
model_config = ConfigDict(
|
|
57
|
+
use_attribute_docstrings=True,
|
|
58
|
+
# The published schema carries the exactly-one rule, so a document that breaks it is
|
|
59
|
+
# refused where every other config mistake is, rather than on the first run.
|
|
60
|
+
json_schema_extra={"oneOf": [{"required": ["input"]}, {"required": ["input_uri"]}]},
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
input: JsonValue | None = None
|
|
64
|
+
"""The value to work on, written inline in the document.
|
|
65
|
+
|
|
66
|
+
A ``null`` written here is read as no input at all, which is the one value that cannot
|
|
67
|
+
be passed inline."""
|
|
68
|
+
|
|
69
|
+
input_uri: StorageUri | None = None
|
|
70
|
+
"""A storage URI to read the input from instead of writing it inline."""
|
|
71
|
+
|
|
72
|
+
save_to: StorageUri | None = None
|
|
73
|
+
"""A storage URI to stream the result to, instead of carrying it inline.
|
|
74
|
+
|
|
75
|
+
The step's output then carries ``output_uri`` and ``output_bytes``: a result worth
|
|
76
|
+
saving is one the next step reads from storage."""
|
|
77
|
+
|
|
78
|
+
max_input: Size = MAX_INPUT_DEFAULT
|
|
79
|
+
"""How much of ``input_uri`` is read into memory.
|
|
80
|
+
|
|
81
|
+
A value has to be whole to be reshaped, so one too large to hold is refused rather than
|
|
82
|
+
truncated: half a document is not a smaller input, it is a wrong one."""
|
|
83
|
+
|
|
84
|
+
@model_validator(mode="after")
|
|
85
|
+
def _require_one_input(self) -> "TransformConfig":
|
|
86
|
+
"""Reject a config that gives both an inline value and a URI, or neither."""
|
|
87
|
+
if (self.input is None) == (self.input_uri is None):
|
|
88
|
+
raise ValueError("a transform reads either input or input_uri, and needs exactly one of the two")
|
|
89
|
+
return self
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class ProgramConfig(TransformConfig):
|
|
93
|
+
"""What a program-shaped engine is told: the shared fields, plus the program itself."""
|
|
94
|
+
|
|
95
|
+
program: str = Field(min_length=1)
|
|
96
|
+
"""The engine's program, in whatever language the engine's kind names."""
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class TransformOutput(BlockModel):
|
|
100
|
+
"""What one reshape produced: the value itself, or where it was written."""
|
|
101
|
+
|
|
102
|
+
value: JsonValue | None = None
|
|
103
|
+
"""The reshaped value, when it was not streamed to storage."""
|
|
104
|
+
|
|
105
|
+
output_uri: str | None = None
|
|
106
|
+
"""Where the result was written, when ``save_to`` asked for it."""
|
|
107
|
+
|
|
108
|
+
output_bytes: int | None = None
|
|
109
|
+
"""How many bytes were written there."""
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class ConvertConfig(TransformConfig):
|
|
113
|
+
"""What one re-encoding is told: where the bytes come from, and the pair of formats."""
|
|
114
|
+
|
|
115
|
+
input: str | None = None # pyright: ignore[reportIncompatibleVariableOverride] - a codec reads text, not JSON
|
|
116
|
+
"""The text to re-encode, written inline in the document."""
|
|
117
|
+
|
|
118
|
+
from_format: str = Field(alias="from", min_length=1)
|
|
119
|
+
"""The format the input is in, named as the engine names it."""
|
|
120
|
+
|
|
121
|
+
to_format: str = Field(alias="to", min_length=1)
|
|
122
|
+
"""The format to produce."""
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class ConvertOutput(BlockModel):
|
|
126
|
+
"""What one re-encoding produced: the text itself, or where it was written."""
|
|
127
|
+
|
|
128
|
+
text: str | None = None
|
|
129
|
+
"""The re-encoded text, when it was not streamed to storage."""
|
|
130
|
+
|
|
131
|
+
output_uri: str | None = None
|
|
132
|
+
"""Where the result was written, when ``save_to`` asked for it."""
|
|
133
|
+
|
|
134
|
+
output_bytes: int | None = None
|
|
135
|
+
"""How many bytes were written there."""
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class Transformer(Operator[ProgramConfig, TransformOutput], ABC):
|
|
139
|
+
"""The ``transform`` verb: reshape one whole value into another by running a program.
|
|
140
|
+
|
|
141
|
+
An engine names its kind, summarises itself in one line, and supplies the two halves of
|
|
142
|
+
running a program: compiling it, which is also what the apply-time check runs, and
|
|
143
|
+
applying it to a value. The frame derives the catalog entry, resolves the input, and
|
|
144
|
+
disposes of the result.
|
|
145
|
+
|
|
146
|
+
An engine touches no HTTP, no file outside storage, and nothing in the environment. It
|
|
147
|
+
is handed a value and returns a value; everything that reaches the world is the frame's.
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
kind: ClassVar[str]
|
|
151
|
+
"""The engine's name, which is the second half of the block id."""
|
|
152
|
+
|
|
153
|
+
summary: ClassVar[str]
|
|
154
|
+
"""The one line the catalog shows for this engine."""
|
|
155
|
+
|
|
156
|
+
local_execution: ClassVar[bool] = False
|
|
157
|
+
"""Whether this engine executes code on the worker, which puts it behind the allowlist."""
|
|
158
|
+
|
|
159
|
+
config_model: ClassVar[type[BaseModel]] = ProgramConfig
|
|
160
|
+
output_model: ClassVar[type[BaseModel]] = TransformOutput
|
|
161
|
+
|
|
162
|
+
def __init_subclass__(cls, **kwargs: Any) -> None:
|
|
163
|
+
"""Derive the catalog entry: the id is ``transform.<kind>`` and the group ``transform``."""
|
|
164
|
+
super().__init_subclass__(**kwargs)
|
|
165
|
+
kind = cls.__dict__.get("kind")
|
|
166
|
+
if kind is not None:
|
|
167
|
+
cls.spec = OperatorSpec(
|
|
168
|
+
id=f"transform.{kind}",
|
|
169
|
+
group="transform",
|
|
170
|
+
summary=cls.summary,
|
|
171
|
+
idempotent=True,
|
|
172
|
+
local_execution=cls.local_execution,
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
@abstractmethod
|
|
176
|
+
def compile(self, program: str) -> object:
|
|
177
|
+
"""Turn a program into whatever this engine applies, raising TransformError on a bad one."""
|
|
178
|
+
...
|
|
179
|
+
|
|
180
|
+
@abstractmethod
|
|
181
|
+
def apply(self, compiled: object, value: JsonValue) -> JsonValue:
|
|
182
|
+
"""Run a compiled program over one whole value."""
|
|
183
|
+
...
|
|
184
|
+
|
|
185
|
+
async def execute(self, config: ProgramConfig, ctx: StepContext) -> TransformOutput | RemoteHandle:
|
|
186
|
+
"""Resolve the input, run the program over it, and inline or store the result."""
|
|
187
|
+
value = await _read_value(config, ctx)
|
|
188
|
+
try:
|
|
189
|
+
result = self.apply(self.compile(config.program), value)
|
|
190
|
+
except TransformError as error:
|
|
191
|
+
raise BlockFailure(str(error), error_class=ErrorClass.REJECTED) from error
|
|
192
|
+
if config.save_to is None:
|
|
193
|
+
return TransformOutput(value=result)
|
|
194
|
+
written = await _write(ctx, config.save_to, json.dumps(result, separators=(",", ":")).encode())
|
|
195
|
+
return TransformOutput(output_uri=config.save_to, output_bytes=written)
|
|
196
|
+
|
|
197
|
+
def check_config(self, config: BaseModel) -> list[str]:
|
|
198
|
+
"""Compile the program at apply, so a bad one is refused before the document is stored."""
|
|
199
|
+
if not isinstance(config, ProgramConfig):
|
|
200
|
+
return []
|
|
201
|
+
try:
|
|
202
|
+
self.compile(config.program)
|
|
203
|
+
except TransformError as error:
|
|
204
|
+
return [str(error)]
|
|
205
|
+
return []
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
class Mapper(Operator[ProgramConfig, TransformOutput], ABC):
|
|
209
|
+
"""The ``map`` verb: replace every element of a list with what a program makes of it.
|
|
210
|
+
|
|
211
|
+
An engine names its kind, summarises itself in one line, and supplies compiling a
|
|
212
|
+
program and applying it -- to one element at a time, not to the whole list. The frame
|
|
213
|
+
derives the catalog entry, resolves the input, refuses an input that is not an array,
|
|
214
|
+
runs the loop, and disposes of the result.
|
|
215
|
+
|
|
216
|
+
The promise is length: the output has one element for every element of the input, in
|
|
217
|
+
input order. The frame builds it one element at a time, so the promise holds by
|
|
218
|
+
construction, and asserts it afterwards so an engine that reached past the loop fails
|
|
219
|
+
itself rather than quietly returning a shorter list.
|
|
220
|
+
|
|
221
|
+
An engine touches no HTTP, no file outside storage, and nothing in the environment. It
|
|
222
|
+
is handed an element and returns an element; everything that reaches the world is the
|
|
223
|
+
frame's.
|
|
224
|
+
"""
|
|
225
|
+
|
|
226
|
+
kind: ClassVar[str]
|
|
227
|
+
"""The engine's name, which is the second half of the block id."""
|
|
228
|
+
|
|
229
|
+
summary: ClassVar[str]
|
|
230
|
+
"""The one line the catalog shows for this engine."""
|
|
231
|
+
|
|
232
|
+
local_execution: ClassVar[bool] = False
|
|
233
|
+
"""Whether this engine executes code on the worker, which puts it behind the allowlist."""
|
|
234
|
+
|
|
235
|
+
config_model: ClassVar[type[BaseModel]] = ProgramConfig
|
|
236
|
+
output_model: ClassVar[type[BaseModel]] = TransformOutput
|
|
237
|
+
|
|
238
|
+
def __init_subclass__(cls, **kwargs: Any) -> None:
|
|
239
|
+
"""Derive the catalog entry: the id is ``map.<kind>`` and the group ``transform``."""
|
|
240
|
+
super().__init_subclass__(**kwargs)
|
|
241
|
+
kind = cls.__dict__.get("kind")
|
|
242
|
+
if kind is not None:
|
|
243
|
+
cls.spec = OperatorSpec(
|
|
244
|
+
id=f"map.{kind}",
|
|
245
|
+
group="transform",
|
|
246
|
+
summary=cls.summary,
|
|
247
|
+
idempotent=True,
|
|
248
|
+
local_execution=cls.local_execution,
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
@abstractmethod
|
|
252
|
+
def compile(self, program: str) -> object:
|
|
253
|
+
"""Turn a program into whatever this engine applies, raising TransformError on a bad one."""
|
|
254
|
+
...
|
|
255
|
+
|
|
256
|
+
@abstractmethod
|
|
257
|
+
def apply(self, compiled: object, value: JsonValue) -> JsonValue:
|
|
258
|
+
"""Run a compiled program over one element and return the element that replaces it."""
|
|
259
|
+
...
|
|
260
|
+
|
|
261
|
+
async def execute(self, config: ProgramConfig, ctx: StepContext) -> TransformOutput | RemoteHandle:
|
|
262
|
+
"""Resolve the input, replace every element, and inline or store the list."""
|
|
263
|
+
elements = _elements(await _read_value(config, ctx), self.spec.id, MAP_PROMISE)
|
|
264
|
+
try:
|
|
265
|
+
compiled = self.compile(config.program)
|
|
266
|
+
except TransformError as error:
|
|
267
|
+
raise BlockFailure(str(error), error_class=ErrorClass.REJECTED) from error
|
|
268
|
+
mapped = self._map_each(compiled, elements)
|
|
269
|
+
assert len(mapped) == len(elements), (
|
|
270
|
+
f"{self.spec.id} produced {len(mapped)} elements from {len(elements)}, breaking the map promise"
|
|
271
|
+
)
|
|
272
|
+
if config.save_to is None:
|
|
273
|
+
return TransformOutput(value=mapped)
|
|
274
|
+
written = await _write(ctx, config.save_to, json.dumps(mapped, separators=(",", ":")).encode())
|
|
275
|
+
return TransformOutput(output_uri=config.save_to, output_bytes=written)
|
|
276
|
+
|
|
277
|
+
def _map_each(self, compiled: object, elements: list[JsonValue]) -> list[JsonValue]:
|
|
278
|
+
"""Apply the engine once per element, in order, naming the element it refused."""
|
|
279
|
+
mapped: list[JsonValue] = []
|
|
280
|
+
for index, element in enumerate(elements):
|
|
281
|
+
try:
|
|
282
|
+
mapped.append(self.apply(compiled, element))
|
|
283
|
+
except TransformError as error:
|
|
284
|
+
raise BlockFailure(f"element {index}: {error}", error_class=ErrorClass.REJECTED) from error
|
|
285
|
+
return mapped
|
|
286
|
+
|
|
287
|
+
def check_config(self, config: BaseModel) -> list[str]:
|
|
288
|
+
"""Compile the program at apply, so a bad one is refused before the document is stored."""
|
|
289
|
+
if not isinstance(config, ProgramConfig):
|
|
290
|
+
return []
|
|
291
|
+
try:
|
|
292
|
+
self.compile(config.program)
|
|
293
|
+
except TransformError as error:
|
|
294
|
+
return [str(error)]
|
|
295
|
+
return []
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
class Filterer(Operator[ProgramConfig, TransformOutput], ABC):
|
|
299
|
+
"""The ``filter`` verb: keep the elements of a list a program answers true for.
|
|
300
|
+
|
|
301
|
+
An engine names its kind, summarises itself in one line, supplies compiling a program,
|
|
302
|
+
and answers one question about one element: keep it, or not. The frame derives the
|
|
303
|
+
catalog entry, resolves the input, refuses an input that is not an array, runs the loop,
|
|
304
|
+
and disposes of the result.
|
|
305
|
+
|
|
306
|
+
The promise is a subset with the elements unmodified. The frame keeps the element it was
|
|
307
|
+
given rather than anything the engine produced, so an engine has no way to change an
|
|
308
|
+
element it was only asked about, and an answer that is not a boolean is a refusal rather
|
|
309
|
+
than a truthiness question the frame would have to decide.
|
|
310
|
+
|
|
311
|
+
An engine touches no HTTP, no file outside storage, and nothing in the environment. It
|
|
312
|
+
is handed an element and returns a verdict; everything that reaches the world is the
|
|
313
|
+
frame's.
|
|
314
|
+
"""
|
|
315
|
+
|
|
316
|
+
kind: ClassVar[str]
|
|
317
|
+
"""The engine's name, which is the second half of the block id."""
|
|
318
|
+
|
|
319
|
+
summary: ClassVar[str]
|
|
320
|
+
"""The one line the catalog shows for this engine."""
|
|
321
|
+
|
|
322
|
+
local_execution: ClassVar[bool] = False
|
|
323
|
+
"""Whether this engine executes code on the worker, which puts it behind the allowlist."""
|
|
324
|
+
|
|
325
|
+
config_model: ClassVar[type[BaseModel]] = ProgramConfig
|
|
326
|
+
output_model: ClassVar[type[BaseModel]] = TransformOutput
|
|
327
|
+
|
|
328
|
+
def __init_subclass__(cls, **kwargs: Any) -> None:
|
|
329
|
+
"""Derive the catalog entry: the id is ``filter.<kind>`` and the group ``transform``."""
|
|
330
|
+
super().__init_subclass__(**kwargs)
|
|
331
|
+
kind = cls.__dict__.get("kind")
|
|
332
|
+
if kind is not None:
|
|
333
|
+
cls.spec = OperatorSpec(
|
|
334
|
+
id=f"filter.{kind}",
|
|
335
|
+
group="transform",
|
|
336
|
+
summary=cls.summary,
|
|
337
|
+
idempotent=True,
|
|
338
|
+
local_execution=cls.local_execution,
|
|
339
|
+
)
|
|
340
|
+
|
|
341
|
+
@abstractmethod
|
|
342
|
+
def compile(self, program: str) -> object:
|
|
343
|
+
"""Turn a program into whatever this engine applies, raising TransformError on a bad one."""
|
|
344
|
+
...
|
|
345
|
+
|
|
346
|
+
@abstractmethod
|
|
347
|
+
def keep(self, compiled: object, value: JsonValue) -> bool:
|
|
348
|
+
"""Answer whether one element is kept."""
|
|
349
|
+
...
|
|
350
|
+
|
|
351
|
+
async def execute(self, config: ProgramConfig, ctx: StepContext) -> TransformOutput | RemoteHandle:
|
|
352
|
+
"""Resolve the input, keep the elements the engine answers true for, and inline or store them."""
|
|
353
|
+
elements = _elements(await _read_value(config, ctx), self.spec.id, FILTER_PROMISE)
|
|
354
|
+
try:
|
|
355
|
+
compiled = self.compile(config.program)
|
|
356
|
+
except TransformError as error:
|
|
357
|
+
raise BlockFailure(str(error), error_class=ErrorClass.REJECTED) from error
|
|
358
|
+
# The element the frame was given, never anything the engine returned: what a filter
|
|
359
|
+
# keeps is what arrived.
|
|
360
|
+
kept = [element for index, element in enumerate(elements) if self._verdict(compiled, element, index)]
|
|
361
|
+
if config.save_to is None:
|
|
362
|
+
return TransformOutput(value=kept)
|
|
363
|
+
written = await _write(ctx, config.save_to, json.dumps(kept, separators=(",", ":")).encode())
|
|
364
|
+
return TransformOutput(output_uri=config.save_to, output_bytes=written)
|
|
365
|
+
|
|
366
|
+
def _verdict(self, compiled: object, element: JsonValue, index: int) -> bool:
|
|
367
|
+
"""Ask the engine about one element, refusing an answer that is not a boolean."""
|
|
368
|
+
try:
|
|
369
|
+
# Read as object rather than bool: the signature says bool, and this is where an
|
|
370
|
+
# engine is held to it.
|
|
371
|
+
answer = cast("object", self.keep(compiled, element))
|
|
372
|
+
except TransformError as error:
|
|
373
|
+
raise BlockFailure(f"element {index}: {error}", error_class=ErrorClass.REJECTED) from error
|
|
374
|
+
if not isinstance(answer, bool):
|
|
375
|
+
raise BlockFailure(
|
|
376
|
+
f"{self.spec.id} answered {answer!r} for element {index}, and a filter's answer is true or false",
|
|
377
|
+
error_class=ErrorClass.REJECTED,
|
|
378
|
+
)
|
|
379
|
+
return answer
|
|
380
|
+
|
|
381
|
+
def check_config(self, config: BaseModel) -> list[str]:
|
|
382
|
+
"""Compile the program at apply, so a bad one is refused before the document is stored."""
|
|
383
|
+
if not isinstance(config, ProgramConfig):
|
|
384
|
+
return []
|
|
385
|
+
try:
|
|
386
|
+
self.compile(config.program)
|
|
387
|
+
except TransformError as error:
|
|
388
|
+
return [str(error)]
|
|
389
|
+
return []
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
class Converter(Operator[ConvertConfig, ConvertOutput], ABC):
|
|
393
|
+
"""The ``convert`` verb: re-encode bytes from one format into another, content preserved.
|
|
394
|
+
|
|
395
|
+
A converter is a codec, not a language: there is no program. An engine names its kind,
|
|
396
|
+
declares the ``(from, to)`` format pairs it supports, and re-encodes bytes. The frame
|
|
397
|
+
derives the catalog entry, resolves the input, refuses an unsupported pair at apply, and
|
|
398
|
+
disposes of the result.
|
|
399
|
+
|
|
400
|
+
An engine touches no HTTP, no file outside storage, and nothing in the environment. It
|
|
401
|
+
is handed bytes and returns bytes; everything that reaches the world is the frame's.
|
|
402
|
+
"""
|
|
403
|
+
|
|
404
|
+
kind: ClassVar[str]
|
|
405
|
+
"""The engine's name, which is the second half of the block id."""
|
|
406
|
+
|
|
407
|
+
summary: ClassVar[str]
|
|
408
|
+
"""The one line the catalog shows for this engine."""
|
|
409
|
+
|
|
410
|
+
pairs: ClassVar[frozenset[tuple[str, str]]]
|
|
411
|
+
"""Every ``(from, to)`` format pair this engine re-encodes between."""
|
|
412
|
+
|
|
413
|
+
local_execution: ClassVar[bool] = False
|
|
414
|
+
"""Whether this engine executes code on the worker, which puts it behind the allowlist."""
|
|
415
|
+
|
|
416
|
+
config_model: ClassVar[type[BaseModel]] = ConvertConfig
|
|
417
|
+
output_model: ClassVar[type[BaseModel]] = ConvertOutput
|
|
418
|
+
|
|
419
|
+
def __init_subclass__(cls, **kwargs: Any) -> None:
|
|
420
|
+
"""Derive the catalog entry: the id is ``convert.<kind>`` and the group ``transform``."""
|
|
421
|
+
super().__init_subclass__(**kwargs)
|
|
422
|
+
kind = cls.__dict__.get("kind")
|
|
423
|
+
if kind is not None:
|
|
424
|
+
cls.spec = OperatorSpec(
|
|
425
|
+
id=f"convert.{kind}",
|
|
426
|
+
group="transform",
|
|
427
|
+
summary=cls.summary,
|
|
428
|
+
idempotent=True,
|
|
429
|
+
local_execution=cls.local_execution,
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
@abstractmethod
|
|
433
|
+
def convert(self, source: bytes, *, source_format: str, target_format: str) -> bytes:
|
|
434
|
+
"""Re-encode one whole payload from one format into another."""
|
|
435
|
+
...
|
|
436
|
+
|
|
437
|
+
async def execute(self, config: ConvertConfig, ctx: StepContext) -> ConvertOutput | RemoteHandle:
|
|
438
|
+
"""Refuse an unsupported pair, then re-encode the input and inline or store the result."""
|
|
439
|
+
unsupported = self._pair_refusal(config.from_format, config.to_format)
|
|
440
|
+
if unsupported is not None:
|
|
441
|
+
raise BlockFailure(unsupported, error_class=ErrorClass.REJECTED)
|
|
442
|
+
source = await _read_bytes(config, ctx)
|
|
443
|
+
try:
|
|
444
|
+
produced = self.convert(source, source_format=config.from_format, target_format=config.to_format)
|
|
445
|
+
except TransformError as error:
|
|
446
|
+
raise BlockFailure(str(error), error_class=ErrorClass.REJECTED) from error
|
|
447
|
+
if config.save_to is None:
|
|
448
|
+
return ConvertOutput(text=produced.decode("utf-8", errors="replace"))
|
|
449
|
+
written = await _write(ctx, config.save_to, produced)
|
|
450
|
+
return ConvertOutput(output_uri=config.save_to, output_bytes=written)
|
|
451
|
+
|
|
452
|
+
def check_config(self, config: BaseModel) -> list[str]:
|
|
453
|
+
"""Refuse a format pair this engine has no codec for, at apply."""
|
|
454
|
+
if not isinstance(config, ConvertConfig):
|
|
455
|
+
return []
|
|
456
|
+
unsupported = self._pair_refusal(config.from_format, config.to_format)
|
|
457
|
+
return [] if unsupported is None else [unsupported]
|
|
458
|
+
|
|
459
|
+
def _pair_refusal(self, source_format: str, target_format: str) -> str | None:
|
|
460
|
+
"""Word the refusal of a pair this engine does not support, naming the ones it does."""
|
|
461
|
+
if (source_format, target_format) in self.pairs:
|
|
462
|
+
return None
|
|
463
|
+
listed = ", ".join(f"{one} to {other}" for one, other in sorted(self.pairs))
|
|
464
|
+
supported = listed or "this engine converts nothing"
|
|
465
|
+
return f"{self.spec.id} does not convert {source_format} to {target_format} ({supported})"
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _elements(value: JsonValue, spec_id: str, promise: str) -> list[JsonValue]:
|
|
469
|
+
"""Read a verb's input as a JSON array, refusing anything else with the promise it broke."""
|
|
470
|
+
if isinstance(value, list):
|
|
471
|
+
return value
|
|
472
|
+
raise BlockFailure(
|
|
473
|
+
f"{spec_id} {promise}, so its input has to be a JSON array, and this one is {_named(value)}",
|
|
474
|
+
error_class=ErrorClass.REJECTED,
|
|
475
|
+
)
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def _named(value: JsonValue) -> str:
|
|
479
|
+
"""Say what a JSON value is, so a refusal names what arrived; only ever called with a non-array."""
|
|
480
|
+
if isinstance(value, dict):
|
|
481
|
+
return "an object"
|
|
482
|
+
if isinstance(value, str):
|
|
483
|
+
return "a string"
|
|
484
|
+
# Before the number check, because a bool is an int in Python and is not one in JSON.
|
|
485
|
+
if isinstance(value, bool):
|
|
486
|
+
return "a boolean"
|
|
487
|
+
if value is None:
|
|
488
|
+
return "null"
|
|
489
|
+
return "a number"
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
async def _read_value(config: ProgramConfig, ctx: StepContext) -> JsonValue:
|
|
493
|
+
"""Resolve a program engine's input: the inline value, or the JSON stored at a URI."""
|
|
494
|
+
if config.input_uri is None:
|
|
495
|
+
return config.input
|
|
496
|
+
payload = await _read_bounded(ctx, config.input_uri, config.max_input)
|
|
497
|
+
try:
|
|
498
|
+
parsed: JsonValue = json.loads(payload)
|
|
499
|
+
except ValueError as error:
|
|
500
|
+
raise BlockFailure(
|
|
501
|
+
f"{config.input_uri} does not hold JSON: {error}", error_class=ErrorClass.REJECTED
|
|
502
|
+
) from error
|
|
503
|
+
return parsed
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
async def _read_bytes(config: ConvertConfig, ctx: StepContext) -> bytes:
|
|
507
|
+
"""Resolve a codec engine's input: the inline text, or the bytes stored at a URI."""
|
|
508
|
+
if config.input_uri is None:
|
|
509
|
+
return (config.input or "").encode()
|
|
510
|
+
return await _read_bounded(ctx, config.input_uri, config.max_input)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
async def _read_bounded(ctx: StepContext, uri: str, limit: int) -> bytes:
|
|
514
|
+
"""Read a stored object whole, refusing one larger than the step said it would hold.
|
|
515
|
+
|
|
516
|
+
Counted as it arrives rather than trusted from a stat, because a recorded size is the
|
|
517
|
+
backend's claim and this is the worker's memory.
|
|
518
|
+
"""
|
|
519
|
+
chunks: list[bytes] = []
|
|
520
|
+
total = 0
|
|
521
|
+
async for chunk in ctx.storage.open_read(uri):
|
|
522
|
+
total += len(chunk)
|
|
523
|
+
if total > limit:
|
|
524
|
+
raise BlockFailure(
|
|
525
|
+
f"{uri} is larger than max_input ({limit} bytes) and is not being read; "
|
|
526
|
+
f"raise max_input, or transform it in pieces",
|
|
527
|
+
error_class=ErrorClass.REJECTED,
|
|
528
|
+
)
|
|
529
|
+
chunks.append(chunk)
|
|
530
|
+
return b"".join(chunks)
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
async def _write(ctx: StepContext, uri: str, payload: bytes) -> int:
|
|
534
|
+
"""Stream a result to storage a chunk at a time, and say how much reached it."""
|
|
535
|
+
written = 0
|
|
536
|
+
async with ctx.storage.open_write(uri) as sink:
|
|
537
|
+
for start in range(0, len(payload), CHUNK_BYTES):
|
|
538
|
+
written += await sink.write(payload[start : start + CHUNK_BYTES])
|
|
539
|
+
ctx.log.info("transform result saved", uri=uri, bytes=written)
|
|
540
|
+
return written
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dirigent-plugin
|
|
3
|
+
Version: 0.9.0
|
|
4
|
+
Summary: The dirigent plugin contract: block specs, protocols, and markers.
|
|
5
|
+
License-Expression: LicenseRef-Proprietary
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Dist: dirigent-common
|
|
8
|
+
Requires-Dist: httpx2>=2.12.0
|
|
9
|
+
Requires-Dist: pluginkit>=0.5.0
|
|
10
|
+
Requires-Dist: pydantic>=2.13.5
|
|
11
|
+
Requires-Python: >=3.13
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# dirigent-plugin
|
|
15
|
+
|
|
16
|
+
The dirigent plugin contract: block specs, protocols, and markers.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
dirigent_plugin/__init__.py,sha256=s_x2QyWvqN3OxvktKvUu7f6SkNdENJQOJl0CKfKB-wY,2133
|
|
2
|
+
dirigent_plugin/blocks.py,sha256=rH5CsxiKtTJIb33VrpgbUPl2HscQKdXRHI0rEpXwpAo,27527
|
|
3
|
+
dirigent_plugin/markers.py,sha256=E2r8eNBvC8FCtpP8yukIa-u8HKwlGYOT5ZBTG8OXrho,1030
|
|
4
|
+
dirigent_plugin/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
dirigent_plugin/transforms.py,sha256=hJQi2oMvaZrLQPZxWz6W8itlcDEwWSjbM9xtkOBclmg,23662
|
|
6
|
+
dirigent_plugin-0.9.0.dist-info/licenses/LICENSE,sha256=LKBm7Cx-WBc1zca4DjGxq99VEpAiWGnZDxIKmntn1hQ,910
|
|
7
|
+
dirigent_plugin-0.9.0.dist-info/WHEEL,sha256=mru_b36sH6joUMnwf7IlFCun3RoDjrNg9RcfBmEcqsE,81
|
|
8
|
+
dirigent_plugin-0.9.0.dist-info/METADATA,sha256=1miNeyyWR5FBSTxoQO--EzXkmppT1BfSpnyQ_Fgr01g,476
|
|
9
|
+
dirigent_plugin-0.9.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Copyright (c) 2026 Morten Olav Hansen <morten@winterop.com>. All rights reserved.
|
|
2
|
+
|
|
3
|
+
This source code and accompanying documentation are the property of
|
|
4
|
+
Morten Olav Hansen. No license, express or implied, is granted to use, copy,
|
|
5
|
+
modify, merge, publish, distribute, sublicense, or sell copies of this
|
|
6
|
+
software or its derivatives.
|
|
7
|
+
|
|
8
|
+
The source is published for reference only. Any use beyond reading
|
|
9
|
+
requires written permission from the copyright holder.
|
|
10
|
+
|
|
11
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
12
|
+
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
13
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT.
|
|
14
|
+
IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES,
|
|
15
|
+
OR OTHER LIABILITY ARISING FROM THE USE OF THE SOFTWARE.
|
|
16
|
+
|
|
17
|
+
Third-party components redistributed with this software, and the licences they
|
|
18
|
+
carry, are listed in THIRD_PARTY_NOTICES.md.
|