dirigent-blocks 0.16.2__tar.gz → 0.16.4__tar.gz

{dirigent_blocks-0.16.2 → dirigent_blocks-0.16.4}/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.4
 Name: dirigent-blocks
-Version: 0.16.2
+Version: 0.16.4
 Summary: Built-in generic operators and sensors for dirigent.
 License-Expression: LicenseRef-Proprietary
 License-File: LICENSE
 Requires-Dist: aio-pika>=9.5.7
 Requires-Dist: aiokafka>=0.12.0
 Requires-Dist: aiosmtplib>=5.1.2
-Requires-Dist: dirigent-common==0.16.2
-Requires-Dist: dirigent-plugin==0.16.2
+Requires-Dist: dirigent-common==0.16.4
+Requires-Dist: dirigent-plugin==0.16.4
 Requires-Dist: httpx2>=2.12.0
 Requires-Dist: jq>=1.8
 Requires-Dist: jsonschema>=4.26.0

{dirigent_blocks-0.16.2 → dirigent_blocks-0.16.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-blocks"
-version = "0.16.2"
+version = "0.16.4"
 description = "Built-in generic operators and sensors for dirigent."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -10,8 +10,8 @@ dependencies = [
     "aio-pika>=9.5.7",
     "aiokafka>=0.12.0",
     "aiosmtplib>=5.1.2",
-    "dirigent-common==0.16.2",
-    "dirigent-plugin==0.16.2",
+    "dirigent-common==0.16.4",
+    "dirigent-plugin==0.16.4",
     "httpx2>=2.12.0",
     "jq>=1.8",
     "jsonschema>=4.26.0",

{dirigent_blocks-0.16.2 → dirigent_blocks-0.16.4}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-blocks"
-version = "0.16.2"
+version = "0.16.4"
 description = "Built-in generic operators and sensors for dirigent."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -10,8 +10,8 @@ dependencies = [
     "aio-pika>=9.5.7",
     "aiokafka>=0.12.0",
     "aiosmtplib>=5.1.2",
-    "dirigent-common==0.16.2",
-    "dirigent-plugin==0.16.2",
+    "dirigent-common==0.16.4",
+    "dirigent-plugin==0.16.4",
     "httpx2>=2.12.0",
     "jq>=1.8",
     "jsonschema>=4.26.0",

{dirigent_blocks-0.16.2 → dirigent_blocks-0.16.4}/src/dirigent_blocks/docker.py

@@ -44,6 +44,7 @@ from dirigent_plugin import (
     ProbeStatus,
     RemoteHandle,
     ShellString,
+    ShellVariables,
     StepContext,
     classify_default,
 )
@@ -369,7 +370,7 @@ def private_parent(ctx: StepContext) -> Path:
     return subprocess.local_root(ctx)
 
 
-class DockerRunConfig(BlockModel):
+class DockerRunConfig(ShellVariables):
     """Which image to run, with what command, environment, mounts, and limits."""
 
     image: str = Field(min_length=1)
@@ -381,9 +382,11 @@ class DockerRunConfig(BlockModel):
     command: Annotated[str | None, ShellString()] = None
     """The command as a shell string, run through ``/bin/sh -c`` inside the container.
 
-    Every ``${...}`` substituted into it is shell-quoted by the engine, so a value that came
-    from a webhook payload is one word rather than one program. Prefer ``argv``: it involves
-    no shell at all."""
+    Every ``${...}`` in it is rewritten by the engine to a variable it sets in the container's
+    environment, so a value that came from a webhook payload is one word the shell never
+    parses, however the reference was quoted -- and inside single quotes, which a shell keeps
+    literal, the command reads that variable's name rather than its value. Prefer ``argv``: it
+    involves no shell at all."""
 
     workdir: str | None = None
     """The working directory inside the container, overriding the image's own."""
@@ -728,9 +731,14 @@ def _command(config: DockerRunConfig) -> list[JsonValue] | None:
 
 
 def _environment(config: DockerRunConfig) -> list[JsonValue]:
-    """Build the container's environment from an allowlist, never from wholesale inheritance."""
+    """Build the container's environment from an allowlist, never from wholesale inheritance.
+
+    The values the engine substituted out of the shell string come last, so a document cannot
+    override what a reference resolved to.
+    """
     inherited = allowed(config.env_allowlist)
-    return [f"{name}={value}" for name, value in {**inherited, **config.env}.items()]
+    built = {**inherited, **config.env, **config.shell_variables}
+    return [f"{name}={value}" for name, value in built.items()]
 
 
 def _nano_cpus(config: DockerRunConfig) -> int | None:

{dirigent_blocks-0.16.2 → dirigent_blocks-0.16.4}/src/dirigent_blocks/git.py

@@ -250,7 +250,7 @@ class GitCheckoutOperator(Operator[GitCheckoutConfig, GitCheckoutOutput]):
         root = subprocess.local_root(ctx)
         settings = ctx.connection(config.connection, GitConnectionConfig)
         target = config.target or ctx.step
-        destination = root / target
+        destination = _destination(root, target)
         remote = public_url(settings.url)
         binary = shutil.which("git", path=subprocess.environment([], {}, root).get("PATH"))
         if binary is None:
@@ -409,8 +409,11 @@ async def _standing(git: _Git, config: GitCheckoutConfig, destination: Path, rem
     A retry of a step, or a second run into a work directory that already holds the checkout,
     should cost one ref listing rather than a second clone. Anything that is not a working
     tree at the wanted commit answers false, and the target is then rebuilt from scratch.
+
+    A target that is itself a symlink is never standing: what it points at is some other
+    directory, so it is unlinked and replaced rather than looked into.
     """
-    if not (destination / ".git").exists():
+    if destination.is_symlink() or not (destination / ".git").exists():
         return False
     code, out, _err = await git.call(destination, "rev-parse", "HEAD")
     if code != 0:
@@ -484,6 +487,37 @@ def classify(stderr: bytes) -> ErrorClass:
     return ErrorClass.UNKNOWN
 
 
+def _destination(root: Path, target: str) -> Path:
+    """Where the checkout lands, once it is certain that path is inside the run's work directory.
+
+    The config refuses a target that is absolute or carries ``..``, which is a check on the
+    text; a symlink an earlier step left in the work directory is not. So every component
+    between the root and the target must be a real directory here, and the directory the
+    checkout lands in must resolve inside the root, before anything looks at the target,
+    clears it or clones into it. The target itself may be a symlink: it is unlinked rather
+    than followed.
+    """
+    base = root.resolve()
+    parts = Path(target).parts
+    walked = base
+    for part in parts[:-1]:
+        walked = walked / part
+        if walked.is_symlink():
+            raise BlockFailure(
+                f"the checkout target {target!r} leads through the symlink {walked}, which can point "
+                f"anywhere; a target is a path of real directories under the run's work directory {base}",
+                error_class=ErrorClass.REJECTED,
+            )
+    destination = walked / parts[-1] if parts else walked
+    landing = destination.parent.resolve()
+    if landing != base and base not in landing.parents:
+        raise BlockFailure(
+            f"the checkout target {target!r} lands at {destination}, which is outside the run's work directory {base}",
+            error_class=ErrorClass.REJECTED,
+        )
+    return destination
+
+
 def _clear(destination: Path) -> None:
     """Empty the target so a clone into it starts from nothing, whatever was there before."""
     if destination.is_symlink() or destination.is_file():

dirigent_blocks-0.16.4/src/dirigent_blocks/jq_runner.py

@@ -0,0 +1,54 @@
+"""The process a jq program is evaluated in, started by ``transform_jq`` and spoken to over its pipes.
+
+jq is a C extension that holds the GIL for as long as a program runs, so a program evaluated
+in the worker's own process holds its event loop too: the step's timeout never fires, the
+lease heartbeat never runs, and the sweeper eventually hands the attempt to another worker
+while this one is still computing. There is no way to interrupt a jq program and no way to
+cancel a thread, so a program runs here, where killing the process ends it.
+
+One request per line in, one reply per line back: ``{"program": ..., "value": ...}`` is
+answered with ``{"outputs": [...]}``, or with ``{"error": "..."}`` carrying jq's own message
+about a program that could not be compiled or met data it cannot work on. The pipe closing is
+how the parent says it is done.
+
+This module is run by path, never imported, so it costs the standard library and jq and
+nothing else to start. Importing it as part of its package would load every block dirigent
+ships in order to evaluate ``.name``.
+"""
+
+import json
+import sys
+from typing import Any
+
+import jq
+
+#: Typed ``Any`` because the jq binding is a C extension that ships no type information.
+libjq: Any = jq
+
+
+def main() -> None:
+    """Answer one request at a time until the pipe closes.
+
+    The last program compiled is kept because a step sends the same one once per element, and
+    only the last, because the process outlives the step and a run's programs would otherwise
+    pile up in it.
+    """
+    out = sys.stdout.buffer
+    source: str | None = None
+    compiled: Any = None
+    for line in sys.stdin.buffer:
+        request = json.loads(line)
+        try:
+            if request["program"] != source:
+                # Named only once it has compiled, so a refused program is not cached as one.
+                compiled = libjq.compile(request["program"])
+                source = request["program"]
+            reply = {"outputs": compiled.input_value(request["value"]).all()}
+        except ValueError as error:
+            reply = {"error": str(error).strip()}
+        out.write(json.dumps(reply).encode() + b"\n")
+        out.flush()
+
+
+if __name__ == "__main__":
+    main()

{dirigent_blocks-0.16.2 → dirigent_blocks-0.16.4}/src/dirigent_blocks/rabbitmq.py

@@ -6,11 +6,11 @@ there is enough, handing the batch downstream as its output.
 
 Unlike a Kafka topic, a RabbitMQ queue keeps its own place: a message is held unacknowledged
 until it is acked or nacked, so the broker rather than the cursor is the bookkeeping. What the
-sensor decides is *when* to acknowledge, and the default is ``on_success``: a poke that parks
-nacks everything it took back onto the queue with ``requeue``, so a batch too small to act on
-is left for the next poke or another consumer rather than swallowed. The cursor here carries
-nothing the sensor needs, only what a reader wants -- how many messages have gone past and the
-last delivery tag seen.
+sensor decides is *when* to acknowledge, and the default is ``on_success``: a poke that parks,
+or that cannot read one of the bodies it took, nacks everything back onto the queue with
+``requeue``, so a batch too small to act on is left for the next poke or another consumer
+rather than swallowed. The cursor here carries nothing the sensor needs, only what a reader
+wants -- how many messages have gone past and the last delivery tag seen.
 
 ``rabbitmq.publish`` goes the other way, and is the sink an upstream step hands a value to: it
 puts one message on an exchange, or on the default exchange where a routing key is a queue name.
@@ -213,11 +213,13 @@ class RabbitConsumeConfig(BlockModel):
     ack: Literal["on_success", "always"] = "on_success"
     """When a message is acknowledged.
 
-    ``on_success`` acknowledges only in the poke that succeeds, and a poke that parks nacks what
-    it took back onto the queue with ``requeue``, so nothing is lost to a batch that was too
-    small and another consumer may take it instead. ``always`` acknowledges every message the
-    moment it is taken, which suits a queue nothing else reads and a step that would rather
-    drop a partial batch than see it twice."""
+    ``on_success`` acknowledges only in the poke that succeeds, once every body in the batch has
+    been read. A poke that parks, or that finds a body it cannot read, nacks what it took back
+    onto the queue with ``requeue``, so nothing is lost to a batch that was too small or to one
+    unreadable message beside sound ones, and another consumer may take it instead. ``always``
+    acknowledges every message the moment it is taken, before any of them is read, which suits a
+    queue nothing else reads and a step that would rather drop a partial or unreadable batch
+    than see it twice."""
 
     value_format: Payload = "json"
     """How a message body is decoded."""
@@ -276,6 +278,8 @@ class RabbitConsumeSensor(Sensor[RabbitConsumeConfig, RabbitConsumeOutput]):
             queue = await _queue(connection, config)
             taken = await _drain(queue, config)
             if config.ack == "always":
+                # The batch is settled before it is read, so a body that will not decode is gone
+                # from the queue with the rest of them.
                 for delivery in taken:
                     await delivery.ack()
             if len(taken) < config.min_messages:
@@ -293,10 +297,10 @@ class RabbitConsumeSensor(Sensor[RabbitConsumeConfig, RabbitConsumeOutput]):
                     next_poll_in=config.poll_every,
                     message=f"{len(taken)} of {config.min_messages} messages",
                 )
+            messages = await _decoded(taken, config)
             if config.ack == "on_success":
                 for delivery in taken:
                     await delivery.ack()
-            messages = [_read(delivery, config) for delivery in taken]
             ctx.log.info("consumed", queue=config.queue, count=len(messages), ack=config.ack)
             return RabbitConsumeOutput(messages=messages, count=len(messages))
         except BlockFailure:
@@ -341,6 +345,28 @@ async def _drain(queue: Queue, config: RabbitConsumeConfig) -> list[Delivery]:
     return taken
 
 
+async def _decoded(taken: list[Delivery], config: RabbitConsumeConfig) -> list[RabbitMessage]:
+    """Read the whole batch, or hand the whole batch back and say which delivery would not read.
+
+    Under ``on_success`` nothing has been acked yet, so a body that is not what the step was
+    promised is requeued along with the sound messages beside it rather than taking them off
+    the queue with it.
+    """
+    messages: list[RabbitMessage] = []
+    for delivery in taken:
+        try:
+            messages.append(_read(delivery, config))
+        except Exception as error:
+            if config.ack == "on_success":
+                for held in taken:
+                    await held.nack(requeue=True)
+            raise BlockFailure(
+                f"delivery {delivery.delivery_tag} of the {len(taken)} taken did not read: {error}",
+                error_class=classify(error),
+            ) from error
+    return messages
+
+
 def _cursor(held: dict[str, JsonValue] | None, taken: list[Delivery]) -> dict[str, JsonValue]:
     """What a reader is told about a wait the broker is doing the bookkeeping for."""
     seen = int(cast("int", (held or {}).get("seen", 0))) + len(taken)

{dirigent_blocks-0.16.2 → dirigent_blocks-0.16.4}/src/dirigent_blocks/shell.py

@@ -15,10 +15,19 @@ from dirigent_blocks import subprocess
 from dirigent_blocks.capture import log_stream, tail
 from dirigent_blocks.environment import reject_reserved
 from dirigent_common import BlockModel, Duration
-from dirigent_plugin import BlockFailure, ErrorClass, Operator, OperatorSpec, RemoteHandle, ShellString, StepContext
-
-
-class ShellRunConfig(BlockModel):
+from dirigent_plugin import (
+    BlockFailure,
+    ErrorClass,
+    Operator,
+    OperatorSpec,
+    RemoteHandle,
+    ShellString,
+    ShellVariables,
+    StepContext,
+)
+
+
+class ShellRunConfig(ShellVariables):
     """What to run, where, with what environment, and for how long."""
 
     argv: list[str] = Field(default_factory=list[str])
@@ -27,9 +36,11 @@ class ShellRunConfig(BlockModel):
     command: Annotated[str | None, ShellString()] = None
     """The command as a shell string, for when a pipe or a redirect is the point.
 
-    Every ``${...}`` substituted into it is shell-quoted by the engine, so a value that came
-    from a webhook payload is one word rather than one program. Prefer ``argv`` anyway: it
-    involves no shell at all."""
+    Every ``${...}`` in it is rewritten by the engine to a variable it sets in this command's
+    environment, so a value that came from a webhook payload is one word the shell never
+    parses, however the reference was quoted -- and inside single quotes, which a shell keeps
+    literal, the command reads that variable's name rather than its value. Prefer ``argv``
+    anyway: it involves no shell at all."""
 
     cwd: str | None = None
     """A directory relative to the run's work directory; never an absolute path."""
@@ -90,6 +101,18 @@ class ShellRunOutput(BlockModel):
     """Whether ``stderr`` above is short of the stream, which is an independent question."""
 
 
+def _environment(config: ShellRunConfig, home: Path) -> dict[str, str]:
+    """Build the command's environment, the substituted values last.
+
+    They are set after ``env`` so a document cannot override what a reference resolved to,
+    and they are set whatever the allowlist holds: the command reads them because the engine
+    put them in it.
+    """
+    environ = subprocess.environment(config.env_allowlist, config.env, home)
+    environ.update(config.shell_variables)
+    return environ
+
+
 class ShellRunOperator(Operator[ShellRunConfig, ShellRunOutput]):
     """Runs a command on the worker, inside the run's work directory and behind the allowlist."""
 
@@ -117,7 +140,7 @@ class ShellRunOperator(Operator[ShellRunConfig, ShellRunOutput]):
             stdout_uri=stdout_uri,
             stderr_uri=stderr_uri,
             timeout_seconds=config.timeout.total_seconds(),
-            environ=subprocess.environment(config.env_allowlist, config.env, directory),
+            environ=_environment(config, directory),
             argv=config.argv or None,
             command=config.command,
         )

dirigent_blocks-0.16.4/src/dirigent_blocks/transform_jq.py

@@ -0,0 +1,318 @@
+"""The ``jq`` engines: a jq program as the transform, map, and filter verbs' program.
+
+A program is compiled here, where a bad one is refused with jq's own message, and evaluated
+in a jq process of its own. jq is a C extension that holds the GIL for as long as a program
+runs, so evaluating one on the worker would hold the event loop with it and starve the step
+timeout and the lease heartbeat that share it. A program cannot be interrupted and a thread
+cannot be cancelled, so the step's timeout ends a program the only way there is: the process
+running it is killed, and the step fails with it.
+"""
+
+import asyncio
+import atexit
+import json
+import subprocess
+import sys
+from collections.abc import Callable
+from contextvars import ContextVar
+from pathlib import Path
+from typing import IO, Annotated, Any, ClassVar, cast
+
+import jq
+from pydantic import BaseModel, Field, JsonValue
+
+from dirigent_common import JQ_MEDIA_TYPE
+from dirigent_plugin import Engine, Filterer, Mapper, ProgramConfig, Transformer, TransformError
+
+#: What a program producing nothing is told to write instead.
+NO_OUTPUT = (
+    "the program produced no output, and a transform step has to produce one; "
+    "a program that means 'possibly nothing' emits [] or null explicitly"
+)
+
+#: Wraps the author's program so jq's ``env`` and ``$ENV`` read an empty object instead of
+#: the worker's environment, which is where dirigent's own secrets live. The program keeps
+#: its own line numbers because the message a bad one is refused with comes from compiling
+#: the author's text, unwrapped.
+SANDBOX = "{} as $ENV | def env: {}; (\n"
+
+#: The script a jq process runs, started by path so that it imports jq and the standard
+#: library and nothing else.
+RUNNER = Path(__file__).with_name("jq_runner.py")
+
+#: What a step is told when the process running its program stopped on its own. It is not a
+#: TransformError, because nothing about the program says it will happen again.
+PROCESS_STOPPED = "the process running the jq program stopped before it answered"
+
+#: Typed ``Any`` because the jq binding is a C extension that ships no type information.
+libjq: Any = jq
+
+
+def _compile(program: str) -> object:
+    """Compile a jq program with env and $ENV shadowed, refusing a bad one with jq's own message.
+
+    What is handed on is the sandboxed source rather than the compiled program, because the
+    process that evaluates it compiles its own copy: compiling here is what refuses a bad
+    program, at apply and again on the worker.
+    """
+    sandboxed = f"{SANDBOX}{program}\n)"
+    try:
+        # The author's text is compiled first, so a refusal carries jq's message about their
+        # program and not about the wrapper.
+        libjq.compile(program)
+        libjq.compile(sandboxed)
+    except ValueError as error:
+        raise TransformError(str(error).strip()) from error
+    return sandboxed
+
+
+def _outputs(compiled: object, value: JsonValue) -> list[JsonValue]:
+    """Run a compiled program over one value in this step's jq process and read its outputs."""
+    return _running().run(cast("str", compiled), value)
+
+
+class _Process:
+    """One jq process, and the two pipes a program and its outputs cross."""
+
+    def __init__(self) -> None:
+        """Start the process and hold the pipes it is spoken to over."""
+        self.process = subprocess.Popen(
+            [sys.executable, str(RUNNER)],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
+        # Popen types both pipes as optional; both were asked for.
+        self._to = cast("IO[bytes]", self.process.stdin)
+        self._from = cast("IO[bytes]", self.process.stdout)
+
+    @property
+    def alive(self) -> bool:
+        """Whether this process is still there to run a program."""
+        return self.process.poll() is None
+
+    def run(self, program: str, value: JsonValue) -> list[JsonValue]:
+        """Send one program and one value, and read back the stream of outputs it produced."""
+        try:
+            self._to.write(json.dumps({"program": program, "value": value}).encode() + b"\n")
+            self._to.flush()
+            reply = self._from.readline()
+        except OSError as error:
+            raise RuntimeError(PROCESS_STOPPED) from error
+        if not reply:
+            # A killed process, which is how a cancelled step ends its program, and how a
+            # program that ran the process out of memory ends itself.
+            raise RuntimeError(PROCESS_STOPPED)
+        answered = cast("dict[str, Any]", json.loads(reply))
+        refusal = answered.get("error")
+        if refusal is not None:
+            raise TransformError(cast("str", refusal))
+        return cast("list[JsonValue]", answered["outputs"])
+
+    def kill(self) -> None:
+        """End whatever is running, which is the only way to stop a jq program, and close it out.
+
+        Waiting is what makes the process dead rather than dying, so the pool sees at once
+        that this one is not to be handed out again; a killed process is reaped at once.
+        Closing the pipes waits for the read a step's thread is still blocked in, which the
+        kill has just ended, and leaves no descriptor behind for a collection to complain of.
+        """
+        self.process.kill()
+        self.process.wait()
+        self._to.close()
+        self._from.close()
+
+
+class _Processes:
+    """The jq processes this worker keeps, one per transform step running at a time.
+
+    A step holds one for as long as its program runs and gives it back after, so two steps
+    never meet on one pipe. A process is started when there is none to hand out and then
+    lives on, because starting one costs more than every program a step runs through it; it
+    ends when the worker does, its pipe closing under it. A process that was killed is not
+    handed out again.
+    """
+
+    def __init__(self) -> None:
+        """Start with no processes; the first step that needs one starts it."""
+        self._idle: list[_Process] = []
+
+    async def take(self) -> _Process:
+        """Hand out an idle process, starting one off the event loop when there is none."""
+        while self._idle:
+            process = self._idle.pop()
+            if process.alive:
+                return process
+        starting = asyncio.ensure_future(asyncio.to_thread(_Process))
+        try:
+            return await asyncio.shield(starting)
+        except asyncio.CancelledError:
+            # A step can be cancelled while its process is still starting. The start is
+            # shielded so the process is finished and kept, rather than left running for a
+            # step that is already gone.
+            starting.add_done_callback(self._started)
+            raise
+
+    def give_back(self, process: _Process) -> None:
+        """Take a process back for the next step, unless it was killed."""
+        if process.alive:
+            self._idle.append(process)
+
+    def shutdown(self) -> None:
+        """Kill every idle process, so none is left running when this one exits."""
+        while self._idle:
+            self._idle.pop().kill()
+
+    def _started(self, starting: "asyncio.Future[_Process]") -> None:
+        """Keep a process whose step gave up on it while it was starting."""
+        if not starting.cancelled() and starting.exception() is None:
+            self.give_back(starting.result())
+
+
+#: The jq processes this worker keeps. One pool per process rather than one per engine: the
+#: three verbs run the same programs the same way, and a step holds a process, not a verb.
+_PROCESSES = _Processes()
+
+atexit.register(_PROCESSES.shutdown)
+
+#: The jq process the step running on this task talks to. A program is evaluated from inside
+#: the thread offload() hands the step's work to, which is where the engine reads it.
+_RUNNING: ContextVar[_Process] = ContextVar("dirigent_jq_process")
+
+
+def _running() -> _Process:
+    """The jq process this step holds, which only a step running through offload() has."""
+    try:
+        return _RUNNING.get()
+    except LookupError as error:
+        missing = "a jq engine is called through offload(), which takes the process its program runs in"
+        raise RuntimeError(missing) from error
+
+
+class JqEngine(Engine):
+    """Where the three jq engines' programs run: in a jq process, never on the worker's loop."""
+
+    async def offload[T](self, work: Callable[[], T]) -> T:
+        """Hold a jq process for the step, and kill it if the step is cancelled or times out."""
+        process = await _PROCESSES.take()
+        token = _RUNNING.set(process)
+        try:
+            return await asyncio.to_thread(work)
+        except asyncio.CancelledError:
+            # Cancelling the await does not stop the thread: it is inside a blocking read,
+            # and killing the process is both what ends the program and what lets that read
+            # return, so nothing is left running behind a step that has already failed.
+            process.kill()
+            raise
+        finally:
+            _RUNNING.reset(token)
+            _PROCESSES.give_back(process)
+
+
+class JqProgramConfig(ProgramConfig):
+    """The program-shaped config of a jq engine, whichever verb is running it.
+
+    All three verbs take the same program in the same language, so all three publish the
+    same config: the frame's fields, with ``program`` narrowed to jq source.
+    """
+
+    program: Annotated[str, Field(min_length=1, json_schema_extra={"contentMediaType": JQ_MEDIA_TYPE})]
+    """The jq program this step runs."""
+
+
+class JqTransformer(JqEngine, Transformer):
+    """Reshapes one whole value by running a jq program over it.
+
+    A jq program opens no file, no socket, and starts nothing: it is handed a value and
+    returns values, which is why this engine executes no code on the worker and needs no
+    allowlist entry. Its two ways of reading an environment, ``env`` and ``$ENV``, are
+    shadowed, so a program sees an empty object rather than the process it runs in -- which
+    is a jq process of dirigent's own, so that a step's timeout can end it.
+
+    A jq program is a stream: it can emit zero, one, or many outputs. One output is the
+    step's value, many outputs are that list of values, and none is a refusal, because a
+    step that produced nothing has no output for the next step to read.
+    """
+
+    kind = "jq"
+    summary = "Reshape a value with a jq program."
+    config_model: ClassVar[type[BaseModel]] = JqProgramConfig
+
+    def compile(self, program: str) -> object:
+        """Compile a jq program, refusing a bad one with jq's own message about it."""
+        return _compile(program)
+
+    def apply(self, compiled: object, value: JsonValue) -> JsonValue:
+        """Run the program over one value, and read its stream of outputs as one result."""
+        produced = _outputs(compiled, value)
+        if not produced:
+            raise TransformError(NO_OUTPUT)
+        return produced[0] if len(produced) == 1 else produced
+
+
+class JqMapper(JqEngine, Mapper):
+    """Replaces every element of a list with what a jq program makes of that element.
+
+    The program is run once per element, with the element as its input, and is sandboxed
+    exactly as ``transform.jq`` is: ``env`` and ``$ENV`` read an empty object.
+
+    A map replaces each element with one thing, so the stream a program emits for an element
+    has to be exactly one output. Zero of them and several of them are both refusals rather
+    than a shorter or longer list, because dropping elements is what ``filter.jq`` is for and
+    changing their number is what ``transform.jq`` is for.
+    """
+
+    kind = "jq"
+    summary = "Replace every element of a list with what a jq program makes of it."
+    config_model: ClassVar[type[BaseModel]] = JqProgramConfig
+
+    def compile(self, program: str) -> object:
+        """Compile a jq program, refusing a bad one with jq's own message about it."""
+        return _compile(program)
+
+    def apply(self, compiled: object, value: JsonValue) -> JsonValue:
+        """Run the program over one element, and take its single output as the replacement."""
+        produced = _outputs(compiled, value)
+        if len(produced) != 1:
+            raise TransformError(
+                f"the program produced {len(produced)} outputs, and a map replaces an element with "
+                f"exactly one; a program that drops elements is a filter.jq step, and one that "
+                f"changes how many there are is a transform.jq step"
+            )
+        return produced[0]
+
+
+class JqFilterer(JqEngine, Filterer):
+    """Keeps the elements of a list a jq program answers true for.
+
+    The program is run once per element, with the element as its input, and is sandboxed
+    exactly as ``transform.jq`` is: ``env`` and ``$ENV`` read an empty object.
+
+    The answer has to be exactly one output, and that output has to be ``true`` or ``false``.
+    jq's own truthiness is not applied: a program emitting ``0`` or ``""`` is a mistake
+    surfaced rather than an element quietly dropped, and an author who means "has readings"
+    writes ``.count > 0``.
+    """
+
+    kind = "jq"
+    summary = "Keep the elements of a list a jq program answers true for."
+    config_model: ClassVar[type[BaseModel]] = JqProgramConfig
+
+    def compile(self, program: str) -> object:
+        """Compile a jq program, refusing a bad one with jq's own message about it."""
+        return _compile(program)
+
+    def keep(self, compiled: object, value: JsonValue) -> bool:
+        """Run the program over one element, and read its single boolean output as the verdict."""
+        produced = _outputs(compiled, value)
+        if len(produced) != 1:
+            raise TransformError(
+                f"the program produced {len(produced)} outputs, and a filter answers one true or false "
+                f"per element; a program that reshapes an element is a map.jq or transform.jq step"
+            )
+        answer = produced[0]
+        if not isinstance(answer, bool):
+            raise TransformError(
+                f"the program answered {json.dumps(answer)}, and a filter answers true or false; jq's "
+                f"truthiness is not applied, so a program meaning 'has readings' writes '.count > 0'"
+            )
+        return answer

dirigent_blocks-0.16.2/src/dirigent_blocks/transform_jq.py

@@ -1,153 +0,0 @@
-"""The ``jq`` engines: a jq program as the transform, map, and filter verbs' program."""
-
-import json
-from typing import Annotated, Any, ClassVar, cast
-
-import jq
-from pydantic import BaseModel, Field, JsonValue
-
-from dirigent_common import JQ_MEDIA_TYPE
-from dirigent_plugin import Filterer, Mapper, ProgramConfig, Transformer, TransformError
-
-#: What a program producing nothing is told to write instead.
-NO_OUTPUT = (
-    "the program produced no output, and a transform step has to produce one; "
-    "a program that means 'possibly nothing' emits [] or null explicitly"
-)
-
-#: Wraps the author's program so jq's ``env`` and ``$ENV`` read an empty object instead of
-#: the worker's environment, which is where dirigent's own secrets live. The program keeps
-#: its own line numbers because the message a bad one is refused with comes from compiling
-#: the author's text, unwrapped.
-SANDBOX = "{} as $ENV | def env: {}; (\n"
-
-#: Typed ``Any`` because the jq binding is a C extension that ships no type information.
-libjq: Any = jq
-
-
-def _compile(program: str) -> object:
-    """Compile a jq program with env and $ENV shadowed, refusing a bad one with jq's own message."""
-    try:
-        # The author's text is compiled first, so a refusal carries jq's message about their
-        # program and not about the wrapper.
-        libjq.compile(program)
-        return libjq.compile(f"{SANDBOX}{program}\n)")
-    except ValueError as error:
-        raise TransformError(str(error).strip()) from error
-
-
-def _outputs(compiled: object, value: JsonValue) -> list[JsonValue]:
-    """Run a compiled program over one value and read its whole stream of outputs."""
-    try:
-        return cast("list[JsonValue]", cast("Any", compiled).input_value(value).all())
-    except ValueError as error:
-        raise TransformError(str(error).strip()) from error
-
-
-class JqProgramConfig(ProgramConfig):
-    """The program-shaped config of a jq engine, whichever verb is running it.
-
-    All three verbs take the same program in the same language, so all three publish the
-    same config: the frame's fields, with ``program`` narrowed to jq source.
-    """
-
-    program: Annotated[str, Field(min_length=1, json_schema_extra={"contentMediaType": JQ_MEDIA_TYPE})]
-    """The jq program this step runs."""
-
-
-class JqTransformer(Transformer):
-    """Reshapes one whole value by running a jq program over it.
-
-    jq opens no file, no socket, and starts no process: a program is handed a value and
-    returns values, which is why this engine executes no code on the worker and needs no
-    allowlist entry. Its two ways of reading the worker's environment, ``env`` and ``$ENV``,
-    are shadowed, so a program sees an empty object rather than the process it runs in.
-
-    A jq program is a stream: it can emit zero, one, or many outputs. One output is the
-    step's value, many outputs are that list of values, and none is a refusal, because a
-    step that produced nothing has no output for the next step to read.
-    """
-
-    kind = "jq"
-    summary = "Reshape a value with a jq program."
-    config_model: ClassVar[type[BaseModel]] = JqProgramConfig
-
-    def compile(self, program: str) -> object:
-        """Compile a jq program, refusing a bad one with jq's own message about it."""
-        return _compile(program)
-
-    def apply(self, compiled: object, value: JsonValue) -> JsonValue:
-        """Run the program over one value, and read its stream of outputs as one result."""
-        produced = _outputs(compiled, value)
-        if not produced:
-            raise TransformError(NO_OUTPUT)
-        return produced[0] if len(produced) == 1 else produced
-
-
-class JqMapper(Mapper):
-    """Replaces every element of a list with what a jq program makes of that element.
-
-    The program is run once per element, with the element as its input, and is sandboxed
-    exactly as ``transform.jq`` is: ``env`` and ``$ENV`` read an empty object.
-
-    A map replaces each element with one thing, so the stream a program emits for an element
-    has to be exactly one output. Zero of them and several of them are both refusals rather
-    than a shorter or longer list, because dropping elements is what ``filter.jq`` is for and
-    changing their number is what ``transform.jq`` is for.
-    """
-
-    kind = "jq"
-    summary = "Replace every element of a list with what a jq program makes of it."
-    config_model: ClassVar[type[BaseModel]] = JqProgramConfig
-
-    def compile(self, program: str) -> object:
-        """Compile a jq program, refusing a bad one with jq's own message about it."""
-        return _compile(program)
-
-    def apply(self, compiled: object, value: JsonValue) -> JsonValue:
-        """Run the program over one element, and take its single output as the replacement."""
-        produced = _outputs(compiled, value)
-        if len(produced) != 1:
-            raise TransformError(
-                f"the program produced {len(produced)} outputs, and a map replaces an element with "
-                f"exactly one; a program that drops elements is a filter.jq step, and one that "
-                f"changes how many there are is a transform.jq step"
-            )
-        return produced[0]
-
-
-class JqFilterer(Filterer):
-    """Keeps the elements of a list a jq program answers true for.
-
-    The program is run once per element, with the element as its input, and is sandboxed
-    exactly as ``transform.jq`` is: ``env`` and ``$ENV`` read an empty object.
-
-    The answer has to be exactly one output, and that output has to be ``true`` or ``false``.
-    jq's own truthiness is not applied: a program emitting ``0`` or ``""`` is a mistake
-    surfaced rather than an element quietly dropped, and an author who means "has readings"
-    writes ``.count > 0``.
-    """
-
-    kind = "jq"
-    summary = "Keep the elements of a list a jq program answers true for."
-    config_model: ClassVar[type[BaseModel]] = JqProgramConfig
-
-    def compile(self, program: str) -> object:
-        """Compile a jq program, refusing a bad one with jq's own message about it."""
-        return _compile(program)
-
-    def keep(self, compiled: object, value: JsonValue) -> bool:
-        """Run the program over one element, and read its single boolean output as the verdict."""
-        produced = _outputs(compiled, value)
-        if len(produced) != 1:
-            raise TransformError(
-                f"the program produced {len(produced)} outputs, and a filter answers one true or false "
-                f"per element; a program that reshapes an element is a map.jq or transform.jq step"
-            )
-        answer = produced[0]
-        if not isinstance(answer, bool):
-            raise TransformError(
-                f"the program answered {json.dumps(answer)}, and a filter answers true or false; jq's "
-                f"truthiness is not applied, so a program meaning 'has readings' writes '.count > 0'"
-            )
-        return answer