dirigent-core 0.16.2__tar.gz → 0.16.3__tar.gz

{dirigent_core-0.16.2 → dirigent_core-0.16.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirigent-core
-Version: 0.16.2
+Version: 0.16.3
 Summary: Dirigent engine core: schema, configuration, and plugin host.
 License-Expression: LicenseRef-Proprietary
 License-File: LICENSE
@@ -10,9 +10,9 @@ Requires-Dist: argon2-cffi>=25.1.0
 Requires-Dist: asyncpg>=0.31.0
 Requires-Dist: cronsim>=2.7
 Requires-Dist: cryptography>=46.0.5
-Requires-Dist: dirigent-client==0.16.2
-Requires-Dist: dirigent-common==0.16.2
-Requires-Dist: dirigent-plugin==0.16.2
+Requires-Dist: dirigent-client==0.16.3
+Requires-Dist: dirigent-common==0.16.3
+Requires-Dist: dirigent-plugin==0.16.3
 Requires-Dist: jsonschema>=4.26.0
 Requires-Dist: opentelemetry-api>=1.44.0
 Requires-Dist: pydantic-settings>=2.15.0

{dirigent_core-0.16.2 → dirigent_core-0.16.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-core"
-version = "0.16.2"
+version = "0.16.3"
 description = "Dirigent engine core: schema, configuration, and plugin host."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -13,9 +13,9 @@ dependencies = [
     "asyncpg>=0.31.0",
     "cronsim>=2.7",
     "cryptography>=46.0.5",
-    "dirigent-client==0.16.2",
-    "dirigent-common==0.16.2",
-    "dirigent-plugin==0.16.2",
+    "dirigent-client==0.16.3",
+    "dirigent-common==0.16.3",
+    "dirigent-plugin==0.16.3",
     "jsonschema>=4.26.0",
     "opentelemetry-api>=1.44.0",
     "pydantic-settings>=2.15.0",

{dirigent_core-0.16.2 → dirigent_core-0.16.3}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-core"
-version = "0.16.2"
+version = "0.16.3"
 description = "Dirigent engine core: schema, configuration, and plugin host."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -13,9 +13,9 @@ dependencies = [
     "asyncpg>=0.31.0",
     "cronsim>=2.7",
     "cryptography>=46.0.5",
-    "dirigent-client==0.16.2",
-    "dirigent-common==0.16.2",
-    "dirigent-plugin==0.16.2",
+    "dirigent-client==0.16.3",
+    "dirigent-common==0.16.3",
+    "dirigent-plugin==0.16.3",
     "jsonschema>=4.26.0",
     "opentelemetry-api>=1.44.0",
     "pydantic-settings>=2.15.0",

{dirigent_core-0.16.2 → dirigent_core-0.16.3}/src/dirigent_core/alerting.py

@@ -27,7 +27,7 @@ from dirigent_common import (
     format_duration,
     render,
 )
-from dirigent_core.database import session_scope
+from dirigent_core.database import is_deadlock, session_scope
 from dirigent_core.engine.definition import load_definition
 from dirigent_core.engine.services import EngineServices
 from dirigent_core.ids import uuid7
@@ -326,7 +326,43 @@ async def raise_for_run(
     has already assembled them, and they are read here when a rule matched and the caller had
     none, so a run nothing watches pays for no facts at all. The rendered document is what the
     templates read as ``report``; it is not stored on the notification.
+
+    A savepoint, and everything that fails inside it is logged rather than raised: the caller
+    has already written the outcome this reports on, and an alert nobody can queue must not
+    cost the work a second execution. A deadlock is the one failure passed on, because the
+    caller answers that by running its whole transaction again.
     """
+    try:
+        async with session.begin_nested():
+            return await _queue_for_run(
+                session,
+                services,
+                run,
+                event,
+                now=now,
+                facts=facts,
+                report=report,
+                report_artifact_id=report_artifact_id,
+            )
+    except Exception as error:
+        if is_deadlock(error):
+            raise
+        _logger.error("alerts not raised", run_id=str(run.id), alert_event=event.value, error=str(error))
+        return []
+
+
+async def _queue_for_run(
+    session: AsyncSession,
+    services: EngineServices,
+    run: Run,
+    event: AlertEvent,
+    *,
+    now: datetime | None = None,
+    facts: "RunFacts | None" = None,
+    report: str | None = None,
+    report_artifact_id: UUID | None = None,
+) -> list[Notification]:
+    """Write down the notifications and the timeline entries one raised event owes."""
     moment = now or utcnow()
     pipeline = await session.get(Pipeline, run.pipeline_id)
     if pipeline is None:  # pragma: no cover - the foreign key makes this unreachable

{dirigent_core-0.16.2 → dirigent_core-0.16.3}/src/dirigent_core/engine/executor.py

@@ -547,13 +547,17 @@ class Engine:
         """Append what the buffer holds to the run, in a transaction of its own.
 
         Committed entries leave the buffer for good, so the outcome cannot write them twice;
-        a flush that failed puts them back, for the next flush or for the outcome.
+        a flush that failed puts them back, for the next flush or for the outcome. The run's
+        lock holds from the first id this takes until the commit that makes it readable, so
+        two attempts flushing at once cannot commit their entries out of id order and leave a
+        stream paging by id past one that was still in flight.
         """
         entries = logger.drain()
         if not entries:
             return
         try:
             async with session_scope(self.sessions) as session:
+                await lock_run(session, logger.run_id)
                 await session.execute(sa.insert(LogEntry), entries)
         except Exception as error:
             logger.restore(entries)

{dirigent_core-0.16.2 → dirigent_core-0.16.3}/src/dirigent_core/engine/references.py

@@ -18,8 +18,7 @@ how a compose file, a shell command or a template reaches a tool with its own br
 """
 
 import re
-import shlex
-from collections.abc import Callable, Container
+from collections.abc import Callable, Collection
 from datetime import datetime
 from typing import Final, cast
 from uuid import UUID
@@ -27,6 +26,7 @@ from uuid import UUID
 from pydantic import BaseModel, ConfigDict, Field, JsonValue
 
 from dirigent_common import JsonMap
+from dirigent_plugin import SHELL_VARIABLE_PREFIX, SHELL_VARIABLES_FIELD
 
 REFERENCE_PATTERN: Final = re.compile(r"(\$+)\{([^{}]+)\}")
 """A run of dollars before a braced name.
@@ -88,28 +88,38 @@ class ReferenceScope(BaseModel):
     """The end of the run's logical data interval, exclusive, when it carries one."""
 
 
-def resolve(value: JsonValue, scope: ReferenceScope, *, quote: bool = False) -> JsonValue:
+def resolve(value: JsonValue, scope: ReferenceScope) -> JsonValue:
     """Resolve every reference in a config value, recursively, preserving structure."""
     match value:
         case str():
-            return _resolve_string(value, scope, quote=quote)
+            return _resolve_string(value, scope)
         case list():
-            return [resolve(element, scope, quote=quote) for element in value]
+            return [resolve(element, scope) for element in value]
         case dict():
-            return {key: resolve(element, scope, quote=quote) for key, element in value.items()}
+            return {key: resolve(element, scope) for key, element in value.items()}
         case _:
             return value
 
 
-def resolve_config(config: JsonMap, scope: ReferenceScope, *, shell_fields: Container[str] = frozenset()) -> JsonMap:
+def resolve_config(config: JsonMap, scope: ReferenceScope, *, shell_fields: Collection[str] = ()) -> JsonMap:
     """Resolve a step's whole config map, which is what the engine hands a block.
 
     ``shell_fields`` names the config keys the block marked with
-    :class:`~dirigent_plugin.ShellString`, whose value is handed to ``sh -c``. In those, every
-    substituted value is shell-quoted, so a parameter that arrived in a webhook payload
-    becomes exactly one word; the metacharacters the pipeline author typed keep their meaning.
+    :class:`~dirigent_plugin.ShellString`, whose value is handed to ``sh -c``. There a
+    substituted value never reaches the shell's parser: each reference is rewritten to a
+    variable of the engine's own, and the values are collected in ``shell_variables`` for the
+    block to set in the command's environment. The metacharacters the pipeline author typed
+    keep their meaning, and a parameter that arrived in a webhook payload is never shell
+    source, however the author quoted the reference.
     """
-    return {key: resolve(value, scope, quote=key in shell_fields) for key, value in config.items()}
+    variables: dict[str, str] = {}
+    resolved = {
+        key: (_resolve_shell(value, scope, variables) if key in shell_fields else resolve(value, scope))
+        for key, value in config.items()
+    }
+    if shell_fields:
+        resolved[SHELL_VARIABLES_FIELD] = cast("JsonValue", variables)
+    return resolved
 
 
 def has_reference(value: str) -> bool:
@@ -124,15 +134,24 @@ def substitute(value: str, render: Callable[[str], str]) -> str:
     """
 
     def one(match: re.Match[str]) -> str:
-        dollars, reference = match.group(1), match.group(2)
-        literal = "$" * (len(dollars) // 2)
-        if len(dollars) % 2 == 0:
-            return f"{literal}{{{reference}}}"
-        return literal + render(reference)
+        literal, resolves = _collapse(match.group(1), match.group(2))
+        return literal + render(match.group(2)) if resolves else literal
 
     return REFERENCE_PATTERN.sub(one, value)
 
 
+def _collapse(dollars: str, reference: str) -> tuple[str, bool]:
+    """The literal text a run of dollars leaves, and whether the reference after it resolves.
+
+    The dollars collapse in pairs, so an even run leaves the braces as text and an odd one
+    makes what follows a reference.
+    """
+    literal = "$" * (len(dollars) // 2)
+    if len(dollars) % 2 == 0:
+        return f"{literal}{{{reference}}}", False
+    return literal, True
+
+
 def references_in(value: JsonValue) -> list[str]:
     """List every reference a value names, skipping the escaped ones."""
     match value:
@@ -146,22 +165,87 @@ def references_in(value: JsonValue) -> list[str]:
             return []
 
 
-def _resolve_string(value: str, scope: ReferenceScope, *, quote: bool = False) -> JsonValue:
-    """Resolve a string, typed when it is one whole reference and textual when embedded.
-
-    Under ``quote`` the whole-reference shortcut is dropped too: handing a command that is
-    entirely one reference to a shell unquoted would let a parameter be an arbitrary program.
-    """
+def _resolve_string(value: str, scope: ReferenceScope) -> JsonValue:
+    """Resolve a string, typed when it is one whole reference and textual when embedded."""
     whole = WHOLE_REFERENCE.match(value)
-    if whole is not None and not quote:
+    if whole is not None:
         return lookup(whole.group(1).strip(), scope)
-    render = _as_shell_word if quote else _as_text
-    return substitute(value, lambda reference: render(lookup(reference.strip(), scope)))
+    return substitute(value, lambda reference: _as_text(lookup(reference.strip(), scope)))
+
 
+def _resolve_shell(value: JsonValue, scope: ReferenceScope, variables: dict[str, str]) -> JsonValue:
+    """Rewrite a shell string so that nothing it substitutes is ever parsed by the shell.
 
-def _as_shell_word(value: JsonValue) -> str:
-    """Render a resolved value as exactly one word for a shell, whatever it contains."""
-    return shlex.quote(_as_text(value))
+    Each reference becomes a reference to a variable the engine invents, and its value is
+    recorded in ``variables`` under that name for the block to set in the command's
+    environment. A variable's value is text a shell expands and never re-reads, so what a
+    webhook payload carries cannot become shell source however the author wrote the
+    reference; what the quoting below decides is only whether that text stays one word.
+
+    A shell string is always text, so the whole-reference shortcut does not apply: a command
+    that is entirely one reference is one word, and therefore the name of a program to run
+    and never a program.
+    """
+    if not isinstance(value, str):
+        return value
+    rewritten: list[str] = []
+    context: tuple[str, ...] = ()
+    read = 0
+    for match in REFERENCE_PATTERN.finditer(value):
+        before = value[read : match.start()]
+        context = _shell_context(before, context)
+        literal, resolves = _collapse(match.group(1), match.group(2))
+        rewritten.append(before + literal)
+        if resolves:
+            name = f"{SHELL_VARIABLE_PREFIX}{len(variables)}"
+            variables[name] = _as_text(lookup(match.group(2).strip(), scope))
+            rewritten.append(_shell_word(name, context))
+        read = match.end()
+    rewritten.append(value[read:])
+    return "".join(rewritten)
+
+
+def _shell_word(name: str, context: tuple[str, ...]) -> str:
+    """A reference to the variable that is one word where the author put it.
+
+    Inside the author's quotes it takes none of its own: a second pair would end theirs and
+    leave the value to be split into words. Anywhere else it takes a pair, or the shell would
+    split the value and expand any glob in it -- bare, and inside a ``$( )`` or a backquoted
+    command, which quote nothing they hold. Inside single quotes, which a shell keeps
+    literal, this text is what the author gets rather than the value.
+    """
+    return f"${name}" if context and context[-1] in "'\"" else f'"${name}"'
+
+
+def _shell_context(text: str, context: tuple[str, ...]) -> tuple[str, ...]:
+    """Where in a shell's quoting this text leaves it, given where it began.
+
+    The stack holds what the shell has opened and not yet closed: a quote, a ``$( )``
+    substitution, or a backquoted one, each of which quotes what is inside it in its own
+    right. Only the innermost decides how a reference there is written, and getting that
+    wrong can only cost a value its word boundaries -- never make it shell source.
+    """
+    stack = list(context)
+    escaped = skipped = False
+    for index, character in enumerate(text):
+        inner = stack[-1] if stack else ""
+        if skipped or escaped:
+            skipped = escaped = False
+        elif inner == "'":
+            if character == "'":
+                stack.pop()
+        elif character == "\\":
+            escaped = True
+        elif inner in '"`' and character == inner:
+            stack.pop()
+        elif character == "$" and text[index + 1 : index + 2] == "(":
+            stack.append("(")
+            skipped = True
+        elif character == "`" or (character in "'\"" and inner != '"'):
+            stack.append(character)
+        elif character == ")" and inner == "(":
+            stack.pop()
+    return tuple(stack)
 
 
 def _as_text(value: JsonValue) -> str:

{dirigent_core-0.16.2 → dirigent_core-0.16.3}/src/dirigent_core/engine/runs.py

@@ -399,7 +399,10 @@ async def cancel_run(
     raised and the attempt reaches its terminal state either way.
     """
     moment = now or utcnow()
-    # The lock comes before the terminal check, not after. Without it, a cancel racing the
+    # The pipeline lock first, because this frees the concurrency slot at the end and taking
+    # it there would meet a run creation holding it and waiting for this very run.
+    await lock_pipeline(session, run.pipeline_id)
+    # The run lock comes before the terminal check, not after. Without it, a cancel racing the
     # outcome transaction that settles the run's last attempt reads "still running" and then
     # overwrites an already-succeeded run with "cancelled".
     await lock_run(session, run.id)
@@ -512,10 +515,16 @@ async def promote_queued_run(session: AsyncSession, services: EngineServices, pi
 
     A run settling frees the slot only if it held it: cancelling a held run settles a run that
     never did, and releasing there would run two of a ``queue`` pipeline at once.
+
+    Releasing and creating are the same read-then-decide-then-write over one slot, so both
+    make it under the pipeline lock. Without it a creation that reads the settling run as
+    still active writes a held run this call cannot see, and that run waits for a sibling
+    that has already gone.
     """
     # The caller's own writes decide whether the slot is free, and the session does not
     # autoflush, so the run and attempts read below would otherwise be the ones on disk.
     await session.flush()
+    await lock_pipeline(session, pipeline_id)
     held: list[Run] = []
     for run in await active_runs(session, pipeline_id):
         if await _occupies_slot(session, run):

{dirigent_core-0.16.2 → dirigent_core-0.16.3}/src/dirigent_core/engine/state.py

@@ -261,10 +261,13 @@ def derive_run_status(states: dict[str, StepState], *, cancelled: bool = False)
 
 
 async def lock_run(session: AsyncSession, run_id: UUID) -> None:
-    """Serialise the outcome transactions of one run, so its status is derived once.
+    """Serialise the transactions of one run, so its status is derived once.
 
     Without this, two workers settling the run's last two attempts each read the other as
-    still in flight and neither concludes the run finished. PostgreSQL only.
+    still in flight and neither concludes the run finished. An attempt's log flush takes it
+    too: an entry's id is taken when its row is inserted and readable only once its
+    transaction commits, so two writers that do not exclude each other can leave a stream
+    paging by id past an entry that was still in flight. PostgreSQL only.
     """
     if session.get_bind().dialect.name != "postgresql":
         return
@@ -272,10 +275,16 @@ async def lock_run(session: AsyncSession, run_id: UUID) -> None:
 
 
 async def lock_pipeline(session: AsyncSession, pipeline_id: UUID) -> None:
-    """Serialise the run-creation decisions of one pipeline, so its concurrency policy holds.
+    """Serialise the concurrency-slot decisions of one pipeline, so its policy holds.
 
-    ``skip`` and ``queue`` are read-then-decide-then-write across processes. PostgreSQL only,
-    as :func:`lock_run` is.
+    ``skip`` and ``queue`` are read-then-decide-then-write across processes, and so is
+    releasing the run one of them held. PostgreSQL only, as :func:`lock_run` is.
+
+    A transaction that takes both locks takes this one first: creating a run, retrying a
+    step, and cancelling all do. The settle path is the single exception, because it holds
+    the run lock from its first statement and reaches the pipeline only at the end, when the
+    run it settled frees the slot. That leaves one cycle of two, which PostgreSQL breaks and
+    ``with_deadlock_retry`` runs again.
     """
     if session.get_bind().dialect.name != "postgresql":
         return