dirigent-core 0.17.2__tar.gz → 0.17.3__tar.gz

{dirigent_core-0.17.2 → dirigent_core-0.17.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirigent-core
-Version: 0.17.2
+Version: 0.17.3
 Summary: Dirigent engine core: schema, configuration, and plugin host.
 License-Expression: LicenseRef-Proprietary
 License-File: LICENSE
@@ -12,9 +12,9 @@ Requires-Dist: argon2-cffi>=25.1.0
 Requires-Dist: asyncpg>=0.31.0
 Requires-Dist: cronsim>=2.7
 Requires-Dist: cryptography>=46.0.5
-Requires-Dist: dirigent-client==0.17.2
-Requires-Dist: dirigent-common==0.17.2
-Requires-Dist: dirigent-plugin==0.17.2
+Requires-Dist: dirigent-client==0.17.3
+Requires-Dist: dirigent-common==0.17.3
+Requires-Dist: dirigent-plugin==0.17.3
 Requires-Dist: jsonschema>=4.26.0
 Requires-Dist: opentelemetry-api>=1.44.0
 Requires-Dist: pydantic-settings>=2.15.0

{dirigent_core-0.17.2 → dirigent_core-0.17.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-core"
-version = "0.17.2"
+version = "0.17.3"
 description = "Dirigent engine core: schema, configuration, and plugin host."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -17,9 +17,9 @@ dependencies = [
     "asyncpg>=0.31.0",
     "cronsim>=2.7",
     "cryptography>=46.0.5",
-    "dirigent-client==0.17.2",
-    "dirigent-common==0.17.2",
-    "dirigent-plugin==0.17.2",
+    "dirigent-client==0.17.3",
+    "dirigent-common==0.17.3",
+    "dirigent-plugin==0.17.3",
     "jsonschema>=4.26.0",
     "opentelemetry-api>=1.44.0",
     "pydantic-settings>=2.15.0",

{dirigent_core-0.17.2 → dirigent_core-0.17.3}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-core"
-version = "0.17.2"
+version = "0.17.3"
 description = "Dirigent engine core: schema, configuration, and plugin host."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -17,9 +17,9 @@ dependencies = [
     "asyncpg>=0.31.0",
     "cronsim>=2.7",
     "cryptography>=46.0.5",
-    "dirigent-client==0.17.2",
-    "dirigent-common==0.17.2",
-    "dirigent-plugin==0.17.2",
+    "dirigent-client==0.17.3",
+    "dirigent-common==0.17.3",
+    "dirigent-plugin==0.17.3",
     "jsonschema>=4.26.0",
     "opentelemetry-api>=1.44.0",
     "pydantic-settings>=2.15.0",

{dirigent_core-0.17.2 → dirigent_core-0.17.3}/src/dirigent_core/alembic/versions/0001_baseline_schema.py

@@ -808,6 +808,9 @@ def upgrade() -> None:
         sa.Column(
             "remote_handle", sa.JSON().with_variant(postgresql.JSONB(astext_type=Text()), "postgresql"), nullable=True
         ),
+        sa.Column(
+            "fetched_output", sa.JSON().with_variant(postgresql.JSONB(astext_type=Text()), "postgresql"), nullable=True
+        ),
         sa.Column(
             "poke_cursor", sa.JSON().with_variant(postgresql.JSONB(astext_type=Text()), "postgresql"), nullable=True
         ),

{dirigent_core-0.17.2 → dirigent_core-0.17.3}/src/dirigent_core/engine/claim.py

@@ -60,6 +60,9 @@ class ClaimedUnit(BaseModel):
     """The run's trace context, so an attempt's span joins the trace the run started in."""
 
     remote_handle: RemoteHandle | None = None
+    fetched_output: JsonMap | None = None
+    """The result a previous call already fetched and committed, which settles this claim."""
+
     poke_cursor: JsonMap | None = None
     """The cursor the last committed poke returned, handed to the next one."""
 

{dirigent_core-0.17.2 → dirigent_core-0.17.3}/src/dirigent_core/engine/context.py

@@ -6,7 +6,8 @@ long step is visible working and its last lines still land with the outcome.
 """
 
 import asyncio
-from collections.abc import Mapping
+from collections.abc import AsyncGenerator, Mapping
+from contextlib import AbstractAsyncContextManager, asynccontextmanager
 from datetime import UTC, datetime, timedelta
 from fnmatch import fnmatch
 from pathlib import Path
@@ -26,7 +27,7 @@ from dirigent_core.messages import RUN_UNKNOWN_CONNECTION, RUN_UNKNOWN_SCHEMA, U
 from dirigent_core.models import Connection, Schema
 from dirigent_core.secrets import SecretBox
 from dirigent_core.storage import AttemptStorage, Storage, connection_binder, work_dir
-from dirigent_plugin import BlockFailure, ConnectionRef, ErrorClass, Logger, Runs
+from dirigent_plugin import BlockFailure, ByteSink, Capture, ConnectionRef, ErrorClass, Logger, Runs
 
 #: What an HTTP connection is assumed to call its fields, so core can build a client for
 #: any connection kind without importing the package that contributed it.
@@ -230,6 +231,19 @@ class BufferedLogger(BaseModel):
             self.full.set()
 
 
+class CapturedStream:
+    """A sink that carries the URI the engine opened it under, for a block to put in its output."""
+
+    def __init__(self, uri: str, sink: ByteSink) -> None:
+        """Bind the storage sink to the URI naming it."""
+        self.uri = uri
+        self._sink = sink
+
+    async def write(self, data: bytes) -> int:
+        """Append bytes to the stream and return how many were accepted."""
+        return await self._sink.write(data)
+
+
 class EngineStepContext:
     """The concrete StepContext the engine hands a block for exactly one call."""
 
@@ -297,6 +311,26 @@ class EngineStepContext:
         """Return the run-scoped URI prefix for intermediate artifacts."""
         return self._scratch
 
+    def capture(self, name: str, *, content_type: str = "text/plain") -> AbstractAsyncContextManager[Capture]:
+        """Open the storage object a block's named stream is written to, under the run's scratch."""
+        uri = f"{self._scratch.rstrip('/')}/{self._segment()}-{name}"
+
+        @asynccontextmanager
+        async def opened() -> AsyncGenerator[Capture]:
+            async with self.storage.open_write(uri, content_type=content_type) as sink:
+                yield CapturedStream(uri, sink)
+
+        return opened()
+
+    def _segment(self) -> str:
+        """The path this attempt's captures sit under: the step, the fan-out item, and the attempt.
+
+        ``{step}[/{item}]/attempt-{n}``. The step key is unique within the run and the item id
+        within a fan-out, so no two attempts of one run write over each other's streams.
+        """
+        item = f"/{self.run_item_id}" if self.run_item_id is not None else ""
+        return f"{self.step}{item}/attempt-{self.attempt}"
+
     @property
     def work(self) -> Path:
         """Return the run's directory on this worker's own filesystem, made on first read.

{dirigent_core-0.17.2 → dirigent_core-0.17.3}/src/dirigent_core/engine/executor.py

@@ -1,9 +1,10 @@
 """The engine proper: claim, call one block, record the outcome, ready the dependents.
 
 Two invariants carry everything here. One transaction per state transition, so there is no
-window in which a step is finished but its dependents have not been told. And a remote
-handle is committed on its own the moment ``execute`` returns it, so submit-then-crash
-recovers into polling rather than into a duplicate job.
+window in which a step is finished but its dependents have not been told. And what a remote
+call hands back is committed on its own the moment it arrives -- the handle ``execute``
+returns, and the result ``fetch`` returns -- so a crash recovers into polling or settling
+rather than into a duplicate job or a second fetch.
 """
 
 import asyncio
@@ -97,6 +98,8 @@ class Produced(BaseModel):
     model_config = ConfigDict(frozen=True)
 
     output: JsonMap
+    fetched: bool = False
+    """True when a fetch returned this output and nothing has committed it yet."""
 
 
 class Submitted(BaseModel):
@@ -320,6 +323,7 @@ class Engine:
             schemas=await load_schemas(session),
             traceparent=run.traceparent,
             remote_handle=RemoteHandle.model_validate(attempt.remote_handle) if attempt.remote_handle else None,
+            fetched_output=attempt.fetched_output,
             poke_cursor=attempt.poke_cursor,
             deadline_at=attempt.deadline_at,
             gone_probes=attempt.gone_probes,
@@ -376,6 +380,8 @@ class Engine:
                 telemetry.record_failure(span, result.failure.message)
             if isinstance(result, Submitted):
                 await self.record_handle(unit, result.handle)
+            if isinstance(result, Produced) and result.fetched:
+                await self._record_fetched(unit, result.output)
             await self._record(
                 unit,
                 result,
@@ -440,6 +446,8 @@ class Engine:
         unit: ClaimedUnit,
     ) -> CallResult:
         """Drive an operator's half of the contract: execute once, then probe and fetch."""
+        if unit.fetched_output is not None:
+            return Produced(output=unit.fetched_output)
         if not unit.is_probe or unit.remote_handle is None:
             with self._timed(unit.block_id, "execute"):
                 produced = await operator.execute(config, context)
@@ -460,8 +468,8 @@ class Engine:
                 )
             case ProbeStatus.SUCCEEDED:
                 with self._timed(unit.block_id, "fetch"):
-                    fetched = await operator.fetch(handle, config, context)
-                return Produced(output=fetched.model_dump(mode="json"))
+                    retrieved = await operator.fetch(handle, config, context)
+                return Produced(output=retrieved.model_dump(mode="json"), fetched=True)
             case ProbeStatus.FAILED:
                 failed = (
                     Failure.unknown(REMOTE_JOB_SAID, detail=probe.message)
@@ -532,6 +540,24 @@ class Engine:
                 return
             attempt.remote_handle = handle.model_dump(mode="json")
 
+    async def _record_fetched(self, unit: ClaimedUnit, output: JsonMap) -> None:
+        """Commit a fetched result on its own, before the outcome that settles it."""
+        await with_deadlock_retry(lambda: self._record_fetched_once(unit, output))
+
+    async def _record_fetched_once(self, unit: ClaimedUnit, output: JsonMap) -> None:
+        """Run that transaction once; the caller runs it again after a deadlock.
+
+        It locks the run and then writes the attempt, the order every outcome uses and the
+        opposite of the claim's. Nothing of it survives a rollback, so it is safe to repeat.
+        """
+        async with session_scope(self.sessions) as session:
+            await lock_run(session, unit.run_id)
+            attempt = await session.get(StepAttempt, unit.attempt_id)
+            if attempt is None or not self._holds_lease(unit, attempt):
+                self._log_lost_lease(unit, attempt, "fetched result")
+                return
+            attempt.fetched_output = output
+
     async def _cancel_what_was_just_submitted(
         self,
         session: AsyncSession,
@@ -746,6 +772,7 @@ class Engine:
         )
         attempt.status = AttemptStatus.SUCCEEDED
         attempt.finished_at = now
+        attempt.fetched_output = None
         attempt.error = None
         attempt.error_code = None
         attempt.error_params = None

{dirigent_core-0.17.2 → dirigent_core-0.17.3}/src/dirigent_core/models.py

@@ -288,6 +288,10 @@ class StepAttempt(Entity):
 
     remote_handle: Mapped[JsonMap | None] = mapped_column(JsonDocument)
 
+    fetched_output: Mapped[JsonMap | None] = mapped_column(JsonDocument)
+    """The result a fetch returned, committed before the outcome so a worker that dies
+    between the two settles from it rather than fetching again."""
+
     poke_cursor: Mapped[JsonMap | None] = mapped_column(JsonDocument)
     """How far a sensor's poke has read, handed to the next poke as ``ctx.cursor``."""