dirigent-examples 0.20.0__tar.gz → 0.21.0__tar.gz

{dirigent_examples-0.20.0 → dirigent_examples-0.21.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: dirigent-examples
-Version: 0.20.0
+Version: 0.21.0
 Summary: The dirigent example corpus: the shelves of runnable documents, installed.
 License-Expression: LicenseRef-Proprietary
 License-File: LICENSE
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: dirigent-client==0.20.0
-Requires-Dist: dirigent-plugin==0.20.0
+Requires-Dist: dirigent-client==0.21.0
+Requires-Dist: dirigent-plugin==0.21.0
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 

{dirigent_examples-0.20.0 → dirigent_examples-0.21.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-examples"
-version = "0.20.0"
+version = "0.21.0"
 description = "The dirigent example corpus: the shelves of runnable documents, installed."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -11,8 +11,8 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-    "dirigent-client==0.20.0",
-    "dirigent-plugin==0.20.0",
+    "dirigent-client==0.21.0",
+    "dirigent-plugin==0.21.0",
 ]
 
 [project.entry-points."dirigent.plugins.v1"]

{dirigent_examples-0.20.0 → dirigent_examples-0.21.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-examples"
-version = "0.20.0"
+version = "0.21.0"
 description = "The dirigent example corpus: the shelves of runnable documents, installed."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -12,8 +12,8 @@ classifiers = [
 ]
 dependencies = [
     # The python/ shelf is SDK scripts a reader runs, so the corpus carries the client.
-    "dirigent-client==0.20.0",
-    "dirigent-plugin==0.20.0",
+    "dirigent-client==0.21.0",
+    "dirigent-plugin==0.21.0",
 ]
 
 [project.entry-points."dirigent.plugins.v1"]

{dirigent_examples-0.20.0 → dirigent_examples-0.21.0}/src/dirigent_examples/shelves/README.md

@@ -18,7 +18,7 @@ The documents live on topic shelves, each with its own README:
 | [`failure/`](failure) | Retries, budgets, timeouts, tolerated failures, and the cleanup edge. |
 | [`transform/`](transform) | The reshaping verbs: jq programs and the `convert.std` codec, no allowlist anywhere. |
 | [`recipes/`](recipes) | One question per file: how to group, join, pivot, clean, convert, check, store, and call. |
-| [`triggers/`](triggers) | What starts a run on its own: the three clocks, and the inbound webhook. |
+| [`triggers/`](triggers) | What starts a run on its own: the three clocks, the inbound webhook, and a watch on a sensor. |
 | [`sensors/`](sensors) | Steps that wait for the world: a drop landing, a clock window opening. |
 | [`queues/`](queues) | A run started by a message: a Kafka topic and a RabbitMQ queue, waited on as sensors. |
 | [`execute/`](execute) | Code on the worker: a shell step behind the allowlist. |

{dirigent_examples-0.20.0 → dirigent_examples-0.21.0}/src/dirigent_examples/shelves/queues/README.md

@@ -42,6 +42,7 @@ On the compose stack the brokers are the `infra/compose.brokers.yaml` overlay in
 | File | What it teaches |
 | --- | --- |
 | [kafka-consume-then-transform.yaml](kafka-consume-then-transform.yaml) | Waiting on a topic: `min_messages` and `max_messages`, why the cursor rather than a consumer group holds the offsets, and the batch read by a downstream transform. |
+| [kafka-watch-then-transform.yaml](kafka-watch-then-transform.yaml) | A topic read as a stream: a watch keeps one run waiting on `kafka.consume` and starts the next from where each batch ended, with no consumer group, while every batch is reshaped in the run that took it. |
 | [kafka-produce-then-consume.yaml](kafka-produce-then-consume.yaml) | Publishing: the two shapes an element of `records` may take, where a message key comes from, what `acks: all` and an idempotent producer do and do not promise, and why this one consumes from `earliest`. |
 | [report-to-kafka.yaml](report-to-kafka.yaml) | Publishing a rendered page: `report.render` hands its text on, and `kafka.produce` puts it on the topic as one keyed record. |
 | [rabbitmq-consume-ack-on-success.yaml](rabbitmq-consume-ack-on-success.yaml) | When a message is acknowledged: why `ack: on_success` is the default, what a poke that parks does with the messages it took, and what `always` gives up in exchange. |
@@ -56,4 +57,5 @@ the state there. It replaces rather than merges, it is stored by the transaction
 attempt -- which makes advancing it at-least-once, so a poke may read the same ground twice --
 and it lives only as long as the waiting attempt, because a poke that succeeds ends the step.
 Anything a later step needs is in the output, which is why `kafka.consume` reports the offsets
-it ended at there.
+it ended at there. A watch is the one thing that carries a cursor past a success: it stores
+where the batch ended and starts its next run there.

dirigent_examples-0.21.0/src/dirigent_examples/shelves/queues/kafka-watch-then-transform.yaml

@@ -0,0 +1,88 @@
+# A topic read as a stream: a watch keeps one run waiting on it, and each batch it takes arms
+# the next run from where that batch ended.
+#
+# NEEDS A BROKER, and a topic to read:
+#   docker compose -f infra/compose.queues.yaml up -d --wait
+#   docker exec -it dirigent-queues-redpanda-1 rpk topic create orders
+#   dg connection create kafka orders-topic --set bootstrap_servers='["127.0.0.1:9092"]'
+#   dg apply examples/queues/kafka-watch-then-transform.yaml
+# Then produce to the topic in another shell, as often as you like, and watch the runs line up:
+#   docker exec -it dirigent-queues-redpanda-1 \
+#     rpk topic produce orders --format '%v\n'   # then type JSON lines and press ctrl-d
+#   dg runs list --pipeline kafka-watch-then-transform
+# On the compose stack the broker is the infra/compose.brokers.yaml overlay and the connection
+# names it as `redpanda:9092`; docs/queues.md is the family's home.
+#
+# Two hops, and what each one hands on:
+#   take       kafka.consume, the watched step. It parks until there is a batch, then hands on
+#              the messages, their count, and the offsets it ended at.
+#   summarise  a jq transform over that batch, in the run that took it -- while the next run
+#              is already waiting on the topic.
+#
+# THE WATCH OWNS THE CURSOR, SO NO group_id. kafka.consume says where a batch ended, the watch
+# stores it, and the next run's first poke reads on from exactly there. Nothing is committed
+# to the broker, nothing on the cluster has to be arranged, and a second pipeline over the same
+# topic still sees every message. Set a group_id as well if the broker should know how far this
+# pipeline has read; the watch's cursor stays what the next run starts from.
+#
+# start: earliest APPLIES ONCE. It says where the very first run begins, which here is the
+# oldest message the broker still holds. Every run after it starts from the watch's cursor, so
+# nothing is replayed and nothing is skipped between two runs.
+#
+# NO deadline. A watched sensor waits as long as the topic is quiet, and that costs a row with a
+# due time, not a worker. `dg watch pause kafka-watch-then-transform follow` stops it, cancelling
+# the run that is waiting and keeping the cursor for `dg watch resume`.
+#
+# READING THE SAME GROUND TWICE. The cursor is stored by the transaction that settles the
+# batch, so a worker that dies before that commit leaves the older offsets behind and the next
+# poke reads those messages again. A step that must not act twice on a message makes itself
+# idempotent; this one only reshapes, so it does not care.
+#
+# TO MAKE IT YOURS: point the connection at your own cluster, change the topic, and replace the
+# summarise step with whatever consumes the batch.
+
+format: dirigent/v1
+kind: pipeline
+code: kafka-watch-then-transform
+name: Watch a Kafka topic, then transform each batch
+description: |
+  A watch on kafka.consume reads a topic run after run from where the last batch ended, with
+  no consumer group, and each batch is reshaped in the run that took it.
+
+tags: [queues, kafka, sensor, transform, triggers]
+
+requires:
+  blocks:
+    - kafka.consume
+    - transform.jq
+  connections:
+    - orders-topic
+
+steps:
+  take:
+    block: kafka.consume
+    poll: 5s
+    config:
+      connection: orders-topic
+      topic: orders
+      min_messages: 1
+      # Bounds what one run carries: a burst of a thousand messages becomes ten runs of a
+      # hundred, each summarised on its own while the next one is taken.
+      max_messages: 100
+      start: earliest
+      value_format: json
+
+  summarise:
+    block: transform.jq
+    depends_on: [take]
+    config:
+      input: ${steps.take.output}
+      program: >-
+        {count: .count, first_offset: (.messages | map(.offset) | min),
+         last_offset: (.messages | map(.offset) | max), resume_from: .cursor}
+
+triggers:
+  watches:
+    - code: follow
+      name: Follow the orders
+      step: take

{dirigent_examples-0.20.0 → dirigent_examples-0.21.0}/src/dirigent_examples/shelves/triggers/README.md

@@ -1,10 +1,10 @@
 # Trigger examples
 
-What starts a run on its own: a clock, or an inbound webhook. A trigger is declared in the
-document and materialised by `dg apply`; the scheduler fires the clocks and the server
-answers the hooks, on the instance the document was applied to. Operational state -- paused,
-last fired, the firing and delivery history -- stays in the instance, so none of it appears
-in these files. [docs/design.md](../../docs/design.md#the-misfire-policy) is the page behind
+What starts a run on its own: a clock, an inbound webhook, or a watch. A trigger is declared
+in the document and materialised by `dg apply`; the scheduler fires the clocks, the server
+answers the hooks, and a watch keeps a run waiting on its sensor, on the instance the document
+was applied to. Operational state -- paused, last fired, the firing and delivery history, the
+cursor a watch holds -- stays in the instance, so none of it appears in these files. [docs/design.md](../../docs/design.md#the-misfire-policy) is the page behind
 the clocks.
 
 Each file is prefixed with the clock kind it teaches, the way `transform/`'s files are
@@ -34,6 +34,7 @@ dg run --local examples/triggers/cron-windowed.yaml --window 2026-06-01..2026-06
 | [managed-and-manual.yaml](managed-and-manual.yaml) | Against a real instance: what an apply materialises, what it removes, and the hand-made schedule it leaves alone. |
 | [document-nightly.yaml](document-nightly.yaml) | A `kind: triggers` document: clocks for a pipeline defined elsewhere, owning its own rows and refusing a pipeline no instance holds. |
 | [webhook-trigger.yaml](webhook-trigger.yaml) | The inbound trigger: a token minted at apply, a strict payload-to-parameter mapping, and everything else in the POST ignored. |
+| [watch-a-sensor.yaml](watch-a-sensor.yaml) | A watch: one run always waiting on `playground.arrive`, the next armed the moment a batch arrives, the runs overlapping and the offsets carrying on from batch to batch. |
 
 A pipeline carries as many schedules as it needs, each with its own zone and its own
 parameters: nightly against production and hourly against staging is two schedules on one

dirigent_examples-0.21.0/src/dirigent_examples/shelves/triggers/watch-a-sensor.yaml

@@ -0,0 +1,85 @@
+# A sensor made a continuous source: one run always waiting on it, the next armed the moment
+# it succeeds.
+#
+# A schedule starts a run at a moment and a webhook when somebody calls. A watch starts one
+# whenever the last one has what it was waiting for: it keeps exactly one run of this pipeline
+# waiting on the `arrive` step, and the transaction that settles that step's success arms the
+# next run. The run that succeeded carries on downstream while the next one waits, so the runs
+# overlap -- batch N is being summarised while batch N+1 is being waited for -- and the run list
+# fills the way a topic being consumed does. Nothing here needs a network or an allowlist:
+#
+#   dg apply examples/triggers/watch-a-sensor.yaml
+#   dg watch list watch-a-sensor                 # the run it has waiting, and where it left off
+#   dg runs list --pipeline watch-a-sensor       # a new run every five seconds or so
+#   dg watch pause watch-a-sensor follow         # cancels the waiting run, keeps the cursor
+#   dg watch resume watch-a-sensor follow        # arms a run that reads on from there
+#
+# `dg run --local` runs the pipeline once, as an ordinary run: a throwaway instance runs the
+# one run it was asked for, so the watch lands paused there.
+#
+# Hop by hop:
+#
+#   arrive      playground.arrive, parked for five seconds, then a batch of three generated
+#               readings. Under the watch its offsets carry on from the batch before: the
+#               first run's are 0 to 2, the next run's 3 to 5, and so on.
+#   summarise   a jq transform over the batch, which reads it the way a step downstream of a
+#               topic would: how many, and the first and last offset.
+#
+# THE WATCH OWNS THE CURSOR. playground.arrive says where a batch ended, and the watch stores
+# that and hands it to the next run's first poke. kafka.consume does the same with its
+# offsets, which is how a watched topic is read on without a consumer group:
+# examples/queues/kafka-watch-then-transform.yaml is the same shape against a real broker.
+#
+# A WAIT THAT FAILS BACKS OFF. Were `arrive` to fail after its retries, the watch would record
+# why (`dg watch show watch-a-sensor follow` reads it as last_error) and arm again after five
+# seconds, doubling up to five minutes, from the last cursor a success stored. A deadline
+# that ran out under on_timeout: skip found nothing, which is no failure: it arms at once.
+
+format: dirigent/v1
+kind: pipeline
+code: watch-a-sensor
+name: Watch a sensor
+description: |
+  A watch keeps one run waiting on a generated batch and arms the next the moment it
+  arrives, so the runs overlap and the offsets carry on from one batch to the next.
+
+tags: [triggers, playground, sensor, jq, starter]
+
+requires:
+  blocks:
+    - playground.arrive
+    - transform.jq
+
+steps:
+  arrive:
+    block: playground.arrive
+    # A watch waits on a root step, one that depends on nothing, and on a sensor: the watch
+    # is refused at apply otherwise.
+    poll: 1s
+    deadline: 1m
+    config:
+      # Five seconds from when the attempt started, so a new batch lands every five seconds
+      # or so. after_pokes is zero because the clock is what this wait is about.
+      after: 5s
+      after_pokes: 0
+      rows: 3
+      fields:
+        station: city
+        reading: pyfloat
+
+  summarise:
+    block: transform.jq
+    depends_on: [arrive]
+    config:
+      input: ${steps.arrive.output}
+      program: >-
+        {count: .count, first_offset: .messages[0].offset,
+         last_offset: .messages[-1].offset, next_offset: .next_offset}
+
+triggers:
+  watches:
+    - code: follow
+      name: Follow the arrivals
+      # The step every armed run waits on. Its success arms the next run; everything
+      # downstream of it runs in the run that succeeded.
+      step: arrive