dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/recipes/storage-manifest-of-a-fan-out.yaml

@@ -0,0 +1,104 @@
+# One file per item, then one manifest listing all of them.
+#
+# In:  a list of regions, as a parameter.
+# Out: one JSON file per region in the run's scratch space, and a manifest naming every file
+#      with its byte count.
+#
+# A fan-out is `for_each` on a step: the run gets one item per element, each with its own
+# status, error, retry budget and output. Two things about it shape this recipe:
+#
+#   for_each is expanded when the run is created, so it reads params.*, run.*, and another
+#   fan-out's grid as ${steps.<step>.items} -- never a step's output. That is why the regions
+#   are a parameter here and not the result of an upstream read: the item grid exists from the
+#   moment the run is visible.
+#   A fan-out step's output is the list of its items' outputs, in item order. So the step
+#   after the fan reads the whole batch at once, and it reads ${steps.<step>.output} --
+#   the list -- rather than ${steps.<step>.output.uri}, which is a field of one item's
+#   output and not of the list.
+#
+# The fanned step is the storage.write itself, which is the shortest form when one step is
+# doing the work and the writing. Where they are two steps, the second maps over the first's
+# grid with for_each: ${steps.<step>.items} and reads its match -- that is
+# examples/patterns/fan-out-item-wise.yaml.
+#
+# ${item} is the element, and it goes into the target as well as into the value, which is
+# what keeps two items from writing the same key. Items run concurrently, so a shared target
+# would be a race with a winner nobody chose.
+#
+# The manifest is the point. A fan-out that writes twelve files and reports nothing leaves
+# "did region nine actually land" to somebody listing a bucket by hand; a manifest step
+# turns the run itself into the answer, and it costs one jq program over outputs the engine
+# already collected.
+#
+# items: continue is the other half of the same idea -- under it one bad region does not
+# fail the step, the failures are recorded against their own items, and the run reports
+# completed_with_errors. The manifest then lists what did land, which is exactly what a
+# partial run needs to say.
+#
+# To change it: -p regions='["east","west"]' fans two ways instead of four.
+#
+#   dg run --local examples/recipes/storage-manifest-of-a-fan-out.yaml
+#   dg run --local examples/recipes/storage-manifest-of-a-fan-out.yaml -p regions='["east"]'
+
+format: dirigent/v1
+kind: pipeline
+code: storage-manifest-of-a-fan-out
+name: A manifest of a fan-out
+description: Write one file per item with for_each, then read the fan's outputs back as one list and record every file in a manifest.
+
+tags: [recipes, storage, transform, graph]
+
+requires:
+  blocks:
+    - storage.write
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    regions:
+      type: array
+      default: [east, west, north, south]
+      items:
+        type: string
+      description: One item, and one file, per element.
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the files are named for.
+
+steps:
+  per_region:
+    block: storage.write
+    for_each: ${params.regions}
+    # items: continue would let one bad region be recorded against its own item instead of
+    # failing the step; the default, fail_fast, is right when a missing region is an outage.
+    items: fail_fast
+    config:
+      # The item is in the key, so two items never write the same file.
+      target: ${run.scratch}/regions/${params.day}/${item}.json
+      value:
+        region: ${item}
+        day: ${params.day}
+        rows:
+          - { station: "${item}-st-0", celsius: -4 }
+          - { station: "${item}-st-1", celsius: -1 }
+          - { station: "${item}-st-2", celsius: 2 }
+
+  manifest:
+    block: transform.jq
+    depends_on: [per_region]
+    config:
+      # The fan-out's output is the list of its items' outputs, so this reads .output and
+      # not .output.uri.
+      input:
+        items: ${steps.per_region.output}
+        day: ${params.day}
+      program: |
+        . as {$items, $day}
+        | {day: $day,
+           files: ($items | length),
+           total_bytes: ([$items[].bytes_written] | add),
+           entries: [$items[] | {uri, bytes: .bytes_written}],
+           empty: [$items[] | select(.bytes_written == 0) | .uri]}

dirigent_examples/shelves/recipes/storage-write-then-read.yaml

@@ -0,0 +1,119 @@
+# Hand work from one step to the next through storage, which is how a big payload travels.
+#
+# In:  a list built inline.
+# Out: two files in the run's scratch space and a summary read back out of the second one.
+#
+# A value flows through step outputs. storage.write is the only way one goes out and
+# storage.read the only way one comes back in, so a chain that wants to hand work along
+# through storage is written as that pair, twice:
+#
+#   storage.write   target, and exactly one of text or value. Output: uri, bytes_written,
+#                   content_type. A value is written as canonical JSON -- sorted keys, no
+#                   spaces -- so the same value writes the same bytes every run.
+#   storage.read    source, and max_size: how much of the object this step will hold, 1mb
+#                   unless it says otherwise. An object above the cap is refused rather than
+#                   truncated, because half a document is not a smaller one, it is a wrong
+#                   one. The object comes back as `value` when it is JSON and as `text`
+#                   otherwise, and a type it cannot decode it refuses instead of guessing.
+#
+# ${run.scratch} is the run's own prefix in whatever storage the instance is configured
+# with. On a --local run that is a directory under a temporary tree that is deleted when the
+# run ends; on a server it is the artifact root, and with an s3 connection configured for
+# the scheme it is a bucket. The document says none of that: it names a scheme-agnostic
+# prefix, and where the bytes land is an instance setting.
+#
+# Every read below is spelled ${steps.<step>.output.uri} rather than the same path typed
+# twice: one place decides where a file went, and a rename is a one-line change.
+#
+# To change it: point the targets at s3://<bucket>/... and nothing else in the document
+# moves.
+#
+#   dg run --local examples/recipes/storage-write-then-read.yaml
+#   dg run --local examples/recipes/storage-write-then-read.yaml -p day=2026-02-01
+
+format: dirigent/v1
+kind: pipeline
+code: storage-write-then-read
+name: Write to storage, then read it back
+description: Put a transform's result in the run's scratch space with storage.write, read it into the next step with storage.read, and do it again with what that step produced.
+
+tags: [recipes, storage, transform, starter]
+
+requires:
+  blocks:
+    - transform.jq
+    - storage.write
+    - storage.read
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the written files are named for.
+    stations:
+      type: integer
+      minimum: 1
+      default: 50
+      description: How many rows are generated.
+
+steps:
+  generate:
+    block: transform.jq
+    config:
+      input:
+        count: ${params.stations}
+        day: ${params.day}
+      program: |
+        . as {$count, $day}
+        | [range($count)
+           | {station: "st-\(.)", day: $day,
+              celsius: (. % 17) - 5,
+              active: (. % 5 != 0)}]
+
+  save:
+    block: storage.write
+    depends_on: [generate]
+    config:
+      target: ${run.scratch}/readings-${params.day}.json
+      value: ${steps.generate.output.value}
+
+  back:
+    block: storage.read
+    depends_on: [save]
+    config:
+      # Where the write said the object landed, not the same path typed twice.
+      source: ${steps.save.output.uri}
+      max_size: 8mb
+
+  cold:
+    block: transform.jq
+    depends_on: [back]
+    config:
+      # The rows as they came off storage, not as they were held in memory upstream.
+      input: ${steps.back.output.value}
+      program: |
+        [.[] | select(.celsius < 0)]
+
+  save_cold:
+    block: storage.write
+    depends_on: [cold]
+    config:
+      target: ${run.scratch}/cold-${params.day}.json
+      value: ${steps.cold.output.value}
+
+  back_cold:
+    block: storage.read
+    depends_on: [save_cold]
+    config:
+      source: ${steps.save_cold.output.uri}
+
+  summary:
+    block: transform.jq
+    depends_on: [back_cold]
+    config:
+      input: ${steps.back_cold.output.value}
+      program: |
+        {cold_rows: length, coldest: (min_by(.celsius) | .station)}

dirigent_examples/shelves/recipes/webhook-post-hmac.yaml

@@ -0,0 +1,132 @@
+# A signed webhook: what gets signed, with what, and what the receiver checks.
+#
+# In:  a small event built inline.
+# Out: the echo service's view of the delivery, including the X-Dirigent-Signature header,
+#      and the step's own `signed: true`.
+#
+# webhook.post serialises the body itself and signs the bytes that go on the wire. That
+# order is the entire security property: if the body were handed to an HTTP client that
+# re-encoded it -- a different key order, different spacing -- the signature would cover
+# bytes the receiver never sees, and every delivery would fail verification for a reason
+# nobody could find.
+#
+# The signature is HMAC-SHA256 over the raw body, hex, in X-Dirigent-Signature. A receiver
+# verifies by computing the same HMAC over the body exactly as it arrived -- before parsing,
+# never after re-serialising -- and comparing in constant time. Dirigent's own inbound
+# webhook trigger does precisely that, so an instance can hand work to another instance with
+# nothing shared but the secret.
+#
+# sign_with names a connection, not a secret. The secret is that connection's hmac_secret, a
+# SecretStr sealed at rest and redacted in every response, and a connection without one
+# fails the step saying so rather than sending an unsigned POST that looks signed.
+#
+# The connection is carried inline here, labelled, so the recipe runs standalone -- the
+# secret below is a demo string in a public file and protects nothing. On an instance it is
+# created once and the document names its code.
+#
+# A signature is not a substitute for TLS, and it is not a replay defence. It says the body
+# came from someone holding the secret; it says nothing about when. A receiver that cares
+# about replay checks a timestamp or an id it has seen before, which is why the body carries
+# both.
+#
+# The same body is delivered twice, once with sign_with and once without, through the same
+# connection: signing is the step's decision, and the output shows what the second delivery
+# is missing.
+#
+# To change it: point the connection's base_url at a receiver you own and give it the same
+# hmac_secret; nothing in the steps changes.
+#
+#   dg run --local examples/recipes/webhook-post-hmac.yaml
+#   dg run --local examples/recipes/webhook-post-hmac.yaml -p day=2026-02-01
+
+format: dirigent/v1
+kind: pipeline
+code: webhook-post-hmac
+name: A signed webhook delivery
+description: POST an event with an HMAC-SHA256 signature over the exact bytes sent, using a secret that lives on a connection.
+
+tags: [recipes, transform, webhook]
+
+requires:
+  blocks:
+    - transform.jq
+    - webhook.post
+
+# Carried so the recipe runs standalone. hmac_secret is a SecretStr; this one is a demo
+# string in a public file and protects nothing.
+connections:
+  echo-webhook:
+    kind: http
+    config:
+      base_url: https://postman-echo.com
+      hmac_secret: a shared secret between two dirigent instances
+      timeout: 30s
+      health_path: /get
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the event reports on.
+
+steps:
+  event:
+    block: transform.jq
+    config:
+      input:
+        day: ${params.day}
+      program: |
+        {kind: "import.finished",
+         day: .day,
+         # A receiver that cares about replay needs an identity and a time; a signature
+         # gives it neither.
+         event_id: "import-\(.day)",
+         at: "\(.day)T06:15:00Z",
+         rows: 1440}
+
+  deliver:
+    block: webhook.post
+    depends_on: [event]
+    config:
+      connection: echo-webhook
+      path: /post
+      body: ${steps.event.output.value}
+      # The connection whose hmac_secret signs the bytes. Unset means an unsigned POST.
+      sign_with: echo-webhook
+      headers:
+        x-event-kind: import.finished
+
+  deliver_unsigned:
+    block: webhook.post
+    depends_on: [event]
+    config:
+      connection: echo-webhook
+      path: /post
+      body: ${steps.event.output.value}
+      # No sign_with, and the same connection: signing is the step's decision, not the
+      # connection's.
+      headers:
+        x-event-kind: import.finished
+
+  receipt:
+    block: transform.jq
+    depends_on: [deliver, deliver_unsigned]
+    config:
+      input:
+        status: ${steps.deliver.output.status}
+        signed: ${steps.deliver.output.signed}
+        echoed: ${steps.deliver.output.json_body}
+        unsigned_signed: ${steps.deliver_unsigned.output.signed}
+        unsigned_echoed: ${steps.deliver_unsigned.output.json_body}
+      program: |
+        {status, signed,
+         # The echo service shows what arrived: the signature header, and the body it
+         # parsed out of the exact bytes that were signed.
+         signature: (.echoed.headers["x-dirigent-signature"] // null),
+         delivered_event: .echoed.json.event_id,
+         content_type: .echoed.headers["content-type"],
+         unsigned: {signed: .unsigned_signed,
+                    signature: (.unsigned_echoed.headers["x-dirigent-signature"] // null)}}

dirigent_examples/shelves/recipes/webhook-post-summary.yaml

@@ -0,0 +1,142 @@
+# Tell somebody what a run did, in a body they can act on rather than a sentence.
+#
+# In:  a batch of load results, some of them failures.
+# Out: a notification POSTed to a receiver, and its answer -- with the summary built as
+#      structure and the human sentence derived from it, not the other way round.
+#
+# A notification body is an interface, so this recipe builds one rather than formatting a
+# message:
+#
+#   Counts before prose. rows, loaded, rejected and a boolean the receiver can route on.
+#   The identifiers to act with: the run id, the pipeline's day, and the rejected keys --
+#   not "3 records failed", which nobody can look up.
+#   A `text` field derived from the numbers, for a chat client that renders one line. It is
+#   built last, from the same object, so it can never disagree with the counts beside it.
+#   A severity the receiver can route on, decided here where the meaning of the numbers is
+#   known, rather than by a rule written in the receiver's config.
+#
+# webhook.post is the block for this rather than http.request, because a notification is a
+# delivery: it takes sign_with when the receiver verifies signatures, it caps the response
+# it reads at 1mb because nobody wants a receiver's HTML error page in a step output, and
+# success_status is how a receiver that answers 202 is accommodated.
+#
+# The delivery is a step in the DAG like any other, so a rule decides when it runs. This one
+# is `all_done`, so the notification is sent whether the batch succeeded or not -- a
+# notifier that only fires on success is one that goes quiet exactly when it matters.
+#
+# To change it: -p failures=0 sends the green version, and severity, text and routing all
+# follow from the numbers without another branch in the document.
+#
+#   dg run --local examples/recipes/webhook-post-summary.yaml
+#   dg run --local examples/recipes/webhook-post-summary.yaml -p failures=0
+
+format: dirigent/v1
+kind: pipeline
+code: webhook-post-summary
+name: A run summary as a webhook
+description: Build a notification body of counts and identifiers, derive its human line from those counts, and deliver it on all_done.
+
+tags: [recipes, transform, webhook]
+
+requires:
+  blocks:
+    - transform.jq
+    - webhook.post
+
+connections:
+  echo-webhook:
+    kind: http
+    config:
+      base_url: https://postman-echo.com
+      timeout: 30s
+      health_path: /get
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the load reports on.
+    rows:
+      type: integer
+      minimum: 1
+      default: 12
+      description: How many records the load handled.
+    failures:
+      type: integer
+      minimum: 0
+      default: 3
+      description: How many of them were rejected; 0 sends the green version.
+
+steps:
+  results:
+    block: transform.jq
+    config:
+      input:
+        rows: ${params.rows}
+        failures: ${params.failures}
+        day: ${params.day}
+      program: |
+        . as {$rows, $failures, $day}
+        | [range($rows)
+           | {key: "\($day)/rec-\(.)",
+              status: (if . < $failures then "rejected" else "loaded" end),
+              reason: (if . < $failures then "period is outside the data set" else null end)}]
+
+  summary:
+    block: transform.jq
+    depends_on: [results]
+    config:
+      input:
+        rows: ${steps.results.output.value}
+        day: ${params.day}
+        run: ${run.id}
+      program: |
+        . as {$rows, $day, $run}
+        | {kind: "load.finished",
+           day: $day,
+           run: $run,
+           rows: ($rows | length),
+           loaded: ([$rows[] | select(.status == "loaded")] | length),
+           rejected: ([$rows[] | select(.status == "rejected")] | length),
+           # The keys, so the receiver can look them up; not a count of them, which nobody
+           # can act on.
+           rejected_keys: [$rows[] | select(.status == "rejected") | .key]}
+        | . + {clean: (.rejected == 0),
+               severity: (if .rejected == 0 then "info"
+                          elif .rejected < (.rows / 2) then "warning"
+                          else "error" end)}
+        # The sentence is derived from the counts, last, so it cannot contradict them.
+        | . + {text: "\(.day): \(.loaded)/\(.rows) loaded, \(.rejected) rejected"}
+
+  notify:
+    block: webhook.post
+    depends_on: [summary]
+    # Sent whether the work above passed or failed: a notifier that only fires on success
+    # goes quiet exactly when somebody needs it.
+    rule: all_done
+    config:
+      connection: echo-webhook
+      path: /post
+      body: ${steps.summary.output.value}
+      headers:
+        # Routing the receiver can dispatch on without parsing the body.
+        x-severity: ${steps.summary.output.value.severity}
+      # A receiver that queues rather than processes answers 202; both count as delivered.
+      success_status: [200, 202, 204]
+
+  delivered:
+    block: transform.jq
+    depends_on: [notify]
+    config:
+      input:
+        status: ${steps.notify.output.status}
+        signed: ${steps.notify.output.signed}
+        echoed: ${steps.notify.output.json_body}
+      program: |
+        {status, signed,
+         severity: .echoed.headers["x-severity"],
+         text: .echoed.json.text,
+         rejected_keys: .echoed.json.rejected_keys}

dirigent_examples/shelves/s3/README.md

@@ -0,0 +1,34 @@
+# S3 examples
+
+Object storage without an S3 block anywhere: `s3://` is a registered URI scheme, so the
+ordinary storage blocks address it the way they address anything else,
+and every document here declares `requires: {storage: [s3]}` so an instance without a
+backend refuses it before storing it.
+
+All five name one connection, `artifacts`, and it is the code the
+[compose stack](../../docs/operations.md) already bootstraps for its own artifact bucket. So
+on the stack there is nothing to set up: apply and run, naming the stack's bucket.
+
+```bash
+dg run s3-round-trip -p day=2026-01-01 -p bucket=dirigent --watch
+```
+
+Under `dg dev` there is no S3 server and no connection, and
+[s3-round-trip.yaml](s3-round-trip.yaml) documents both: the rustfs container to start, the
+`artifacts` connection to create, and the `DIRIGENT_STORAGE_CONNECTIONS` line that makes it
+serve the scheme.
+
+```bash
+dg apply examples/s3/s3-round-trip.yaml
+dg run s3-round-trip -p day=2026-01-01 --watch
+```
+
+## Pipelines
+
+| File | What it teaches |
+| --- | --- |
+| [s3-round-trip.yaml](s3-round-trip.yaml) | The whole seam: an artifact out to a bucket and back, moved by the ordinary storage blocks. |
+| [s3-copy-and-verify.yaml](s3-copy-and-verify.yaml) | Promote-then-consume: a copy between prefixes, and `storage.exists` standing between the copy and the reader. |
+| [s3-csv-report.yaml](s3-csv-report.yaml) | A report delivered to a bucket: the csv-report chain with the converter's `target` one word away from scratch. |
+| [report-to-s3.yaml](report-to-s3.yaml) | A rendered markdown page delivered to a bucket: `report.render` hands its text on and `storage.write` puts it in the object, content type and all. |
+| [s3-parquet-report.yaml](s3-parquet-report.yaml) | A typed parquet dataset written straight to a bucket, for an analysis to open (needs `dirigent-parquet`). |

dirigent_examples/shelves/s3/report-to-s3.yaml

@@ -0,0 +1,93 @@
+# The same rendered page as examples/recipes/report-to-file.yaml, delivered to a bucket.
+#
+# NEEDS AN S3-COMPATIBLE ENDPOINT, set up exactly as s3-round-trip.yaml documents: the same
+# rustfs container, the same artifacts connection, the same DIRIGENT_STORAGE_CONNECTIONS.
+#
+# Three hops, and what each one hands on:
+#   summary  transform.jq, the numbers the page is about.
+#   page     report.render, the markdown. Output: text, content_type, text_bytes.
+#   deliver  storage.write, the text to an s3:// URI. Output: uri, bytes_written,
+#            content_type -- and on S3 the content type is stored with the object, so a
+#            browser opening the link is told what it is.
+#
+# THE RENDER DOES NOT KNOW WHERE THE PAGE GOES. It renders text and hands it on; only the
+# step below it names a destination, and changing s3:// to file:// is the whole difference
+# between this document and the recipe it was copied from. That is what schemes are for.
+#
+#      dg apply examples/s3/report-to-s3.yaml
+#      dg run report-to-s3 -p day=2026-01-01 --watch
+#
+# The page is at s3://<bucket>/reports/<day>.md when the run settles.
+
+format: dirigent/v1
+kind: pipeline
+code: report-to-s3
+name: A rendered report delivered to a bucket
+description: Render a markdown page from a Jinja template and write it to an object in S3.
+
+tags: [s3, report, storage, transform]
+
+requires:
+  blocks:
+    - transform.jq
+    - report.render
+    - storage.write
+  connections:
+    - artifacts
+  storage:
+    - s3
+
+params:
+  type: object
+  required: [day]
+  additionalProperties: false
+  properties:
+    day:
+      type: string
+      format: date
+      description: The day the report is about, and the object it is named as.
+    bucket:
+      type: string
+      default: dirigent-archive
+      description: The bucket the page is delivered into.
+
+steps:
+  summary:
+    block: transform.jq
+    config:
+      input:
+        day: ${params.day}
+        readings:
+          - {station: st-1, region: east, celsius: 4.5}
+          - {station: st-3, region: west, celsius: 1.2}
+          - {station: st-4, region: north, celsius: -3.4}
+      program: |
+        {day: .day,
+         stations: (.readings | length),
+         coldest: (.readings | min_by(.celsius) | .station),
+         rows: .readings}
+
+  page:
+    block: report.render
+    depends_on: [summary]
+    config:
+      values:
+        summary: ${steps.summary.output.value}
+      content_type: text/markdown
+      template: |
+        # Readings for {{ summary.day }}
+
+        {{ summary.stations }} stations reported; the coldest was {{ summary.coldest }}.
+
+        {% for row in summary.rows %}
+        - **{{ row.station }}** ({{ row.region }}): {{ row.celsius }}C
+        {% endfor %}
+
+  deliver:
+    block: storage.write
+    depends_on: [page]
+    config:
+      target: s3://${params.bucket}/reports/${params.day}.md
+      text: ${steps.page.output.text}
+      # Stored on the object, so whoever fetches it is told it is markdown.
+      content_type: ${steps.page.output.content_type}