dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/s3/s3-copy-and-verify.yaml

@@ -0,0 +1,105 @@
+# Copy an object between prefixes, then prove it landed before anything reads it.
+#
+# NEEDS AN S3-COMPATIBLE ENDPOINT, set up exactly as s3-round-trip.yaml documents: the same
+# rustfs container, the same artifacts connection, the same DIRIGENT_STORAGE_CONNECTIONS.
+#
+# The shape is promote-then-consume: a copy from an incoming prefix to a published one,
+# storage.exists standing between the copy and the reader. The sensor is not decoration --
+# on eventually-consistent stores the object's arrival is a fact worth waiting on, and a
+# reader gated on it never sees the half-published state.
+#
+#      dg apply examples/s3/s3-copy-and-verify.yaml
+#      dg run s3-copy-and-verify -p day=2026-01-01 --watch
+#
+# The first two steps exist so the example carries its own input: a real pipeline starts at
+# seed, with the incoming object written by whatever produced it.
+#
+# Hop by hop:
+#
+#   payload   the readings, as a value in the run.
+#   staged    storage.write puts that value in the incoming prefix as json. A value reaches
+#             storage here and nowhere else.
+#   seed      convert.std re-encodes that object into the ndjson a producer would have
+#             dropped. A converter is a storage-object operation: it reads one URI and
+#             writes another, and the bytes never enter the run.
+#   promote   the copy from incoming to published.
+#   landed    the sensor, standing between the copy and the reader.
+#   read_back convert.std again, published ndjson back to json, so the result is readable.
+
+format: dirigent/v1
+kind: pipeline
+code: s3-copy-and-verify
+name: Copy and verify
+description: Promote an object to a published prefix, and prove it landed before reading it.
+
+tags: [s3, sensor, storage, transform]
+
+requires:
+  blocks:
+    - storage.write
+    - storage.copy
+    - storage.exists
+    - value.const
+    - convert.std
+  connections:
+    - artifacts
+  storage:
+    - s3
+
+params:
+  type: object
+  required: [day]
+  additionalProperties: false
+  properties:
+    day:
+      type: string
+      format: date
+      description: The day whose object is promoted.
+    bucket:
+      type: string
+      default: dirigent-archive
+      description: The bucket both prefixes live in.
+
+steps:
+  payload:
+    block: value.const
+    config:
+      value:
+        - { station: st-1, day: "${params.day}", celsius: 4.5 }
+        - { station: st-3, day: "${params.day}", celsius: 1.2 }
+  staged:
+    block: storage.write
+    depends_on: [payload]
+    config:
+      target: s3://${params.bucket}/incoming/${params.day}.json
+      value: "${steps.payload.output.value}"
+  seed:
+    # The example's own input: the incoming object a real producer would have written.
+    block: convert.std
+    depends_on: [staged]
+    config:
+      source: "${steps.staged.output.uri}"
+      target: s3://${params.bucket}/incoming/${params.day}.ndjson
+      from: json
+      to: ndjson
+  promote:
+    block: storage.copy
+    depends_on: [seed]
+    config:
+      source: s3://${params.bucket}/incoming/${params.day}.ndjson
+      target: s3://${params.bucket}/published/${params.day}.ndjson
+  landed:
+    block: storage.exists
+    depends_on: [promote]
+    poll: 5s
+    deadline: 2m
+    config:
+      uri: s3://${params.bucket}/published/${params.day}.ndjson
+  read_back:
+    block: convert.std
+    depends_on: [landed]
+    config:
+      source: "${steps.landed.output.uri}"
+      target: s3://${params.bucket}/published/${params.day}.json
+      from: ndjson
+      to: json

dirigent_examples/shelves/s3/s3-csv-report.yaml

@@ -0,0 +1,87 @@
+# A csv report delivered to a bucket, where the people who asked for it can fetch it.
+#
+# NEEDS AN S3-COMPATIBLE ENDPOINT, set up exactly as s3-round-trip.yaml documents: the same
+# rustfs container, the same artifacts connection, the same DIRIGENT_STORAGE_CONNECTIONS.
+#
+# This is examples/transform/csv-report.yaml with a real destination: the same records, the
+# same flattening, the same codec -- and s3:// instead of the run's scratch, so the file
+# outlives the run and lands where a person or another system reads it. Changing the
+# destination was one word, which is the point of schemes.
+#
+#      dg apply examples/s3/s3-csv-report.yaml
+#      dg run s3-csv-report -p day=2026-01-01 --watch
+#
+# The report is at s3://<bucket>/reports/<day>.csv when the run settles.
+#
+# Hop by hop:
+#
+#   readings  the records, as a value in the run.
+#   rows      jq flattens them into the shape a csv can hold. Still a value.
+#   staged    storage.write puts the rows in the bucket as json, because a converter works
+#             on objects rather than on values.
+#   report    convert.std re-encodes that object as the csv people fetch.
+
+format: dirigent/v1
+kind: pipeline
+code: s3-csv-report
+name: A report delivered to a bucket
+description: Shape records into flat rows and write them as a csv object in S3.
+
+tags: [s3, storage, transform]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+    - storage.write
+    - convert.std
+  connections:
+    - artifacts
+  storage:
+    - s3
+
+params:
+  type: object
+  required: [day]
+  additionalProperties: false
+  properties:
+    day:
+      type: string
+      format: date
+      description: The day the report is about, and the object it is named as.
+    bucket:
+      type: string
+      default: dirigent-archive
+      description: The bucket the report is delivered into.
+
+steps:
+  readings:
+    block: value.const
+    config:
+      value:
+        - { station: st-1, region: east, celsius: 4.5, tags: [ok, new] }
+        - { station: st-3, region: west, celsius: 1.2, tags: [ok] }
+        - { station: st-4, region: north, celsius: -3.4, tags: [] }
+  rows:
+    block: transform.jq
+    depends_on: [readings]
+    config:
+      input: ${steps.readings.output.value}
+      # Flatten for the codec: the tag list becomes one joined cell, because only this
+      # program knows what the separator should mean.
+      program: |
+        [.[] | {station, region, celsius, tags: (.tags | join(" "))}]
+  staged:
+    block: storage.write
+    depends_on: [rows]
+    config:
+      target: s3://${params.bucket}/reports/${params.day}.json
+      value: "${steps.rows.output.value}"
+  report:
+    block: convert.std
+    depends_on: [staged]
+    config:
+      source: "${steps.staged.output.uri}"
+      target: s3://${params.bucket}/reports/${params.day}.csv
+      from: json
+      to: csv

dirigent_examples/shelves/s3/s3-parquet-report.yaml

@@ -0,0 +1,77 @@
+# A parquet dataset delivered to a bucket, with its types along for the ride.
+#
+# NEEDS AN S3-COMPATIBLE ENDPOINT, set up exactly as s3-round-trip.yaml documents: the same
+# rustfs container, the same artifacts connection, the same DIRIGENT_STORAGE_CONNECTIONS.
+#
+# This is examples/transform/parquet-round-trip.yaml with a real destination. Parquet is
+# the format a downstream analysis actually wants -- columnar, typed, compact -- and what
+# lands in the bucket is a dataset a reader opens with pandas or duckdb, not a csv that
+# lost its numbers on the way out. convert.arrow ships in the dirigent-parquet pack;
+# installing it puts the block in the catalog.
+#
+#      dg apply examples/s3/s3-parquet-report.yaml
+#      dg run s3-parquet-report -p day=2026-01-01 --watch
+#
+# The dataset is at s3://<bucket>/datasets/<day>.parquet when the run settles.
+#
+# Hop by hop:
+#
+#   readings  the records, as a value in the run, flat as parquet columns need.
+#   staged    storage.write puts them in the bucket as json, which is the form a converter
+#             can read: convert.arrow works on objects, never on a value in the run.
+#   dataset   convert.arrow re-encodes that object as parquet, types and all.
+
+format: dirigent/v1
+kind: pipeline
+code: s3-parquet-report
+name: A parquet dataset in a bucket
+description: Write typed records as a parquet object in S3, where an analysis can read them.
+
+tags: [s3, storage, transform]
+
+requires:
+  blocks:
+    - value.const
+    - storage.write
+    - convert.arrow
+  connections:
+    - artifacts
+  storage:
+    - s3
+
+params:
+  type: object
+  required: [day]
+  additionalProperties: false
+  properties:
+    day:
+      type: string
+      format: date
+      description: The day the dataset covers, and the object it is named as.
+    bucket:
+      type: string
+      default: dirigent-archive
+      description: The bucket the dataset is delivered into.
+
+steps:
+  readings:
+    block: value.const
+    config:
+      value:
+        - { station: st-1, region: east, celsius: 4.5, active: true }
+        - { station: st-3, region: west, celsius: 1.2, active: true }
+        - { station: st-4, region: north, celsius: -3.4, active: false }
+  staged:
+    block: storage.write
+    depends_on: [readings]
+    config:
+      target: s3://${params.bucket}/datasets/${params.day}.json
+      value: "${steps.readings.output.value}"
+  dataset:
+    block: convert.arrow
+    depends_on: [staged]
+    config:
+      source: "${steps.staged.output.uri}"
+      target: s3://${params.bucket}/datasets/${params.day}.parquet
+      from: json
+      to: parquet

dirigent_examples/shelves/s3/s3-round-trip.yaml

@@ -0,0 +1,125 @@
+# Moving an artifact out to object storage and back, with no S3-specific block anywhere.
+#
+# NEEDS AN S3-COMPATIBLE ENDPOINT. dirigent-storage-s3 is installed and every block below
+# exists; what is missing is a credential and a server to point it at.
+#
+# ON THE COMPOSE STACK there is nothing to do: the stack runs an S3 server, keeps artifacts
+# in its bucket, and bootstraps the `artifacts` connection this document names. Apply and
+# run, passing the stack's own bucket:
+#
+#   dg run s3-round-trip -p day=2026-01-01 -p bucket=dirigent --watch
+#
+# The steps below are for `dg dev`, which has neither.
+#
+# A storage backend is not a step. There is no "upload to S3" block and no "download from
+# S3" block: s3:// is a registered URI scheme, so the ordinary storage.copy moves bytes
+# between file:// and s3:// in either direction, and the ordinary storage.exists waits on
+# an s3:// object. Changing where the archive lives is one word.
+#
+# 1. Start an S3-compatible server. Any implementation will do; this is the one dirigent's
+#    own storage lane runs against:
+#
+#      docker run -d --name dirigent-s3 -p 9000:9000 \
+#        -e RUSTFS_ACCESS_KEY=dirigent-test-key \
+#        -e RUSTFS_SECRET_KEY=dirigent-test-secret \
+#        rustfs/rustfs:1.0.0-rc.4
+#
+# 2. Create the connection, and tell the instance that it is what serves s3://. The first
+#    is the credential; the second is which of possibly several s3 connections the scheme
+#    is configured from:
+#
+#      dg connection create s3 artifacts \
+#        --set endpoint_url=http://127.0.0.1:9000 \
+#        --set access_key_id=dirigent-test-key \
+#        --set secret_access_key=dirigent-test-secret \
+#        --set path_style=true \
+#        --set bucket=dirigent-archive
+#
+#      export DIRIGENT_STORAGE_CONNECTIONS='{"s3": "artifacts"}'
+#
+# 3. Create the bucket the run addresses, then apply and run:
+#
+#      dg apply examples/s3/s3-round-trip.yaml
+#      dg run s3-round-trip -p day=2026-01-01 --watch
+#
+# The bucket always comes from the URI, never from the connection: the connection's own
+# bucket field is only what `dg connection check artifacts` probes.
+#
+# A real pipeline stops after the first copy. The download here makes the result checkable
+# without a second system to look at.
+
+format: dirigent/v1
+kind: pipeline
+code: s3-round-trip
+name: S3 round trip
+description: Move an artifact from local storage out to S3 and back, then confirm it landed.
+
+tags: [s3, http, sensor, storage]
+
+requires:
+  blocks:
+    - http.request
+    - storage.write
+    - storage.copy
+    - storage.exists
+  connections:
+    - artifacts
+  storage:
+    - s3
+
+params:
+  type: object
+  required: [day]
+  additionalProperties: false
+  properties:
+    day:
+      type: string
+      format: date
+      description: The day whose artifact is round-tripped.
+    bucket:
+      type: string
+      default: dirigent-archive
+      description: The bucket the artifact is archived into.
+
+steps:
+  produce:
+    block: http.request
+    config:
+      url: https://postman-echo.com/post
+      method: POST
+      body:
+        day: "${params.day}"
+
+  stage:
+    block: storage.write
+    depends_on: [produce]
+    # The local artifact the round trip is about, written on file:// because that is where
+    # ${run.scratch} lives. The copy below is the only line that mentions s3.
+    config:
+      target: "${run.scratch}/local/${params.day}.json"
+      value: "${steps.produce.output.body}"
+
+  upload:
+    block: storage.copy
+    depends_on: [stage]
+    config:
+      source: "${steps.stage.output.uri}"
+      target: "s3://${params.bucket}/daily/${params.day}.json"
+
+  wait_for_object:
+    block: storage.exists
+    depends_on: [upload]
+    # S3 acknowledges a write before every reader can see it, so the upload settling is not
+    # the same as the object being there. This waits for the second thing.
+    poll: 2s
+    deadline: 1m
+    config:
+      uri: "s3://${params.bucket}/daily/${params.day}.json"
+      min_size: 1
+
+  download:
+    block: storage.copy
+    depends_on: [wait_for_object]
+    config:
+      source: "${steps.wait_for_object.output.uri}"
+      target: "${run.scratch}/roundtrip/${params.day}.json"

dirigent_examples/shelves/schemas/README.md

@@ -0,0 +1,36 @@
+# Example schemas
+
+Each file here is a plain JSON Schema (Draft 2020-12): the shape a read is expected to return,
+written down once so a pipeline can be refused the moment a payload moves out from under it. A
+schema is **locally authored** -- a picture you hold of the payload, never something fetched or
+introspected from the source. Each is applied on its own with `dg schema create`, or with the
+directory it sits in, and a document that references one names it in `requires.schemas`. A document may instead **carry** a
+schema in its own top-level `schemas:` section for a standalone or `--local` run; a server
+refuses a document that carries one, so a shared instance holds its schemas here.
+
+Apply one the way a person would:
+
+```bash
+dg schema create examples/schemas/ou-record.json
+```
+
+A directory apply lands them the same way: a server pointed at a directory stores every schema
+it finds there before the pipelines, so a mounted corpus needs no separate step.
+
+A schema carries its own identity in its keywords, so there is nothing else to pass:
+
+- `$id` becomes the `code` the schema is addressed by (falling back to the file's stem).
+- `title` becomes its name.
+- `description` becomes its body.
+
+| File | The shape it pins |
+| --- | --- |
+| [ou-record.json](ou-record.json) | A single organisation-unit record: a string `id`, a `name`, and an integer `level` |
+| [echo-reading.json](echo-reading.json) | What a fetch of the echo service answers with: an `args` map carrying `station` and `day`, and the `url` |
+| [station-reading.json](station-reading.json) | The row a transform is held to before it is posted on: a `station`, a `day`, and a numeric `celsius` |
+
+`ou-record` is the shape the two [validation documents](../validate) check a payload against, and
+the two reading shapes are the gates either side of the transform in
+[http-fetch-validate-post.yaml](../recipes/http-fetch-validate-post.yaml).
+The [JSON Schema guide](../../docs/json-schema.md) walks through how a schema like it is
+built, keyword by keyword.

dirigent_examples/shelves/schemas/echo-reading.json

@@ -0,0 +1,18 @@
+{
+  "$id": "echo-reading",
+  "title": "Echoed reading batch",
+  "description": "What a fetch of the echo service is held to before anything reads into it: an `args` map carrying the query the call sent, and the `url` it was answered from. A source that changes shape under a pipeline is refused here rather than producing a wrong number three steps later.",
+  "type": "object",
+  "required": ["args", "url"],
+  "properties": {
+    "args": {
+      "type": "object",
+      "required": ["station", "day"],
+      "properties": {
+        "station": { "type": "string", "minLength": 1 },
+        "day": { "type": "string", "format": "date" }
+      }
+    },
+    "url": { "type": "string" }
+  }
+}

dirigent_examples/shelves/schemas/ou-record.json

@@ -0,0 +1,13 @@
+{
+  "$id": "ou-record",
+  "title": "Organisation unit record",
+  "description": "A single organisation-unit record: a string id, a name, and an integer level. The shape a payload is held to before a downstream step reads any of the three.",
+  "type": "object",
+  "required": ["id", "name", "level"],
+  "additionalProperties": false,
+  "properties": {
+    "id": { "type": "string" },
+    "name": { "type": "string" },
+    "level": { "type": "integer" }
+  }
+}

dirigent_examples/shelves/schemas/station-reading.json

@@ -0,0 +1,13 @@
+{
+  "$id": "station-reading",
+  "title": "Station reading",
+  "description": "The row a transform is held to before it is posted onward: a station, the day it reports for, and a temperature that is a number rather than the string a query carried. The gate a document puts after its own jq program, so a typo in the program is refused instead of delivered.",
+  "type": "object",
+  "required": ["station", "day", "celsius"],
+  "additionalProperties": false,
+  "properties": {
+    "station": { "type": "string", "minLength": 1 },
+    "day": { "type": "string", "format": "date" },
+    "celsius": { "type": "number" }
+  }
+}

dirigent_examples/shelves/sensors/README.md

@@ -0,0 +1,16 @@
+# Sensor examples
+
+Steps that wait for the world instead of doing something to it: each poke is one cheap,
+read-only question, the run holds no worker slot between pokes, and `deadline` with
+`on_timeout` says what a day without an answer means.
+
+```bash
+dg run --local examples/sensors/time-window.yaml --enable-unsafe shell.run
+```
+
+## Pipelines
+
+| File | What it teaches |
+| --- | --- |
+| [sensor-gate.yaml](sensor-gate.yaml) | `storage.exists` gating a load: wait for today's drop, and skip the day if it never lands. |
+| [time-window.yaml](time-window.yaml) | The clock as a condition: hold until the local time is inside a window, in the window's own zone. |

dirigent_examples/shelves/sensors/sensor-gate.yaml

@@ -0,0 +1,65 @@
+# A sensor gating the rest of the pipeline, and what happens when it never fires.
+#
+# storage.exists is a sensor: it observes the world and changes nothing. Each poke is a
+# durable, scheduled poll, so waiting hours costs hours of rows, not hours of a held worker.
+#
+# poll, deadline, and on_timeout are step-level engine semantics, uniform across every
+# block, never buried in the block's own config. on_timeout: skip is the load-bearing
+# choice here: no drop today is not a failure, it is a day with nothing to load, so the
+# sensor skips and the default all_success edges skip the rest of the branch with it.
+#
+# The deadline is short so the skip is watchable. Nothing ever writes the drop, so after ten
+# seconds the sensor skips, the branch behind it skips, and the run succeeds with nothing
+# done:
+#   dg run --local examples/sensors/sensor-gate.yaml -p day=2026-01-01 --enable-unsafe shell.run
+#
+# Note where the drop is looked for. file:// is rooted at the instance's artifact root and
+# refuses to address anything outside it, so a stored pipeline cannot turn a copy step into
+# an arbitrary-file read. Pointing this at a real drop means an s3:// URI, or a path under
+# that root -- never an absolute path from the document.
+#
+# confirm uses shell.run, which executes code on the worker and needs the allowlist:
+#   export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["shell.run"]'
+
+format: dirigent/v1
+kind: pipeline
+code: sensor-gate
+name: A sensor gates the run
+description: Wait for today's drop to land, then load it; skip the day if it never does.
+
+tags: [sensor, execute, storage]
+
+params:
+  type: object
+  required: [day]
+  properties:
+    day:
+      type: string
+      format: date
+
+steps:
+  wait_for_drop:
+    block: storage.exists
+    # Each poke is a scheduled row, not a held worker, so the poll interval is chosen for
+    # how soon the drop matters rather than for what it costs to wait.
+    poll: 2s
+    deadline: 10s
+    # No drop today is a day with nothing to load, not a failure. The skip carries down the
+    # branch through the default all_success edges, and the run still succeeds.
+    on_timeout: skip
+    config:
+      uri: "${run.scratch}/drops/${params.day}.parquet"
+      min_size: 1
+
+  load:
+    block: storage.copy
+    depends_on: [wait_for_drop]
+    config:
+      source: "${steps.wait_for_drop.output.uri}"
+      target: "${run.scratch}/loaded/${params.day}.parquet"
+
+  confirm:
+    block: shell.run
+    depends_on: [load]
+    config:
+      argv: [echo, "loaded ${steps.wait_for_drop.output.size} bytes for ${params.day}"]

dirigent_examples/shelves/sensors/time-window.yaml

@@ -0,0 +1,61 @@
+# Gating a step on the wall clock, in a timezone the document names out loud.
+#
+# A schedule says when a run starts. time.window says when a step may proceed, which is a
+# different question: "load whenever the export lands, but never touch the warehouse outside
+# the maintenance window" is a property of the step, not of the trigger.
+#
+# It is a sensor, so it inherits the whole waiting apparatus: poll, deadline, and on_timeout
+# are step-level engine semantics here exactly as they are for storage.exists. A closed
+# window parks until it opens rather than polling on the step's cadence, so waiting out a
+# Sunday costs a handful of pokes.
+#
+# The timezone defaults to UTC and should always be stated. Europe/Oslo below is read as
+# Oslo's clock, daylight saving included, no matter where the worker runs.
+#
+# The window here is wide open -- the whole day, every day -- so the example runs without
+# waiting. Narrow it to see the wait:
+#   dg run --local examples/sensors/time-window.yaml --enable-unsafe shell.run
+#
+# A real maintenance window is the commented shape below the open one:
+#   after: "01:00"
+#   before: "04:00"
+#   days: [sat, sun]
+#
+# Windows may cross midnight: after 22:00 before 04:00 is one night window, and when days is
+# set it names the day the window opens on, so [fri] runs Friday evening into Saturday.
+
+format: dirigent/v1
+kind: pipeline
+code: time-window
+name: Inside a time window
+description: Wait until the local clock is inside a window, then do the work it guards.
+
+tags: [sensor, execute, window]
+
+requires:
+  blocks:
+    - time.window
+    - shell.run
+
+steps:
+  maintenance_window:
+    block: time.window
+    poll: 5s
+    deadline: 1m
+    on_timeout: skip
+    config:
+      # Open all day so the example runs without waiting; a real window is narrow, and the
+      # commented shape in the header is what one looks like.
+      after: "00:00"
+      before: "23:59"
+      # Always state it. The default is UTC, and a window meant as local time silently
+      # becomes the wrong hours wherever the worker happens to run.
+      timezone: Europe/Oslo
+
+  reindex:
+    block: shell.run
+    depends_on: [maintenance_window]
+    config:
+      argv:
+        - echo
+        - "the window opened at ${steps.maintenance_window.output.entered_at} (${steps.maintenance_window.output.timezone})"

dirigent_examples/shelves/sql/README.md

@@ -0,0 +1,52 @@
+# SQL examples
+
+The `sql.*` family reads from and writes to a database. `sql.query` runs one statement and
+hands its rows on; `sql.execute` runs a list of statements as one transaction.
+[docs/sql.md](../../docs/sql.md) is the family's home.
+
+Both are **ordinary** blocks: they run no command a document supplies and reach nothing but the
+database their connection names, so no id has to be allowlisted to run them.
+
+```bash
+dg run --local examples/sql/sql-sqlite-roundtrip.yaml
+dg run --local examples/sql/sql-query-to-storage.yaml
+```
+
+Those two need nothing at all -- no network, no daemon, no server. Each builds a SQLite
+database in the run's own work directory, so the whole example is self-contained. The DuckDB one
+needs the engine (`dirigent-blocks[duckdb]`) and the parquet pack, and nothing else; the last
+names a real PostgreSQL and says so in its own header.
+
+```bash
+dg run --local examples/sql/duckdb-parquet-to-report.yaml --keep
+```
+
+The rule the whole family turns on: **a value is bound, never interpolated**. The statement is
+a constant in the document, every value is a named `:parameter`, and a `${...}` reference
+resolves into `params` and never into the SQL text. A parameter that reads as SQL is compared
+as a string and matches nothing.
+
+The first two documents carry their `sql` connection in a `connections:` section, because a
+`--local` run has no instance to hold one. On a server the connection is created once and the
+document names it, with the password set as its own sealed field:
+
+```bash
+dg connection create sql warehouse-read \
+  --set url=postgresql+asyncpg://reader@db.example:5432/warehouse \
+  --set password=... \
+  --set read_only=true
+```
+
+On the compose stack that database is the `infra/compose.sql.yaml` overlay
+(`make docker-run-sql`), seeded with the `reading` table the third document queries, and the
+connection names it as `reader@warehouse:5432/warehouse`.
+
+## Pipelines
+
+| File | What it teaches |
+| --- | --- |
+| [sql-sqlite-roundtrip.yaml](sql-sqlite-roundtrip.yaml) | The family end to end: a table created and filled in one transaction, read back with a bound parameter, and the rows referenced by a downstream step. |
+| [sql-query-to-storage.yaml](sql-query-to-storage.yaml) | Rows into a file: the query hands them on as its output, `storage.write` puts them in an object, and `max_rows` bounds what the step will carry. |
+| [duckdb-parquet-to-report.yaml](duckdb-parquet-to-report.yaml) | The engine that reads files: a parquet artifact queried by DuckDB through a bound `read_parquet(:source)`, and the answer copied out as a csv artifact. Needs the `dirigent-blocks[duckdb]` extra and the parquet pack. |
+| [sql-postgres-readonly.yaml](sql-postgres-readonly.yaml) | A referenced PostgreSQL connection with `read_only: true` and its password sealed separately, queried inside the run's window. Validates offline; it cannot run without an instance. |
+| [warehouse.sql](warehouse.sql) | Not a pipeline: the roles and the one table the compose stack's warehouse is seeded with, mounted by `infra/compose.sql.yaml` |