dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/sql/duckdb-parquet-to-report.yaml

@@ -0,0 +1,146 @@
+# SQL over files: DuckDB reads a parquet artifact and writes a csv one.
+#
+# NEEDS NOTHING BUT THE ENGINE: no network, no daemon, no server, no allowlist. It does need
+# duckdb on the worker, which is the dirigent-blocks[duckdb] extra, and convert.arrow, which
+# is the dirigent-parquet pack.
+#   dg run --local examples/sql/duckdb-parquet-to-report.yaml --keep
+# docs/sql.md is the family's home.
+#
+# RUNS WHEREVER THE ARTIFACTS LIVE. ${run.scratch} is a local directory under `dg dev` and a
+# bucket on the compose stack, and this document does not care: a file:// artifact becomes a
+# path duckdb opens, and an s3:// one is opened by duckdb's httpfs extension on the credentials
+# the s3 scheme is configured from. The image ships that extension; a bare install does
+# `python -c "import duckdb; duckdb.connect().execute('INSTALL httpfs')"` once.
+#
+# Five hops, and what each one hands on:
+#   readings    three records as a constant, standing in for whatever produced them.
+#   as_json     storage.write puts them in the run's scratch. Output: the uri they landed at.
+#   store       convert.arrow re-encodes that object as parquet. Output: the target uri.
+#   summarise   sql.query reads that parquet FILE and groups it. Output: the rows, inline.
+#   report      sql.execute writes the answer out again as a csv artifact.
+#
+# WHY THE WRITE IS ITS OWN HOP. convert.arrow works on storage objects the way storage.copy
+# does -- a source uri, a target uri, no value -- so records a step is holding become an
+# object through storage.write before a codec can touch them.
+#
+# THE ENGINE IS THE CONNECTION'S URL, AND NOTHING ELSE CHANGES. These are the same two blocks
+# the sqlite and postgres examples use, with the same fields and the same rules. A url naming
+# duckdb picks the engine that reads files; a url naming postgres picks the one that reads a
+# server. That is the whole of what "an engine of the family" means.
+#
+# WHY :memory: IS THE RIGHT DATABASE HERE. Nothing is stored in duckdb: the data lives in the
+# parquet file, and duckdb is the thing that reads it. An in-memory database is then a query
+# engine with no state of its own, opened and dropped inside each step. Name a duckdb file
+# instead when a pipeline wants tables that outlive one step.
+#
+# A FILE IS NAMED BY A BOUND PARAMETER, NEVER BY TEXT IN THE STATEMENT. read_parquet(:source)
+# takes the file the same way a WHERE clause takes a value, so the family's one rule holds
+# here too: ${...} resolves into params and never into the sql. The value is a storage URI,
+# and on a duckdb connection a file:// URI inside the run's own directories arrives as the path
+# duckdb opens, while an s3:// one is handed over whole for httpfs to open. A file:// URI
+# outside the run is refused; gs:// and azure:// still ask to be copied in first.
+#
+# BOTH DIRECTIONS. The read is read_parquet in summarise; the write is COPY ... TO :target in
+# report, which is how a query's answer becomes an artifact a later step, or a person,
+# collects. csv here because the report is meant to be read; write (FORMAT parquet) instead
+# and the output is a typed file the next pipeline reads back with read_parquet.
+#
+# WHY THE WRITE IS sql.execute AND NOT sql.query. COPY returns no rows -- it returns a count
+# -- and sql.query exists to hand rows on. sql.execute is the block that writes, which is also
+# why the connection it names is not read_only.
+#
+# TO MAKE IT YOURS: point store's target at a bucket, replace the constant with the step that
+# actually produces your rows, and grow the statement. Aggregation, joins across two parquet
+# files, a window function -- it is one query engine over files, so the answer to "how do I
+# join these" is a JOIN.
+
+format: dirigent/v1
+kind: pipeline
+code: duckdb-parquet-to-report
+name: Query a parquet file with DuckDB
+description: Write a parquet artifact, group it with DuckDB over the file itself, and copy the answer out as csv.
+
+tags: [sql, storage, transform, parquet]
+
+requires:
+  blocks:
+    - value.const
+    - storage.write
+    - convert.arrow
+    - sql.query
+    - sql.execute
+
+connections:
+  analysis:
+    kind: sql
+    config:
+      # No file: the tables this pipeline reads are the parquet files it names, so the
+      # database itself is empty and lives only as long as the step that opens it.
+      url: "duckdb:///:memory:"
+
+steps:
+  readings:
+    block: value.const
+    config:
+      value:
+        - { station: st-1, region: east, celsius: 4.5 }
+        - { station: st-2, region: east, celsius: 6.1 }
+        - { station: st-3, region: west, celsius: 1.2 }
+        - { station: st-4, region: north, celsius: -3.4 }
+
+  as_json:
+    block: storage.write
+    depends_on: [readings]
+    config:
+      target: ${run.scratch}/readings.json
+      value: ${steps.readings.output.value}
+
+  store:
+    block: convert.arrow
+    depends_on: [as_json]
+    config:
+      source: ${steps.as_json.output.uri}
+      # The parquet file the two SQL steps below read.
+      target: ${run.scratch}/readings.parquet
+      from: json
+      to: parquet
+
+  summarise:
+    block: sql.query
+    depends_on: [store]
+    config:
+      connection: analysis
+      sql: >
+        SELECT region, COUNT(*) AS stations, ROUND(AVG(celsius), 2) AS mean_celsius
+        FROM read_parquet(:source)
+        WHERE celsius >= :floor
+        GROUP BY region
+        ORDER BY region
+      params:
+        # The artifact the previous step wrote, as a uri. The block resolves it to the path
+        # duckdb opens, because this is a duckdb connection.
+        source: "${steps.store.output.target}"
+        # An ordinary bound value beside the file, to show they are the same mechanism.
+        floor: -10
+      max_rows: 100
+
+  report:
+    block: sql.execute
+    depends_on: [store]
+    config:
+      connection: analysis
+      statements:
+        # One statement, reading the parquet and writing the csv, because nothing is kept
+        # between two steps of an in-memory database.
+        - >
+          COPY (
+            SELECT region, COUNT(*) AS stations, ROUND(AVG(celsius), 2) AS mean_celsius
+            FROM read_parquet(:source)
+            GROUP BY region
+            ORDER BY region
+          ) TO :target (FORMAT csv, HEADER)
+      params:
+        source: "${steps.store.output.target}"
+        # The write side of the same rule: the target is a uri too, and the csv lands in the
+        # run's scratch space where storage.copy or a person can pick it up.
+        target: "${run.scratch}/regions.csv"

dirigent_examples/shelves/sql/sql-postgres-readonly.yaml

@@ -0,0 +1,111 @@
+# Reading a PostgreSQL warehouse through a connection that cannot write, whatever a step asks.
+#
+# NEEDS AN INSTANCE holding the connection, and a PostgreSQL it can reach. Unlike the other two
+# documents on this shelf, this one names its connection instead of carrying it, so there is
+# nothing here to run without a server:
+#   dg connection create sql warehouse-read \
+#     --set url=postgresql+asyncpg://reader@db.example:5432/warehouse \
+#     --set password=... \
+#     --set read_only=true
+#   dg connection check warehouse-read
+#   dg apply examples/sql/sql-postgres-readonly.yaml && dg run sql-postgres-readonly
+# On the compose stack that PostgreSQL is the infra/compose.sql.yaml overlay, seeded with this
+# table and a reader role, and the connection names it as `reader@warehouse:5432/warehouse`.
+# It is still a valid document without any of that: an instance that does not hold the
+# connection refuses it at apply, naming what is missing, rather than failing on the first run.
+# `dg apply --dry-run` is how to see that answer without writing anything.
+# docs/sql.md is the family's home.
+#
+# Two hops, and what each one hands on:
+#   recent  the rows for one site since the run's window opened, bound as parameters.
+#           Output: the rows, their count and the columns.
+#   report  a constant step reading them, standing in for whatever actually consumes them.
+#
+# THE PASSWORD IS SET SEPARATELY, AND THAT IS ENFORCED. A url written
+# postgresql+asyncpg://reader:secret@db.example/warehouse is REFUSED when the connection is
+# created: url is a plain field, so a password in it would sit unencrypted in the database and
+# come back out of the API. password is a sealed field -- encrypted at rest, redacted in every
+# response -- and the two are put together in memory at connect time and nowhere else.
+#
+# READ-ONLY IS THE DATABASE'S ANSWER, NOT A CHECK HERE. With read_only: true the connection
+# opens every session as SET TRANSACTION READ ONLY, so a statement that writes is refused by
+# PostgreSQL itself. sql.execute refuses this connection before it opens anything at all. The
+# pattern is one connection per role: this one for every reporting pipeline, a separate
+# warehouse-write for the few that load data.
+#
+# max_rows IS A PROMISE ABOUT THE OUTPUT. A step's output is stored with the run and read back
+# whole, so it is not the place for an unbounded result. The query is narrowed by both
+# parameters and then bounded again here; a result past the bound fails the step rather than
+# arriving truncated. sql-query-to-storage.yaml is what to do when the answer really is large.
+#
+# THE WINDOW IS WHY THIS IS SCHEDULABLE. ${run.window.start} is the start of the interval the
+# run covers, which a scheduled or backfilled run carries. Bound as a parameter it makes one
+# document work for today, for yesterday, and for a backfill of last March, with no edit.
+#
+# TO MAKE IT YOURS: point the connection at your warehouse, replace the query with your own,
+# and give the report step a real consumer -- a transform, a validate.schema, or an
+# http.request that posts the rows on.
+
+format: dirigent/v1
+kind: pipeline
+code: sql-postgres-readonly
+name: Read a warehouse through a read-only connection
+description: Query a PostgreSQL warehouse over a connection that refuses writes, with both values bound as parameters.
+
+tags: [sql, starter]
+
+requires:
+  blocks:
+    - sql.query
+    - value.const
+  connections:
+    # Named, not carried: the instance holds it, and this document refuses to apply where it
+    # is absent rather than failing the first time it runs.
+    - warehouse-read
+
+params:
+  type: object
+  additionalProperties: false
+  properties:
+    site:
+      type: string
+      default: north
+      description: The site to report on, bound as a parameter rather than spliced into the SQL.
+
+steps:
+  recent:
+    block: sql.query
+    deadline: 5m
+    config:
+      connection: warehouse-read
+      # Both filters are placeholders. Nothing a parameter contains can change what this
+      # statement does: the text is fixed when the document is applied.
+      sql: >-
+        SELECT id, site, seen, value
+        FROM reading
+        WHERE site = :site AND seen >= CAST(CAST(:since AS text) AS timestamptz)
+        ORDER BY seen
+      params:
+        site: "${params.site}"
+        # The interval this run covers. A scheduled run carries one; an ad hoc run carries one
+        # only if it was started with --window, and a step reading it otherwise fails rather
+        # than quietly widening to everything.
+        since: "${run.window.start}"
+      # The cast in the statement is what PostgreSQL needs: asyncpg types a parameter by where
+      # it sits, so a JSON string bound straight into a timestamp column is refused by the
+      # driver. Casting through text lets the server do the conversion it knows.
+      max_rows: 500
+      # Longer than a fast query needs and shorter than a runaway one takes. On PostgreSQL it
+      # is also set as the session's statement_timeout, so the server cancels the query rather
+      # than leaving it running after the worker stopped waiting.
+      timeout: 2m
+
+  report:
+    block: value.const
+    depends_on: [recent]
+    config:
+      value:
+        rows: "${steps.recent.output.rows}"
+        row_count: "${steps.recent.output.row_count}"
+        # How long the database took, which is the number worth watching as a table grows.
+        duration_ms: "${steps.recent.output.duration_ms}"

dirigent_examples/shelves/sql/sql-query-to-storage.yaml

@@ -0,0 +1,85 @@
+# A query's rows, and the hop that puts them in a file.
+#
+# NEEDS NOTHING: no network, no daemon, no server, no allowlist. The database is a SQLite file
+# in the run's own work directory, and the rows land in the run's scratch space.
+#   dg run --local examples/sql/sql-query-to-storage.yaml --keep
+# docs/sql.md is the family's home.
+#
+# Three hops, and what each one hands on:
+#   build     creates the table and inserts three rows. Output: the row counts.
+#   extract   selects them. Output: rows, row_count and columns.
+#   save      storage.write puts those rows in the run's scratch space as json. Output: the
+#             uri they landed at, how many bytes went there, and what the object is.
+#
+# A QUERY HANDS ROWS ON, AND NOTHING ELSE. sql.query carries its rows in the step's output,
+# where a transform maps them, a validate.schema checks them and a reference names them. A
+# file is one more hop, not a mode: storage.write is the only way a value leaves a run, so
+# the rows that belong in a file go to it, and the rows that belong to the next step stay in
+# the output.
+#
+# max_rows IS THE LINE, AND IT IS ABOUT MEMORY. An output is stored with the run and read back
+# whole, so a hundred thousand rows in one is a hundred thousand rows in the database and in
+# every read of that run. The block fails the step at that bound rather than truncating,
+# because half an answer is not a smaller answer -- and since the rows are carried either way,
+# writing them to a file does not raise it. A result too large to carry is one a query narrows,
+# with a GROUP BY, a LIMIT, or a WHERE the database evaluates instead of the worker.
+#
+# WHAT THE FILE IS. storage.write puts a value down as canonical JSON -- sorted keys, no
+# spaces -- so reading.json is one array of objects keyed by column name. A convert.std step
+# after this one re-spells it as ndjson or csv, and convert.arrow as parquet; each of those
+# reads the uri this step wrote.
+#
+# TO MAKE IT YOURS: point save's target at a bucket rather than scratch if the file should
+# outlive the run, and follow it with the step that consumes it -- a conversion, a
+# storage.copy to a landing bucket, or the load that reads it back somewhere else.
+
+format: dirigent/v1
+kind: pipeline
+code: sql-query-to-storage
+name: Write a query result to storage
+description: Read a table with sql.query and hand its rows to storage.write, which is how a result becomes a file.
+
+tags: [sql, storage]
+
+requires:
+  blocks:
+    - sql.execute
+    - sql.query
+    - storage.write
+
+connections:
+  work-db:
+    kind: sql
+    config:
+      # Relative, so the database lands in this run's work directory and needs nothing on the
+      # host. The same connection as the roundtrip example, for the same reason.
+      url: sqlite+aiosqlite:///demo.db
+
+steps:
+  build:
+    block: sql.execute
+    config:
+      connection: work-db
+      statements:
+        - CREATE TABLE reading (id INTEGER PRIMARY KEY, site TEXT NOT NULL, value REAL NOT NULL)
+        - INSERT INTO reading (id, site, value) VALUES (1, :north, 2.5), (2, :south, 4.0), (3, :north, 3.5)
+      params:
+        north: north
+        south: south
+
+  extract:
+    block: sql.query
+    depends_on: [build]
+    config:
+      connection: work-db
+      sql: SELECT id, site, value FROM reading ORDER BY id
+      max_rows: 100
+
+  save:
+    block: storage.write
+    depends_on: [extract]
+    config:
+      # A file inside the run's scratch space, so it is cleaned up with the run. A bucket URI
+      # here is what makes the result outlive it.
+      target: "${run.scratch}/reading.json"
+      value: "${steps.extract.output.rows}"

dirigent_examples/shelves/sql/sql-sqlite-roundtrip.yaml

@@ -0,0 +1,114 @@
+# A database built, written and read back inside one run, with every value bound.
+#
+# NEEDS NOTHING: no network, no daemon, no server, no allowlist. The database is a SQLite file
+# in the run's own work directory, and both sql blocks are ordinary.
+#   dg run --local examples/sql/sql-sqlite-roundtrip.yaml
+# docs/sql.md is the family's home.
+#
+# Three hops, and what each one hands on:
+#   build   creates the table and inserts two rows, all in one transaction. Output: how many
+#           rows each statement touched.
+#   read    selects those rows back with a bound parameter. Output: the rows, their count and
+#           the column names.
+#   report  a constant step that reads the rows, proving they are an ordinary reference any
+#           downstream step can name.
+#
+# THE CONNECTION HOLDS THE DATABASE, THE DOCUMENT HOLDS THE STATEMENTS. A step never writes a
+# URL: it names a sql connection, and the connection is what carries the database and the
+# sealed password that opens it. Moving a pipeline from staging to production is then an edit
+# to one connection and to no pipeline. This one is carried in the document because a --local
+# run has no instance to hold it; on a server it would be created once with
+# `dg connection create sql ...` and the document would name it and stop there.
+#
+# WHY THE URL IS RELATIVE. A sqlite database written with a relative path is resolved against
+# the run's work directory, so this run builds its database, uses it, and leaves it with
+# everything else the run wrote. That is what makes this example runnable anywhere with no
+# setup at all; a real pipeline names a database that already exists.
+#
+# BINDING IS THE POINT. Not one value is written into the SQL text. Every one is a named
+# parameter -- :site, :value -- and the parameters travel to the database beside the statement,
+# so a value that reads as SQL is compared as a string and matches nothing. ${...} resolves
+# into params, never into sql, and that is the whole rule.
+#
+# ONE TRANSACTION. sql.execute takes a list, and the list is one transaction: the CREATE and
+# both INSERTs commit together or not at all. A DDL statement reports -1 because SQLite does
+# not say how many rows a CREATE touched, which is what -1 means everywhere in row_counts.
+#
+# TO MAKE IT YOURS: point the connection at a real database, drop the build step, and keep the
+# read step -- reading a table with bound parameters is what most pipelines actually want.
+
+format: dirigent/v1
+kind: pipeline
+code: sql-sqlite-roundtrip
+name: Write a table and read it back
+description: Create a SQLite database in the run's work directory, insert rows with bound parameters, and read them back.
+
+tags: [sql]
+
+requires:
+  blocks:
+    - sql.execute
+    - sql.query
+    - value.const
+
+connections:
+  work-db:
+    kind: sql
+    config:
+      # Relative, so it lands in this run's work directory and needs nothing on the host. The
+      # driver is written out because these blocks speak to a database over an async one, and
+      # a url without one is refused when the connection is written.
+      url: sqlite+aiosqlite:///demo.db
+
+params:
+  type: object
+  additionalProperties: false
+  properties:
+    site:
+      type: string
+      default: north
+      description: The site the read step asks for, bound as a parameter rather than spliced into the SQL.
+
+steps:
+  build:
+    block: sql.execute
+    config:
+      connection: work-db
+      statements:
+        - CREATE TABLE reading (id INTEGER PRIMARY KEY, site TEXT NOT NULL, value REAL NOT NULL)
+        # Both inserts share one params map, which is why they name different columns of it
+        # rather than repeating a value.
+        - INSERT INTO reading (id, site, value) VALUES (1, :north, :north_value)
+        - INSERT INTO reading (id, site, value) VALUES (2, :south, :south_value)
+      params:
+        north: north
+        north_value: 2.5
+        south: south
+        south_value: 4.0
+
+  read:
+    block: sql.query
+    depends_on: [build]
+    config:
+      connection: work-db
+      # One statement. A document that needs two writes two steps, or uses sql.execute.
+      sql: SELECT id, site, value FROM reading WHERE site = :site ORDER BY id
+      params:
+        # The pipeline parameter reaches the database as a bound value and never as text in
+        # the statement above, which is what makes it safe to let somebody else set it.
+        site: "${params.site}"
+      # Two rows fit comfortably; a query that could return more than this fails the step
+      # rather than truncating, so a wider result is one to narrow or to raise this for.
+      max_rows: 10
+
+  report:
+    block: value.const
+    depends_on: [read]
+    config:
+      value:
+        # A list of objects keyed by column name, which is the shape every downstream step
+        # sees: a transform maps it, a validate.schema checks it, an http.request posts it.
+        rows: "${steps.read.output.rows}"
+        # The count is there whether the rows were carried inline or saved to storage.
+        row_count: "${steps.read.output.row_count}"
+        columns: "${steps.read.output.columns}"

dirigent_examples/shelves/sql/warehouse.sql

@@ -0,0 +1,42 @@
+-- What the warehouse in infra/compose.sql.yaml holds: two roles and the table the sql
+-- examples on this shelf query. The stack mounts this file; nothing else reads it.
+--
+-- Run by the postgres image's entrypoint on first start, connected to the warehouse database
+-- as its superuser. \getenv reads each role's password from the environment, so no password
+-- is written here.
+
+\getenv reader_password WAREHOUSE_READER_PASSWORD
+\getenv writer_password WAREHOUSE_WRITER_PASSWORD
+
+-- The shape examples/sql/sql-postgres-readonly.yaml selects: one row per reading, narrowed by
+-- site and by the run's window.
+CREATE TABLE reading (
+    id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    site text NOT NULL,
+    seen timestamptz NOT NULL,
+    value double precision NOT NULL
+);
+
+-- Seeded relative to now, so a run whose window is the last hour or the last day finds rows
+-- however long ago the volume was created.
+INSERT INTO reading (site, seen, value) VALUES
+    ('north', now() - interval '3 hours', 11.5),
+    ('north', now() - interval '2 hours', 12.25),
+    ('north', now() - interval '1 hour', 10.75),
+    ('north', now() - interval '20 minutes', 13.0),
+    ('south', now() - interval '90 minutes', 8.5),
+    ('south', now() - interval '30 minutes', 9.25);
+
+-- SELECT and nothing else. A statement that writes through this role is refused by the
+-- database itself, whatever the connection or the document asks for.
+CREATE ROLE reader LOGIN PASSWORD :'reader_password';
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO reader;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO reader;
+
+-- The role a loading pipeline uses: the four statements, and the sequence an identity column
+-- draws from.
+CREATE ROLE writer LOGIN PASSWORD :'writer_password';
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO writer;
+GRANT USAGE ON ALL SEQUENCES IN SCHEMA public TO writer;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO writer;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE ON SEQUENCES TO writer;

dirigent_examples/shelves/transform/README.md

@@ -0,0 +1,36 @@
+# Transform examples
+
+These pipelines reshape data with the transform verbs -- `transform`, `map`, `filter` and
+`convert` -- which run a program or a codec against a value and execute nothing on the
+worker. None of them needs an allowlist entry, a network, or anything installed first.
+[docs/transforms.md](../../docs/transforms.md) is the page behind them: a verb is a contract
+with a promise its frame enforces, and a kind is the engine that keeps it.
+
+A file that stars an engine is prefixed with its kind, the way a block id's
+`<verb>.<kind>` names it: `jq-` for the jq engines, `std-` for the `convert.std` codec. The
+rest are named for the format they round-trip.
+
+```bash
+dg run --local examples/transform/jq-reshape.yaml
+```
+
+## Pipelines
+
+| File | What it teaches |
+| --- | --- |
+| [jq-reshape.yaml](jq-reshape.yaml) | The whole-value reshape: one jq program between steps, then a fan-out that reads its list once per element. |
+| [jq-filter-and-map.yaml](jq-filter-and-map.yaml) | The two element-wise verbs against the whole-value one: a subset, a list of the same length, and a reshape. |
+| [jq-group-and-aggregate.yaml](jq-group-and-aggregate.yaml) | `group_by` and arithmetic: per-group sums and means with a total beside them. |
+| [jq-join-two-sources.yaml](jq-join-two-sources.yaml) | Two upstream outputs composed into one inline input, joined with `INDEX`, unmatched rows kept with a null name. |
+| [jq-stream-through-storage.yaml](jq-stream-through-storage.yaml) | The two doors on storage: `storage.write` puts a value in an object and `storage.read` brings one back, with the reshape between them. |
+| [std-convert-fan-out.yaml](std-convert-fan-out.yaml) | The codec: csv to json, reshaped with jq, fanned out over the regions, and written back as csv. |
+| [csv-report.yaml](csv-report.yaml) | Records shaped into flat rows and written as a csv artifact, with nothing on the allowlist. |
+| [ndjson-round-trip.yaml](ndjson-round-trip.yaml) | ndjson: a JSON array re-spelled one record per line, and read back. |
+| [yaml-config-to-json.yaml](yaml-config-to-json.yaml) | yaml: one document is one value, so a config becomes the object it describes, and comes back a document. |
+| [xml-feed-to-ndjson.yaml](xml-feed-to-ndjson.yaml) | xml: a feed's elements as one record per line, the mapping that makes attributes and children keys, and the whole document as one object. |
+| [parquet-round-trip.yaml](parquet-round-trip.yaml) | Records to parquet and back, the types surviving where csv would flatten them to strings (needs `dirigent-parquet`). |
+
+Every program on this shelf is reference-free, so jq compiles them when the document is
+applied and a syntax error is an issue beside every other one the document has. That is why
+the data is always the step's `input` and never spliced into the program: a config carrying
+a `${...}` cannot be checked until the run.

dirigent_examples/shelves/transform/csv-report.yaml

@@ -0,0 +1,55 @@
+# A csv report, written where a person can fetch it.
+#
+# Four hops turn records into a file: a fixed value stands in for whatever produced the
+# records, a jq program shapes them into flat rows, storage.write puts those rows in the
+# run's scratch as json, and convert.std re-encodes that object as csv -- so the run's
+# Output tab lists a report.csv somebody can read back. Nothing here is on the allowlist.
+#
+# convert.std works on storage objects, the way storage.copy does: it names a source uri
+# and a target uri and never carries a value. The write is how the rows a step is holding
+# become an object it can read.
+#
+# The rows must be flat: a nested value has no csv spelling and is refused naming the row
+# and the key, so the jq step is where a list becomes one joined cell.
+#
+#   dg run --local examples/transform/csv-report.yaml --keep
+
+format: dirigent/v1
+kind: pipeline
+code: csv-report
+name: A report as csv
+description: Shape records into flat rows, store them as json, and re-encode that as a csv artifact.
+
+tags: [transform, storage]
+
+steps:
+  readings:
+    block: value.const
+    config:
+      value:
+        - { station: st-1, region: east, celsius: 4.5, tags: [ok, new] }
+        - { station: st-3, region: west, celsius: 1.2, tags: [ok] }
+        - { station: st-4, region: north, celsius: -3.4, tags: [] }
+  rows:
+    block: transform.jq
+    depends_on: [readings]
+    config:
+      input: ${steps.readings.output.value}
+      # Flatten for the codec: the tag list becomes one joined cell, because only this
+      # program knows what the separator should mean.
+      program: |
+        [.[] | {station, region, celsius, tags: (.tags | join(" "))}]
+  store:
+    block: storage.write
+    depends_on: [rows]
+    config:
+      target: ${run.scratch}/rows.json
+      value: ${steps.rows.output.value}
+  report:
+    block: convert.std
+    depends_on: [store]
+    config:
+      source: ${steps.store.output.uri}
+      target: ${run.scratch}/report.csv
+      from: json
+      to: csv

dirigent_examples/shelves/transform/jq-filter-and-map.yaml

@@ -0,0 +1,70 @@
+# The two constrained verbs, and what they buy over doing everything in one program.
+#
+# filter.jq and map.jq run a jq program once per element of a list. Neither can do what the
+# other does: a filter's program answers true or false and the frame keeps the element it
+# was given, so a filtered list is a subset with nothing modified, and a map's program
+# returns one replacement per element, so a mapped list is exactly as long as its input.
+# Written as one transform.jq program the same work is one line -- and nothing but reading
+# it tells you whether it dropped rows, added rows, or edited them in place.
+#
+# The steps read: keep the active readings, convert each one to fahrenheit, then summarise.
+# The third step is a transform because it genuinely is one: it changes the shape of the
+# whole value rather than working element by element, and that is the verb that says so.
+#
+# The per-element contracts are strict on purpose. A map program emitting nothing, or two
+# things, for one element is refused rather than quietly changing the length, and a filter
+# program answering 0 or "" is refused rather than read as false: jq's truthiness is not
+# applied, so a program meaning "has readings" writes .count > 0.
+#
+# The input is written inline so the example needs no network. In a real pipeline it is a
+# reference to an upstream step's output, which is how a value that lives in storage arrives:
+# through a storage.read of its own.
+#
+#   dg run --local examples/transform/jq-filter-and-map.yaml
+
+format: dirigent/v1
+kind: pipeline
+code: jq-filter-and-map
+name: Filter and map with jq
+description: Keep the active readings with filter.jq, convert each with map.jq, and summarise them.
+
+tags: [transform]
+
+requires:
+  blocks:
+    - filter.jq
+    - map.jq
+    - transform.jq
+
+steps:
+  active:
+    block: filter.jq
+    config:
+      input:
+        - { station: st-1, region: east, status: active, celsius: 4.5 }
+        - { station: st-2, region: east, status: retired, celsius: 11.0 }
+        - { station: st-3, region: west, status: active, celsius: 1.2 }
+        - { station: st-4, region: north, status: active, celsius: -3.4 }
+      # The answer is the verdict and nothing else: the elements that come out are the ones
+      # that went in, unmodified, because the frame keeps them rather than the program.
+      program: |
+        .status == "active"
+
+  fahrenheit:
+    block: map.jq
+    depends_on: [active]
+    config:
+      input: "${steps.active.output.value}"
+      # One output per element, so this list is as long as the one above it.
+      program: |
+        {station, region, fahrenheit: (.celsius * 9 / 5 + 32 | round)}
+
+  summary:
+    block: transform.jq
+    depends_on: [fahrenheit]
+    config:
+      # A whole-value reshape rather than an element-wise one, which is why it is the third
+      # verb and not one of the other two.
+      input: "${steps.fahrenheit.output.value}"
+      program: |
+        {stations: length, regions: (map(.region) | unique), warmest: (max_by(.fahrenheit) | .station)}