dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/transform/jq-group-and-aggregate.yaml

@@ -0,0 +1,70 @@
+# The analytics reshape: a flat list in, one row per group out, and a total beside them.
+#
+# group_by is the whole of it. jq sorts the list by the key and hands back a list of lists,
+# one per distinct key, so an aggregate is `map(.rainfall_mm) | add` over each of those
+# lists and the group's key is `.[0].region`. The groups come out in sorted key order --
+# east, north, west -- because group_by sorts before it groups.
+#
+# The total is computed from the original list rather than by adding the group totals up.
+# Both are correct here; only one stays correct when the groups are filtered.
+#
+# The program contains no ${...} reference, so jq compiles it when the document is applied
+# and a typo is an issue at steps.by_region.config rather than a run that fails tonight. A
+# config that carries a reference cannot be compiled early, which is why the data is the
+# input and never spliced into the program.
+#
+# The readings are written inline so the example needs no network. In a real pipeline this is
+# a reference to an upstream step's output: an http.request's body, or a storage.read's value.
+#
+# What it prints, and the arithmetic to check it against:
+#
+#   east   2 stations, 12 + 8           = 20mm, mean 10
+#   north  3 stations, 30 + 10 + 5      = 45mm, mean 15
+#   west   1 station,  7                =  7mm, mean 7
+#   total  6 stations, 20 + 45 + 7      = 72mm, mean 12
+#
+#   dg run --local examples/transform/jq-group-and-aggregate.yaml
+
+format: dirigent/v1
+kind: pipeline
+code: jq-group-and-aggregate
+name: Group and aggregate with jq
+description: Group a flat list of readings by region, aggregate each group, and total them.
+
+tags: [transform]
+
+requires:
+  blocks:
+    - transform.jq
+
+steps:
+  by_region:
+    block: transform.jq
+    config:
+      input:
+        - { station: st-1, region: east, rainfall_mm: 12 }
+        - { station: st-2, region: east, rainfall_mm: 8 }
+        - { station: st-3, region: north, rainfall_mm: 30 }
+        - { station: st-4, region: north, rainfall_mm: 10 }
+        - { station: st-5, region: north, rainfall_mm: 5 }
+        - { station: st-6, region: west, rainfall_mm: 7 }
+      # One output, so the step's value is that one object.
+      program: |
+        . as $readings
+        | {
+            regions: [
+              $readings
+              | group_by(.region)[]
+              | {
+                  region: .[0].region,
+                  stations: length,
+                  total_mm: (map(.rainfall_mm) | add),
+                  mean_mm: (map(.rainfall_mm) | add / length)
+                }
+            ],
+            total: {
+              stations: ($readings | length),
+              total_mm: ($readings | map(.rainfall_mm) | add),
+              mean_mm: ($readings | map(.rainfall_mm) | add / length)
+            }
+          }

dirigent_examples/shelves/transform/jq-join-two-sources.yaml

@@ -0,0 +1,98 @@
+# Enriching one dataset from another: two upstream steps, one join, and no reference in
+# any of the three programs.
+#
+# The pattern is the third step's config. Two outputs are composed into a single inline
+# input, keyed by what they are:
+#
+#   input:
+#     stations: ${steps.stations.output.value}
+#     readings: ${steps.readings.output.value}
+#
+# so the program destructures .stations and .readings and never mentions a step. That keeps
+# the references in typed config, where the engine resolves them, and keeps the program a
+# constant string that jq compiles when the document is applied. A program with a ${...}
+# spliced into it could only be compiled on the worker, on the first run.
+#
+# INDEX(.id) turns the stations list into an object keyed by id, which is the lookup table:
+# one pass to build it, then constant-time reads, instead of scanning the list per reading.
+#
+# st-9 has no station record, and the join keeps its row with a null name rather than
+# dropping it -- an outer join, not an inner one. `// null` is what says so. jq already
+# yields null for a missing key, so the operator changes no output; it is written because a
+# reader should see the decision in the program rather than infer it from jq's semantics.
+# A pipeline that wants unmatched readings gone writes `select(...)` and means it.
+#
+# The two upstream programs are projections over inline parameter defaults, so the example
+# needs no network. In a real pipeline they are two http.request steps against two systems,
+# and this is the step that puts the two answers together.
+#
+# What it prints: three readings, st-1 and st-3 carrying a station_name and a region, st-9
+# carrying null for both.
+#
+#   dg run --local examples/transform/jq-join-two-sources.yaml
+
+format: dirigent/v1
+kind: pipeline
+code: jq-join-two-sources
+name: Join two sources with jq
+description: Enrich a list of readings with the station records they refer to, keeping unmatched rows.
+
+tags: [transform, graph]
+
+requires:
+  blocks:
+    - transform.jq
+
+params:
+  type: object
+  additionalProperties: false
+  properties:
+    stations:
+      type: array
+      description: The station records to enrich from.
+      items:
+        type: object
+      default:
+        - { id: st-1, name: Bo Central, region: east }
+        - { id: st-2, name: Kenema North, region: east }
+        - { id: st-3, name: Makeni West, region: west }
+    readings:
+      type: array
+      description: The readings to enrich, one of which names an unknown station.
+      items:
+        type: object
+      default:
+        - { station: st-1, celsius: 4.5 }
+        - { station: st-3, celsius: 1.2 }
+        - { station: st-9, celsius: -2.0 }
+
+steps:
+  stations:
+    block: transform.jq
+    config:
+      input: ${params.stations}
+      # A projection, standing in for whatever the station system actually returns.
+      program: |
+        [.[] | {id, name, region}]
+
+  readings:
+    block: transform.jq
+    config:
+      input: ${params.readings}
+      program: |
+        [.[] | {station, celsius}]
+
+  join:
+    block: transform.jq
+    depends_on: [stations, readings]
+    config:
+      # The two outputs, composed into one value under names the program knows.
+      input:
+        stations: ${steps.stations.output.value}
+        readings: ${steps.readings.output.value}
+      program: |
+        . as {$stations, $readings}
+        | ($stations | INDEX(.id)) as $by_id
+        | [$readings[]
+           | . + {station_name: ($by_id[.station].name // null),
+                  region: ($by_id[.station].region // null)}]

dirigent_examples/shelves/transform/jq-reshape.yaml

@@ -0,0 +1,91 @@
+# Reshaping data between two steps, with no subprocess and no allowlist entry.
+#
+# transform.jq runs a jq program over one whole value. The program is compiled when the
+# document is applied, so a syntax error is an issue at steps.<name>.config beside every
+# other one the document has, rather than a run that fails at three in the morning. jq opens
+# no file and no socket, so the engine executes nothing on the worker: unlike every example
+# that reaches for shell.run to do this, this one needs no --enable-unsafe and no network.
+#
+# The shape is the one a reshape usually has. reshape pulls the active readings out of a
+# payload and flattens them; per_region is mapped over the regions and reads that list once
+# per element; summary reads the whole fan-out back as one value. Note where the per-item
+# value goes: the item is data in config.input, not text spliced into the program, so the
+# program stays a program and the data stays data.
+#
+# A jq program is a stream of outputs, and the step's output follows it: one output is the
+# value, several are the list of them, and none at all is refused, because a step has to
+# produce something for the next one to read.
+#
+# The input is written inline so the example needs no network. In a real pipeline it is a
+# reference to an upstream step's output: an http.request's body, or the value a storage.read
+# brought in. The result travels the same way, and reaches a file through a storage.write.
+#
+#   dg run --local examples/transform/jq-reshape.yaml
+#   dg run --local examples/transform/jq-reshape.yaml -p regions='["east"]'
+
+format: dirigent/v1
+kind: pipeline
+code: jq-reshape
+name: Reshape with jq
+description: Reshape a reading payload with jq, then map the result over one region each.
+
+tags: [transform]
+
+requires:
+  blocks:
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    regions:
+      type: array
+      default: [east, west, north]
+      items:
+        type: string
+
+steps:
+  reshape:
+    block: transform.jq
+    config:
+      input:
+        generated_at: "2026-01-01T06:00:00Z"
+        readings:
+          - { station: st-1, region: east, status: active, celsius: 4 }
+          - { station: st-2, region: east, status: retired, celsius: null }
+          - { station: st-3, region: west, status: active, celsius: 1.2 }
+          - { station: st-4, region: north, status: active, celsius: -3.4 }
+          # A second active east reading, so mean_celsius is (4 + 8) / 2 = 6, an average
+          # of two rather than a passthrough of one.
+          - { station: st-5, region: east, status: active, celsius: 8 }
+      # One output, so the step's output value is that one array.
+      program: |
+        [.readings[] | select(.status == "active") | {station, region, celsius}]
+
+  per_region:
+    block: transform.jq
+    depends_on: [reshape]
+    # for_each is expanded when the run is created, so it reads params and not a step's
+    # output. The reshaped list is read in config, where an output reference belongs.
+    for_each: "${params.regions}"
+    config:
+      input:
+        region: "${item}"
+        rows: "${steps.reshape.output.value}"
+      program: |
+        . as {$region, $rows}
+        | {region: $region,
+           stations: [$rows[] | select(.region == $region) | .station],
+           mean_celsius: ([$rows[] | select(.region == $region) | .celsius]
+             | if length > 0 then add / length else null end)}
+
+  summary:
+    block: transform.jq
+    depends_on: [per_region]
+    config:
+      # A fan-out step's output is the list of its items' outputs, in item order, and a
+      # transform's output is an object carrying value, so the list is unwrapped once here.
+      input: "${steps.per_region.output}"
+      program: |
+        [.[] | .value | select(.stations | length > 0)]
+        | {regions_with_data: length, stations: (map(.stations) | flatten | sort)}

dirigent_examples/shelves/transform/jq-stream-through-storage.yaml

@@ -0,0 +1,112 @@
+# A value leaving the run and coming back: storage.write out, storage.read in.
+#
+# A transform works on a value and answers with one, so a list that should survive as a
+# file leaves through storage.write and returns through storage.read. Those two blocks are
+# the only doors, and this document walks through both of them: generate builds a list,
+# store puts it in the run's scratch space, reload reads it back as a value, cold keeps the
+# readings below zero, keep writes those out as their own file, and summary folds what cold
+# produced down to the handful of fields the run answers with.
+#
+# What each hop hands the next:
+#
+#   - a transform's output is value, which is what the next transform's input reads
+#   - a write's output is uri, bytes_written and content_type, so the read that follows
+#     names ${steps.store.output.uri} rather than repeating the path
+#   - a read's output is value for the json family and text otherwise, decided by what the
+#     object says it is; readings.json says application/json by its extension
+#
+# A read holds the object whole, so max_size bounds it and an object past that is refused
+# rather than truncated. Bytes nobody has to look at move with storage.copy instead.
+#
+# The list is built in jq from range, so the example needs no network and no fixture file.
+# In a real pipeline generate is an http.request and its body goes straight to store.
+#
+# What it prints, and the arithmetic to check it against: celsius is (i % 40) - 10, so ten
+# of every forty stations are below zero, and 2000 stations give 500 cold ones. The coldest
+# is st-0 at -10, and the three regions all appear among them.
+#
+#   dg run --local examples/transform/jq-stream-through-storage.yaml
+#   dg run --local examples/transform/jq-stream-through-storage.yaml -p stations=200
+
+format: dirigent/v1
+kind: pipeline
+code: jq-stream-through-storage
+name: A value through storage
+description: Write a generated list to storage, read it back as a value, and write the smaller result out the same way.
+
+tags: [transform, storage]
+
+requires:
+  blocks:
+    - transform.jq
+    - storage.read
+    - storage.write
+
+params:
+  type: object
+  additionalProperties: false
+  properties:
+    stations:
+      type: integer
+      minimum: 1
+      maximum: 100000
+      default: 2000
+      description: How many station readings to generate.
+
+steps:
+  generate:
+    block: transform.jq
+    config:
+      input:
+        count: ${params.stations}
+      # About 100KB at the default count.
+      program: |
+        [range(.count) as $i
+         | {station: "st-\($i)",
+            region: ["east", "west", "north"][$i % 3],
+            celsius: ($i % 40) - 10}]
+
+  store:
+    block: storage.write
+    depends_on: [generate]
+    config:
+      target: ${run.scratch}/readings.json
+      value: ${steps.generate.output.value}
+
+  reload:
+    block: storage.read
+    depends_on: [store]
+    config:
+      source: ${steps.store.output.uri}
+      # The default is 1mb, and the generated list is well under it at every station count
+      # this document allows.
+      max_size: 8mb
+
+  cold:
+    block: transform.jq
+    depends_on: [reload]
+    config:
+      # readings.json is application/json, so the read decoded it and the value is the list
+      # itself rather than its text.
+      input: ${steps.reload.output.value}
+      program: |
+        [.[] | select(.celsius < 0) | {station, region, celsius}]
+
+  keep:
+    block: storage.write
+    depends_on: [cold]
+    config:
+      target: ${run.scratch}/cold.json
+      value: ${steps.cold.output.value}
+
+  summary:
+    block: transform.jq
+    depends_on: [cold]
+    config:
+      # No second read: the cold list is already a value in this run, and a write is for
+      # what leaves it. This output is small, so it inlines, and it is the run's answer.
+      input: ${steps.cold.output.value}
+      program: |
+        {cold_stations: length,
+         coldest: (min_by(.celsius) | .station),
+         regions: (map(.region) | unique)}

dirigent_examples/shelves/transform/ndjson-round-trip.yaml

@@ -0,0 +1,56 @@
+# One record per line, which is what a consumer that streams wants.
+#
+# ndjson is the spelling for a sequence that is appended to or read without parsing the
+# whole: each line is one complete JSON record. This writes an array to storage, converts
+# that object to ndjson and back, and reads the result in again -- and the round trip is
+# the point, because the three formats say the same thing, so converting is re-spelling,
+# never reshaping.
+#
+# A conversion is a storage-object operation: source uri in, target uri out, no value in
+# between. storage.write is what turns the records this run holds into an object to convert,
+# and storage.read is what brings the last one back as a value the run can show.
+#
+#   dg run --local examples/transform/ndjson-round-trip.yaml --keep
+
+format: dirigent/v1
+kind: pipeline
+code: ndjson-round-trip
+name: One record per line
+description: Re-spell a stored JSON array as ndjson, convert it back, and read the result in as a value.
+
+tags: [transform, storage]
+
+steps:
+  records:
+    block: value.const
+    config:
+      value:
+        - { station: st-1, celsius: 4.5 }
+        - { station: st-3, celsius: 1.2 }
+  store:
+    block: storage.write
+    depends_on: [records]
+    config:
+      target: ${run.scratch}/records.json
+      value: ${steps.records.output.value}
+  as_lines:
+    block: convert.std
+    depends_on: [store]
+    config:
+      source: ${steps.store.output.uri}
+      target: ${run.scratch}/records.ndjson
+      from: json
+      to: ndjson
+  back:
+    block: convert.std
+    depends_on: [as_lines]
+    config:
+      source: ${steps.as_lines.output.target}
+      target: ${run.scratch}/round-trip.json
+      from: ndjson
+      to: json
+  read_back:
+    block: storage.read
+    depends_on: [back]
+    config:
+      source: ${steps.back.output.target}

dirigent_examples/shelves/transform/parquet-round-trip.yaml

@@ -0,0 +1,68 @@
+# Records into parquet and back, where the types survive the trip.
+#
+# Parquet is the typed spelling at rest: where csv turns every value into a string, a
+# parquet column knows it holds an integer or a float, and convert.arrow infers that schema
+# from the records as it writes. This stores a batch as json, converts it to parquet, reads
+# that file back into json, and loads it as a value -- and the numbers come back numbers
+# rather than strings.
+#
+# Every convert step here names two uris and carries nothing: parquet is bytes, and a
+# conversion is a storage-object operation either way. storage.write is the hop that puts
+# the records in storage to begin with, and storage.read the hop that brings the result
+# back where a later step, or a person reading the run, can see it.
+#
+# convert.arrow ships in the dirigent-parquet pack; installing it is what puts the block in
+# the catalog.
+#
+#   dg run --local examples/transform/parquet-round-trip.yaml --keep
+
+format: dirigent/v1
+kind: pipeline
+code: parquet-round-trip
+name: A parquet round trip
+description: Write records to a parquet file, then read them back with their types intact.
+
+tags: [transform, storage]
+
+requires:
+  blocks:
+    - value.const
+    - storage.read
+    - storage.write
+    - convert.arrow
+
+steps:
+  readings:
+    block: value.const
+    config:
+      value:
+        - { station: st-1, region: east, celsius: 4.5, active: true }
+        - { station: st-3, region: west, celsius: 1.2, active: true }
+        - { station: st-4, region: north, celsius: -3.4, active: false }
+  store:
+    block: storage.write
+    depends_on: [readings]
+    config:
+      target: ${run.scratch}/readings.json
+      value: ${steps.readings.output.value}
+  write:
+    block: convert.arrow
+    depends_on: [store]
+    config:
+      source: ${steps.store.output.uri}
+      target: ${run.scratch}/readings.parquet
+      from: json
+      to: parquet
+  read_back:
+    block: convert.arrow
+    depends_on: [write]
+    config:
+      source: ${steps.write.output.target}
+      target: ${run.scratch}/round-trip.json
+      from: parquet
+      to: json
+  load:
+    block: storage.read
+    depends_on: [read_back]
+    config:
+      source: ${steps.read_back.output.target}

dirigent_examples/shelves/transform/std-convert-fan-out.yaml

@@ -0,0 +1,142 @@
+# A csv re-encoded once and then reshaped, with no subprocess and no allowlist entry.
+#
+# convert.std is a codec, not a language: it has no program, only a source, a target, a
+# from and a to, and the pair is checked when the document is applied. It converts between
+# json, ndjson and csv, and everything a csv carries is text, so every cell it reads out is
+# a string. That is the first thing this document has to deal with: celsius arrives as
+# "4.5", and the jq program says tonumber where it wants a number. A codec that guessed
+# would not be a codec.
+#
+# The shape is csv in, csv out, and storage is what the two blocks meet over. source writes
+# the inline csv as an object; parse re-encodes it as json; rows reads that json in as a
+# value; active reshapes it with jq; per_region is mapped over the regions; report gathers
+# the fan-out; store puts that array back in storage, and as_csv converts it to csv again.
+#
+# Two things about crossing between the two blocks. A convert step names uris and carries no
+# value at all, so a value reaching one goes through storage.write and a value coming out of
+# one through storage.read. And a csv cell holds one value, so the stations of a region are
+# joined into one before they are converted -- a nested value is refused rather than
+# silently stringified.
+#
+#   dg run --local examples/transform/std-convert-fan-out.yaml --keep
+#   dg run --local examples/transform/std-convert-fan-out.yaml -p regions='["east"]'
+
+format: dirigent/v1
+kind: pipeline
+code: std-convert-fan-out
+name: Convert and fan out
+description: |
+  Store an inline csv, re-encode it as json, reshape it with jq, and fan out over the
+  regions.
+
+  Two blocks meet here, and storage is what they meet over:
+
+  1. `convert.std` is a **codec** -- a `from` and a `to`, over a source and a target uri
+  2. `transform.jq` is a **language** -- a program, over a value
+
+  A value reaches a codec through `storage.write` and leaves it through `storage.read`.
+
+tags: [transform, graph, storage]
+
+requires:
+  blocks:
+    - convert.std
+    - transform.jq
+    - storage.read
+    - storage.write
+
+params:
+  type: object
+  properties:
+    regions:
+      type: array
+      default: [east, west, north]
+      items:
+        type: string
+
+steps:
+  source:
+    block: storage.write
+    config:
+      target: ${run.scratch}/stations.csv
+      # Written inline so the example needs no network. In a real pipeline this object is
+      # the csv an upstream http.request already put in storage, and parse starts there.
+      text: |
+        station,region,status,celsius
+        st-1,east,active,4.5
+        st-2,east,retired,
+        st-3,west,active,1.2
+        st-4,north,active,-3.4
+        st-5,north,active,-1.6
+      content_type: text/csv
+
+  parse:
+    block: convert.std
+    depends_on: [source]
+    config:
+      source: ${steps.source.output.uri}
+      target: ${run.scratch}/stations.json
+      from: csv
+      to: json
+
+  rows:
+    block: storage.read
+    depends_on: [parse]
+    config:
+      source: ${steps.parse.output.target}
+
+  active:
+    block: transform.jq
+    depends_on: [rows]
+    config:
+      # Every cell came out of the csv as a string, so the program says tonumber for the one
+      # column it wants as a number.
+      input: ${steps.rows.output.value}
+      program: |
+        [.[] | select(.status == "active") | {station, region, celsius: (.celsius | tonumber)}]
+
+  per_region:
+    block: transform.jq
+    depends_on: [active]
+    # for_each is expanded when the run is created, so it reads params and not a step's
+    # output: the regions cannot come out of the csv the run is about to parse.
+    for_each: "${params.regions}"
+    config:
+      input:
+        region: "${item}"
+        rows: "${steps.active.output.value}"
+      program: |
+        . as {$region, $rows}
+        | [$rows[] | select(.region == $region)] as $here
+        | {region: $region,
+           stations: ([$here[] | .station] | join(" ")),
+           mean_celsius: (if $here == [] then null else ([$here[] | .celsius] | add / length) end)}
+
+  report:
+    block: transform.jq
+    depends_on: [per_region]
+    config:
+      # A fan-out step's output is the list of its items' outputs, in item order, so the
+      # object carrying value is unwrapped once here.
+      input: "${steps.per_region.output}"
+      program: |
+        [.[] | .value]
+
+  store:
+    block: storage.write
+    depends_on: [report]
+    config:
+      target: ${run.scratch}/regions.json
+      value: ${steps.report.output.value}
+
+  as_csv:
+    block: convert.std
+    depends_on: [store]
+    config:
+      source: ${steps.store.output.uri}
+      target: ${run.scratch}/regions.csv
+      # The header is the union of the keys the rows have, in first-seen order, and
+      # storage.write put them down as canonical JSON, so that order is alphabetical. A null
+      # mean -- a region the csv had no active reading for -- is an empty cell.
+      from: json
+      to: csv