dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/recipes/jq-running-totals.yaml

@@ -0,0 +1,82 @@
+# Running totals, deltas, and a moving average over an ordered series.
+#
+# In:  daily rainfall readings for one station, deliberately out of order in the input.
+# Out: the same days sorted, each carrying cumulative, delta and mean_window, plus the
+#      final total.
+#
+# reduce and foreach are the same machine with different exhausts. reduce threads an
+# accumulator through a stream and emits the final state once; foreach threads the same
+# accumulator and emits once per element, which is exactly what a running total is. Reaching
+# for reduce and rebuilding the list afterwards is the common way to write this twice.
+#
+# Sorting first is not decoration. A cumulative column over unsorted input is a column of
+# numbers that means nothing, and the input here arrives out of order so the sort has
+# something to do.
+#
+# The moving average is the one part foreach cannot do alone, because it looks backwards
+# more than one step. jq indexes a list, so the window is read by position: for element i,
+# the slice .[i-2:i+1] is the last three days, and a slice with a negative start would wrap
+# from the end -- hence the max with 0. The first two days therefore average one and two
+# days, which is a decision the recipe makes visible rather than emitting null.
+#
+# To change it: -p window=7 widens the moving average to a week.
+#
+#   dg run --local examples/recipes/jq-running-totals.yaml
+#   dg run --local examples/recipes/jq-running-totals.yaml -p window=7
+
+format: dirigent/v1
+kind: pipeline
+code: jq-running-totals
+name: Running totals and moving averages
+description: Sort a daily series, then add cumulative sums, day-on-day deltas, and a moving average over a configurable window.
+
+tags: [recipes, transform, jq]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    window:
+      type: integer
+      minimum: 1
+      default: 3
+      description: How many days the moving average covers, the current day included.
+
+steps:
+  rows:
+    block: value.const
+    config:
+      value:
+        - { date: "2026-01-03", rainfall_mm: 4 }
+        - { date: "2026-01-01", rainfall_mm: 0 }
+        - { date: "2026-01-05", rainfall_mm: 1 }
+        - { date: "2026-01-02", rainfall_mm: 12 }
+        - { date: "2026-01-04", rainfall_mm: 7 }
+
+  series:
+    block: transform.jq
+    depends_on: [rows]
+    config:
+      input:
+        rows: ${steps.rows.output.value}
+        window: ${params.window}
+      program: |
+        . as {$rows, $window}
+        # ISO dates sort as strings, so no date parsing is needed to order the series.
+        | ($rows | sort_by(.date)) as $ordered
+        | ($ordered | map(.rainfall_mm)) as $values
+        | {days: [foreach range($ordered | length) as $i (
+                    {total: 0, previous: null};
+                    {total: (.total + $values[$i]),
+                     previous: (if $i == 0 then null else $values[$i - 1] end)};
+                    $ordered[$i]
+                    + {cumulative: .total,
+                       delta: (if .previous == null then null
+                               else $values[$i] - .previous end),
+                       mean_window: (($values[([$i - $window + 1, 0] | max):$i + 1])
+                                     | add / length)})],
+           total: ($values | add)}

dirigent_examples/shelves/recipes/jq-string-cleaning.yaml

@@ -0,0 +1,88 @@
+# Clean the strings a spreadsheet exported: whitespace, case, punctuation, and codes.
+#
+# In:  rows whose name, code and email arrived with padding, mixed case, doubled inner
+#      spaces, and a code carrying a suffix nobody wanted.
+# Out: the same rows cleaned, plus a `changed` list naming the fields each row's cleaning
+#      actually touched.
+#
+# Every verb here is one jq builtin doing one thing, and the order matters: trim before
+# comparing, collapse inner whitespace before splitting, and lowercase an email but never a
+# name. A single gsub doing all of it at once is the version nobody can debug six months
+# later.
+#
+#   sub("^\\s+"; "") / sub("\\s+$"; "")   trim, anchored at each end. jq has no trim verb,
+#                                         and ltrimstr only removes a literal prefix.
+#   gsub("\\s+"; " ")                     collapse a run of inner whitespace to one space.
+#   ascii_downcase                        case-fold. It is ASCII-only by name and by
+#                                         behaviour, so it leaves accented letters alone
+#                                         rather than mangling them.
+#   capture("(?<code>[A-Z]+-[0-9]+)")     pull a code out of a longer string by naming the
+#                                         part wanted; the named group becomes a field.
+#   test(...)                             ask a yes-or-no question about a string without
+#                                         changing it.
+#
+# The `changed` list is what makes cleaning reviewable: it is computed by comparing each
+# field before and after, so a run says which rows it had to touch rather than leaving that
+# to a diff nobody takes.
+#
+# To change it: -p title_case=true also capitalises each word of a name, which is the kind
+# of rule that is right for a report and wrong for a key.
+#
+#   dg run --local examples/recipes/jq-string-cleaning.yaml
+#   dg run --local examples/recipes/jq-string-cleaning.yaml -p title_case=true
+
+format: dirigent/v1
+kind: pipeline
+code: jq-string-cleaning
+name: Clean strings with jq
+description: Trim, collapse, case-fold and extract the strings a spreadsheet export brings in, and report which fields changed.
+
+tags: [recipes, transform, jq]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    title_case:
+      type: boolean
+      default: false
+      description: Capitalise each word of a cleaned name.
+
+steps:
+  rows:
+    block: value.const
+    config:
+      value:
+        - { name: "  harbour   station ", code: "ST-1 (retired)", email: "Ops@Example.ORG " }
+        - { name: "RIDGE station", code: "site ST-2", email: "ridge@example.org" }
+        - { name: "delta station", code: "ST-3", email: "  DELTA@EXAMPLE.ORG" }
+
+  cleaned:
+    block: transform.jq
+    depends_on: [rows]
+    config:
+      input:
+        rows: ${steps.rows.output.value}
+        title_case: ${params.title_case}
+      program: |
+        def trim: sub("^\\s+"; "") | sub("\\s+$"; "");
+        def squeeze: gsub("\\s+"; " ");
+        def titled: split(" ") | map(if length > 0
+                                     then (.[0:1] | ascii_upcase) + (.[1:] | ascii_downcase)
+                                     else . end) | join(" ");
+        . as {$rows, $title_case}
+        | [$rows[]
+           | . as $before
+           | {name: (.name | trim | squeeze | if $title_case then titled else . end),
+              # The code is whatever looks like a code inside the cell; a row with none
+              # keeps a null rather than the raw cell, so the failure is visible downstream.
+              code: (.code | trim | (capture("(?<code>[A-Z]+-[0-9]+)").code // null)),
+              email: (.email | trim | ascii_downcase),
+              looks_like_email: (.email | trim | test("^[^@\\s]+@[^@\\s]+\\.[a-zA-Z]{2,}$"))}
+           | . as $after
+           | $after + {changed: [$before | to_entries[]
+                                 | select($after[.key] != .value) | .key]}]

dirigent_examples/shelves/recipes/jq-top-n.yaml

@@ -0,0 +1,85 @@
+# The top N rows by a measure, with ties and the long tail both accounted for.
+#
+# In:  a list of stations with an uptime percentage and a reading count.
+# Out: {top, tail_summary, ties} -- the leaders, one row summarising everything below them,
+#      and the rows that tie with the last leader.
+#
+# jq has no "top n" builtin, and does not need one: sort_by ascends, reverse descends, and a
+# slice takes the head. Writing it as sort_by(-.x) instead only works for numbers, and fails
+# quietly the day the measure becomes a string, so the reverse spelling is the one to learn.
+#
+# Two things a naive top-n gets wrong, and both are in the output:
+#
+#   Ties.     A cut at position N splits a tie arbitrarily. The ties list re-reads the input
+#             for every row whose measure equals the last leader's, so a run says when the
+#             cut was arbitrary instead of pretending it was not.
+#   The tail. Dropping the rest hides how much was dropped. One summary row -- how many, and
+#             what they add up to -- keeps the total honest and costs one more slice.
+#
+# The slice is bounded by the list rather than by N: .[0:$n] on a list shorter than N is the
+# whole list rather than an error, which is jq being forgiving in the one place a report
+# wants it to be.
+#
+# To change it: -p n=1 narrows it, -p measure=readings ranks by the other column.
+#
+#   dg run --local examples/recipes/jq-top-n.yaml
+#   dg run --local examples/recipes/jq-top-n.yaml -p n=1 -p measure=readings
+
+format: dirigent/v1
+kind: pipeline
+code: jq-top-n
+name: Top N with ties and a tail
+description: Rank rows by a measure and keep the leaders, the rows tied with the last of them, and a summary of everything below.
+
+tags: [recipes, transform, jq]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    n:
+      type: integer
+      minimum: 1
+      default: 3
+      description: How many leaders to keep.
+    measure:
+      type: string
+      default: uptime
+      description: The column the ranking reads.
+
+steps:
+  rows:
+    block: value.const
+    config:
+      value:
+        - { station: st-1, uptime: 99.4, readings: 1440 }
+        - { station: st-2, uptime: 87.0, readings: 1200 }
+        - { station: st-3, uptime: 99.9, readings: 1439 }
+        # A tie with st-2 on uptime, which is what the ties list exists to report.
+        - { station: st-4, uptime: 87.0, readings: 300 }
+        - { station: st-5, uptime: 62.5, readings: 900 }
+
+  ranked:
+    block: transform.jq
+    depends_on: [rows]
+    config:
+      input:
+        rows: ${steps.rows.output.value}
+        n: ${params.n}
+        measure: ${params.measure}
+      program: |
+        . as {$rows, $n, $measure}
+        | ($rows | sort_by(.[$measure]) | reverse) as $ordered
+        | ($ordered | .[0:$n]) as $top
+        | ($ordered | .[$n:]) as $rest
+        | {top: $top,
+           tail_summary: {rows: ($rest | length),
+                          readings: ($rest | map(.readings) | add // 0)},
+           ties: (if ($top | length) == 0 then []
+                  else ($top[-1][$measure]) as $cut
+                       | [$ordered[] | select(.[$measure] == $cut) | .station]
+                  end)}

dirigent_examples/shelves/recipes/jq-validate-in-jq-vs-schema.yaml

@@ -0,0 +1,124 @@
+# Two ways to check data, in one document, so the difference is visible rather than argued.
+#
+# In:  a batch of rows, four good and two bad -- one missing a field, one with a level that
+#      is a string.
+# Out: {accepted, rejected, counts} from the jq check, and the accepted rows again from a
+#      validate.schema gate that could not have passed the bad ones.
+#
+# The two checks answer different questions, and a pipeline usually wants both:
+#
+#   jq              sorts a batch. It looks at each row, keeps the ones it likes, and reports
+#                   the rest with a reason -- so a bad row costs one line in a report rather
+#                   than a failed run. It is a program, so nothing outside this document
+#                   knows what it enforced.
+#   validate.schema is a gate. The shape is declarative, named, and reusable, and a value
+#                   that does not fit fails the run at the boundary rather than travelling
+#                   on. Nothing downstream has to trust that a jq program upstream was
+#                   thorough: the gate's output is the proof, which is why the last step
+#                   here reads ${steps.gate.output.value} and not the sorter's accepted list.
+#
+# So: jq decides which rows to keep, the schema decides what a kept row is allowed to be.
+# Running the gate on rows a jq program already sorted is not redundant -- it is the check
+# that the sorter and the schema still agree, and the day they stop the run says so.
+#
+# The rejected rows keep their reason and their index, because "6 rows in, 4 rows out" is
+# not a report anybody can act on.
+#
+# To change it: -p min_level=2 tightens what the sorter accepts. The schema is unchanged,
+# and the gate still passes, because a tighter sorter cannot produce a row the schema
+# refuses -- which is exactly the relationship the two checks should have.
+#
+# The schema is carried in the document, so no server stores this one: it runs with
+# `dg run --local`, and on an instance the same shape is created once with `dg schema create`.
+#
+#   dg run --local examples/recipes/jq-validate-in-jq-vs-schema.yaml
+#   dg run --local examples/recipes/jq-validate-in-jq-vs-schema.yaml -p min_level=2
+
+format: dirigent/v1
+kind: pipeline
+code: jq-validate-in-jq-vs-schema
+name: Checking in jq versus checking with a schema
+description: Sort a batch into accepted and rejected rows with jq, then hold the accepted ones to a carried schema at a gate.
+
+tags: [recipes, transform, validate, jq]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+    - validate.schema
+
+schemas:
+  recipe-org-unit-list:
+    type: array
+    items:
+      type: object
+      required: [id, name, level]
+      additionalProperties: false
+      properties:
+        id: { type: string, minLength: 1 }
+        name: { type: string, minLength: 1 }
+        level: { type: integer, minimum: 1 }
+
+params:
+  type: object
+  properties:
+    min_level:
+      type: integer
+      minimum: 1
+      default: 1
+      description: The lowest level the jq sorter accepts.
+
+steps:
+  batch:
+    block: value.const
+    config:
+      value:
+        - { id: ou-1, name: Sierra Leone, level: 1 }
+        - { id: ou-2, name: Bo, level: 2 }
+        # No level at all.
+        - { id: ou-3, name: Kenema }
+        - { id: ou-4, name: Bombali, level: 2 }
+        # A level that arrived as a string, which is what a csv source always produces.
+        - { id: ou-5, name: Kailahun, level: "3" }
+        - { id: ou-6, name: Moyamba, level: 3 }
+
+  sorted:
+    block: transform.jq
+    depends_on: [batch]
+    config:
+      input:
+        rows: ${steps.batch.output.value}
+        min_level: ${params.min_level}
+      program: |
+        . as {$rows, $min_level}
+        | [range($rows | length) as $i
+           | $rows[$i]
+           | {row: ., index: $i,
+              problem: (if (.level | type) != "number" then "level is \(.level | type)"
+                        elif .level < $min_level then "level below \($min_level)"
+                        else null end)}] as $checked
+        | {accepted: [$checked[] | select(.problem == null) | .row],
+           rejected: [$checked[] | select(.problem != null) | {index, id: .row.id, problem}],
+           counts: {seen: ($rows | length),
+                    accepted: [$checked[] | select(.problem == null)] | length,
+                    rejected: [$checked[] | select(.problem != null)] | length}}
+
+  gate:
+    block: validate.schema
+    depends_on: [sorted]
+    config:
+      # The whole accepted list is checked at once, so one bad row fails the batch rather
+      # than half a batch reaching the sink.
+      input: ${steps.sorted.output.value.accepted}
+      schema: recipe-org-unit-list
+
+  load:
+    block: transform.jq
+    depends_on: [gate]
+    config:
+      # Read from the gate, never from the sorter: this reference is what proves in the
+      # document that nothing unchecked got this far.
+      input: ${steps.gate.output.value}
+      program: |
+        {loaded: length, ids: map(.id)}

dirigent_examples/shelves/recipes/jq-window-dates.yaml

@@ -0,0 +1,82 @@
+# Date arithmetic in jq: a window of days, ISO weeks, and the strptime trap.
+#
+# In:  a start date and a length in days, as parameters.
+# Out: {window: {start, end}, days: [...], weeks: [{iso_week, days, dates}], trap: {...}}.
+#
+# jq has no date type. It has epoch seconds and two conversions on either side of them, and
+# every date calculation here is arithmetic on those seconds:
+#
+#   "2026-01-05T00:00:00Z" | fromdateiso8601   -> 1767571200
+#   1767571200 | todate                        -> "2026-01-05T00:00:00Z"
+#
+# The trap has a name and it is strptime. strptime does not answer with a number: it answers
+# with jq's broken-down time, an array of eight fields, and arithmetic on that array is an
+# error rather than a wrong date. mktime turns it into seconds, and only then does strftime
+# have something to format. The trap object at the end of the output shows both spellings so
+# the difference is on screen: .broken_down is the array, .seconds is the number.
+#
+# A date-only string has no time and no zone, so "2026-01-05" is completed to midnight UTC
+# before it is parsed. Doing that explicitly is the difference between a window that means
+# the same thing on every worker and one that moves with the machine's timezone.
+#
+# ISO weeks come from %G and %V together, never from %Y and %V: the last days of December
+# belong to week 01 of the next ISO year, and %Y-%V would file them under the year that just
+# ended. The window below is chosen to cross a year boundary so that shows up in the output.
+#
+# Adding days as 86400 seconds is exact here because everything is UTC. It is not exact in a
+# zone with daylight saving, where a day is sometimes 23 or 25 hours; a pipeline that needs
+# local days does the arithmetic on dates rather than on seconds.
+#
+# To change it: -p start=2026-03-01 -p days=14 moves and widens the window.
+#
+#   dg run --local examples/recipes/jq-window-dates.yaml
+#   dg run --local examples/recipes/jq-window-dates.yaml -p start=2026-03-01 -p days=14
+
+format: dirigent/v1
+kind: pipeline
+code: jq-window-dates
+name: Date windows and ISO weeks in jq
+description: Build a window of days from a start date, bucket it into ISO weeks, and show why strptime alone is not a date.
+
+tags: [recipes, transform, jq]
+
+requires:
+  blocks:
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    start:
+      type: string
+      format: date
+      default: "2026-12-28"
+      description: The first day of the window, as YYYY-MM-DD.
+    days:
+      type: integer
+      minimum: 1
+      default: 10
+      description: How many days the window covers, the start day included.
+
+steps:
+  window:
+    block: transform.jq
+    config:
+      input:
+        start: ${params.start}
+        days: ${params.days}
+      program: |
+        . as {$start, $days}
+        # The date is completed to midnight UTC before parsing, so the window means the same
+        # thing wherever the worker runs.
+        | ($start + "T00:00:00Z" | fromdateiso8601) as $from
+        | [range($days) | $from + (. * 86400)] as $seconds
+        | {window: {start: ($from | todate),
+                    end: ($seconds[-1] | todate)},
+           days: [$seconds[] | strftime("%Y-%m-%d")],
+           weeks: ([$seconds[] | {iso_week: strftime("%G-W%V"), date: strftime("%Y-%m-%d")}]
+             | group_by(.iso_week)
+             | map({iso_week: .[0].iso_week, days: length, dates: map(.date)})),
+           trap: {broken_down: ($start | strptime("%Y-%m-%d")),
+                  seconds: ($start | strptime("%Y-%m-%d") | mktime),
+                  formatted: ($start | strptime("%Y-%m-%d") | mktime | strftime("%G-W%V"))}}

dirigent_examples/shelves/recipes/json-to-csv-flattening.yaml

@@ -0,0 +1,141 @@
+# Nested JSON into a csv somebody opens in a spreadsheet, with the nesting decided on.
+#
+# In:  an API-shaped payload: an envelope around a list of records, each with a nested
+#      object, a list, and a field the report should not carry.
+# Out: a csv in the run's scratch space, and its text read back for the run's Output tab.
+#
+# A csv row is flat, so this is three decisions rather than a conversion:
+#
+#   The envelope.  The codec wants the array, not the object wrapping it, so the jq step
+#                  reaches into .results before anything else happens.
+#   The nesting.   A nested value has no csv spelling and convert.std refuses it naming the
+#                  row and the key. Every nested field is therefore pulled up into a column
+#                  with a name a person reading the header will understand -- site_name, not
+#                  site.name, because a dot in a header is a thing spreadsheets fight over.
+#   The lists.     tags is a list, so the report joins it with a separator it chooses. That
+#                  choice belongs here, in the step that knows the values contain no
+#                  semicolons, and not in a codec that would have to guess.
+#
+# Column order is the last decision, and it is the reason the projection is written out
+# field by field rather than generated: a report's header is part of its contract with
+# whoever reads it, and a program that emits whatever keys it finds changes that header the
+# day the source adds a field.
+#
+# Keeping that order is why the rows leave jq as text. storage.write serialises a `value`
+# as canonical JSON, which sorts the keys, and the csv header is the key order of the first
+# record; written as `text`, the bytes are the ones jq produced and the header is the one
+# the projection spells.
+#
+# Five hops, and what each one hands on:
+#   payload   value.const, the envelope a real pipeline would have fetched.
+#   rows      transform.jq, the flat records, serialised in the step. Output: value, a
+#             string of JSON.
+#   stored    storage.write, that text under a name, said to be JSON. Output: uri,
+#             bytes_written.
+#   csv       convert.std, one URI to another. A conversion is a storage-object operation
+#             like storage.copy: it never holds the records as a value, which is why the
+#             rows are written before it and read after it.
+#   preview   storage.read, the csv back as text, so the run's output shows the file
+#             without anybody opening it.
+#
+# To change it: -p separator=, changes what joins a list cell -- and is the fastest way to
+# see why the default is a semicolon in a comma-separated file.
+#
+#   dg run --local examples/recipes/json-to-csv-flattening.yaml
+#   dg run --local examples/recipes/json-to-csv-flattening.yaml -p separator=,
+
+format: dirigent/v1
+kind: pipeline
+code: json-to-csv-flattening
+name: Nested JSON to a flat csv
+description: Reach into an envelope, flatten the nested fields into named columns, join the list cells, and convert the written rows into a csv artifact.
+
+tags: [recipes, storage, transform, starter]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+    - storage.write
+    - convert.std
+    - storage.read
+
+params:
+  type: object
+  properties:
+    separator:
+      type: string
+      default: ";"
+      description: What joins the values of a list-valued field inside one cell.
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the csv is named for.
+
+steps:
+  payload:
+    block: value.const
+    config:
+      value:
+        generated_at: "2026-01-01T06:00:00Z"
+        page: 1
+        results:
+          - id: st-1
+            site: { name: Harbour, region: east, coords: { lat: 59.91, lon: 10.75 } }
+            tags: [coastal, tidal]
+            readings: { count: 1440, mean_celsius: 4.5 }
+            internal_note: do not publish
+          - id: st-2
+            site: { name: Ridge, region: west, coords: { lat: 60.39, lon: 5.32 } }
+            tags: [alpine]
+            readings: { count: 1200, mean_celsius: -1.0 }
+            internal_note: recalibrated
+
+  rows:
+    block: transform.jq
+    depends_on: [payload]
+    config:
+      input:
+        payload: ${steps.payload.output.value}
+        separator: ${params.separator}
+      program: |
+        . as {$payload, $separator}
+        | [$payload.results[]
+           | {station: .id,
+              site_name: .site.name,
+              region: .site.region,
+              lat: .site.coords.lat,
+              lon: .site.coords.lon,
+              tags: (.tags | join($separator)),
+              readings: .readings.count,
+              mean_celsius: .readings.mean_celsius}]
+        | tojson
+
+  stored:
+    block: storage.write
+    depends_on: [rows]
+    config:
+      target: ${run.scratch}/stations-${params.day}.json
+      # text rather than value: a value is written as canonical JSON, and canonical JSON
+      # sorts the keys the csv header is built from.
+      text: ${steps.rows.output.value}
+      content_type: application/json
+
+  csv:
+    block: convert.std
+    depends_on: [stored]
+    config:
+      # Where the write said the rows landed, not the same path typed twice.
+      source: ${steps.stored.output.uri}
+      # The file is the point of a report run, so it goes to storage under a dated name.
+      target: ${run.scratch}/stations-${params.day}.csv
+      from: json
+      to: csv
+
+  preview:
+    block: storage.read
+    depends_on: [csv]
+    config:
+      source: ${steps.csv.output.target}
+      max_size: 1mb