dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/recipes/large-output-to-storage.yaml

@@ -0,0 +1,134 @@
+# A payload well past the inline threshold, and the two places size actually matters.
+#
+# In:  a row count, defaulting to 5000 rows -- a few hundred kilobytes of JSON.
+# Out: one large file in the run's scratch space, a csv beside it, and a small summary read
+#      back out of the first.
+#
+# Two different caps get confused with each other, and only one of them is a decision this
+# document makes:
+#
+#   inline_artifact_max   an instance setting, 16KB by default. Every successful attempt
+#                         keeps its whole structured output, whatever its size, so
+#                         ${steps.x.output.*} never pays a storage round trip. What the cap
+#                         governs is where the artifact copy of that output lives: at or
+#                         below it, inlined in the attempt's reference row; above it,
+#                         streamed to the run's scratch prefix with the row holding the URI
+#                         instead. Nothing in a document says any of this, and the file it
+#                         leaves behind is named after the attempt rather than after
+#                         anything a person was looking for.
+#   max_size              what this document says, on the storage.read below. A value comes
+#                         back into the run only through a read, and a read holds what it
+#                         reads, so the step names the ceiling it will hold and an object
+#                         above it is refused rather than truncated.
+#
+# So the write is not an optimisation of the threshold, it is the alternative to relying on
+# it: a result worth keeping is a result named, written and read back deliberately, and a
+# result nobody names a file for is one the engine spills for you and nobody finds.
+#
+# Five hops, and what each one hands on:
+#   generate  transform.jq, the big list. Its output is far past the threshold, so this is
+#             the attempt whose artifact copy the engine spills for you.
+#   store     storage.write, the same value under a name a person can look for.
+#   as_csv    convert.std, URI to URI. The codec never holds the list in either direction.
+#   reread    storage.read, the file back as a value, bounded by max_size.
+#   summary   transform.jq, a handful of numbers -- the shape that belongs in a step output,
+#             which is read by a person on a run page and by a reference in another step's
+#             config, and neither of those wants a megabyte.
+#
+# To change it: -p rows=200000 makes the file tens of megabytes. Past 32mb the reread is
+# refused naming the object and the cap, and raising max_size is the deliberate act that
+# allows it.
+#
+#   dg run --local examples/recipes/large-output-to-storage.yaml
+#   dg run --local examples/recipes/large-output-to-storage.yaml -p rows=50000
+
+format: dirigent/v1
+kind: pipeline
+code: large-output-to-storage
+name: A large result through storage
+description: Generate a payload far past the inline artifact threshold, write it to storage under a name of its own, and keep only a bounded summary in a step output.
+
+tags: [recipes, storage, transform]
+
+requires:
+  blocks:
+    - transform.jq
+    - storage.write
+    - convert.std
+    - storage.read
+
+params:
+  type: object
+  properties:
+    rows:
+      type: integer
+      minimum: 1
+      default: 5000
+      description: How many rows are generated; 5000 is a few hundred kilobytes of JSON.
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the file is named for.
+
+steps:
+  generate:
+    block: transform.jq
+    config:
+      input:
+        rows: ${params.rows}
+        day: ${params.day}
+      program: |
+        . as {$rows, $day}
+        | [range($rows)
+           | {id: "row-\(.)",
+              day: $day,
+              station: "st-\(. % 250)",
+              celsius: ((. % 37) - 12),
+              note: "generated for the storage recipe"}]
+
+  store:
+    block: storage.write
+    depends_on: [generate]
+    config:
+      target: ${run.scratch}/large-${params.day}.json
+      value: ${steps.generate.output.value}
+
+  as_csv:
+    block: convert.std
+    depends_on: [store]
+    config:
+      source: ${steps.store.output.uri}
+      target: ${run.scratch}/large-${params.day}.csv
+      from: json
+      to: csv
+
+  reread:
+    block: storage.read
+    depends_on: [store, as_csv]
+    config:
+      source: ${steps.store.output.uri}
+      # Written out here because this is the step where a bigger -p rows would meet it.
+      max_size: 32mb
+
+  summary:
+    block: transform.jq
+    depends_on: [reread]
+    config:
+      input: ${steps.reread.output.value}
+      program: |
+        {rows: length,
+         stations: ([.[].station] | unique | length),
+         mean_celsius: ([.[].celsius] | add / length)}
+
+  sizes:
+    block: transform.jq
+    depends_on: [store, as_csv, summary]
+    config:
+      input:
+        json_bytes: ${steps.store.output.bytes_written}
+        csv_bytes: ${steps.as_csv.output.bytes_written}
+        summary: ${steps.summary.output.value}
+      program: |
+        . + {inline_threshold_bytes: 16384,
+             json_over_threshold: (.json_bytes > 16384)}

dirigent_examples/shelves/recipes/map-enrich-with-lookup.yaml

@@ -0,0 +1,96 @@
+# Enrich every row from a lookup table, and see why the table cannot live in the map.
+#
+# In:  a list of readings, and a small dimension table of stations.
+# Out: one enriched row per reading, same length as the input, each carrying the station's
+#      name and region and a derived fahrenheit column.
+#
+# A map.jq program is handed one element and nothing else. There is no second input, no
+# --arg, and no way for the program to reach another step's output, so the lookup table has
+# to arrive attached to the elements. That attaching is a whole-value job, and it is the
+# transform.jq step above the map: it indexes the dimension once and merges the matched
+# fields into each row.
+#
+# Splicing the table into the program text with a ${...} would work exactly once, and would
+# cost the thing that makes a jq step cheap: a program with a reference in it cannot be
+# compiled when the document is applied, so a syntax error would wait for the run.
+#
+# What is left for map.jq is the per-element arithmetic, and there the verb is worth having.
+# The frame calls the program once per element and asserts the length afterwards, so this
+# step cannot drop a row or invent one, and a program that fails does it naming the element
+# by its 0-based index instead of failing the whole batch anonymously.
+#
+# To change it: -p require_match=true makes the enrichment refuse a reading whose station is
+# not in the dimension. That run fails on purpose, with "element 2: no station row for
+# st-9" -- the loud alternative to the null-filled row the default produces.
+#
+#   dg run --local examples/recipes/map-enrich-with-lookup.yaml
+#   dg run --local examples/recipes/map-enrich-with-lookup.yaml -p require_match=true
+
+format: dirigent/v1
+kind: pipeline
+code: map-enrich-with-lookup
+name: Enrich rows from a lookup table
+description: Merge a dimension table into every row with a whole-value step, then compute the per-row columns with map.jq.
+
+tags: [recipes, transform, map]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+    - map.jq
+
+params:
+  type: object
+  properties:
+    require_match:
+      type: boolean
+      default: false
+      description: Fail the element whose station is in no dimension row, rather than filling nulls.
+
+steps:
+  stations:
+    block: value.const
+    config:
+      value:
+        - { id: st-1, name: Harbour, region: east }
+        - { id: st-2, name: Ridge, region: west }
+
+  readings:
+    block: value.const
+    config:
+      value:
+        - { station: st-1, celsius: 4 }
+        - { station: st-2, celsius: -1 }
+        - { station: st-9, celsius: 21 }
+
+  attach:
+    block: transform.jq
+    depends_on: [stations, readings]
+    config:
+      input:
+        stations: ${steps.stations.output.value}
+        readings: ${steps.readings.output.value}
+        require_match: ${params.require_match}
+      program: |
+        . as {$stations, $readings, $require_match}
+        | ($stations | INDEX(.id)) as $by_id
+        | [$readings[]
+           | . + {name: ($by_id[.station].name // null),
+                  region: ($by_id[.station].region // null),
+                  require_match: $require_match}]
+
+  enrich:
+    block: map.jq
+    depends_on: [attach]
+    config:
+      input: ${steps.attach.output.value}
+      # error() fails the element, and the frame reports which index it was, so a run that
+      # refuses an unmatched reading still says which one.
+      program: |
+        if .require_match and .name == null
+        then error("no station row for \(.station)")
+        else {station, name, region,
+              celsius,
+              fahrenheit: (.celsius * 9 / 5 + 32 | round)}
+        end

dirigent_examples/shelves/recipes/ndjson-to-parquet.yaml

@@ -0,0 +1,130 @@
+# Records written as parquet, and read back to prove what landed.
+#
+# In:  a list of typed records, built inline.
+# Out: a parquet file in the run's scratch space, and the same records read back out of it.
+#
+# A conversion is a storage-object operation, like storage.copy: it reads one URI, writes
+# another, and never holds the records as a value. So a value in the run is written before
+# it is converted, and a result the run wants to look at is read back after -- which is what
+# the first and last hops here are. Parquet makes that concrete, being bytes rather than
+# text: there has never been an inline spelling for it.
+#
+# Six hops, and what each one hands on:
+#   rows        value.const, the records a real pipeline would have fetched.
+#   stored      storage.write, that value as a JSON object. Output: uri, bytes_written.
+#   as_ndjson   convert.std, json to ndjson. One record per line, no array to hold, which
+#               is the shape a stream of records already has.
+#   write       convert.arrow, ndjson to parquet. Output: source, target, bytes_written.
+#   read_back   convert.arrow, parquet to json, into a file of its own.
+#   landed      storage.read, that file as a value, which is what the summary counts.
+#
+# convert.arrow infers one type per column from the records: boolean, int64, float64, or
+# string. A column whose rows disagree is refused naming the column rather than coerced, and
+# integers and floats unify to a float column because JSON calls both a number.
+#
+# The read back is not decoration. It is how a run says what actually landed rather than
+# what was intended: the file's own bytes are decoded, and the row count and the columns
+# come from the file, so a truncated or half-written object shows up here as a failure
+# instead of downstream tomorrow.
+#
+# convert.arrow ships in dirigent-parquet, on pyarrow, where convert.std deliberately stands
+# on the standard library alone. Installing that pack is what puts the block in the catalog,
+# and naming it in requires.blocks is what refuses this document on an instance without it.
+#
+# To change it: point the targets at s3://<bucket>/... and the same steps write and read an
+# object store, because the scheme is the only thing that changes.
+#
+#   dg run --local examples/recipes/ndjson-to-parquet.yaml
+#   dg run --local examples/recipes/ndjson-to-parquet.yaml -p day=2026-02-01
+
+format: dirigent/v1
+kind: pipeline
+code: ndjson-to-parquet
+name: Records to parquet and back
+description: Write typed records into the run's scratch space, convert them through ndjson into parquet, and read the file back to see what landed.
+
+tags: [recipes, storage, transform, parquet, starter]
+
+requires:
+  blocks:
+    - value.const
+    - storage.write
+    - convert.std
+    - convert.arrow
+    - storage.read
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the written file is named for.
+
+steps:
+  rows:
+    block: value.const
+    config:
+      value:
+        - { station: st-1, celsius: 4.5, readings: 1440, active: true }
+        - { station: st-2, celsius: -1.0, readings: 1200, active: false }
+        - { station: st-3, celsius: 7.25, readings: 1439, active: true }
+
+  stored:
+    block: storage.write
+    depends_on: [rows]
+    config:
+      target: ${run.scratch}/readings-${params.day}.json
+      value: ${steps.rows.output.value}
+
+  as_ndjson:
+    block: convert.std
+    depends_on: [stored]
+    config:
+      source: ${steps.stored.output.uri}
+      target: ${run.scratch}/readings-${params.day}.ndjson
+      from: json
+      to: ndjson
+
+  write:
+    block: convert.arrow
+    depends_on: [as_ndjson]
+    config:
+      source: ${steps.as_ndjson.output.target}
+      target: ${run.scratch}/readings-${params.day}.parquet
+      from: ndjson
+      to: parquet
+
+  read_back:
+    block: convert.arrow
+    depends_on: [write]
+    config:
+      # The URI the write reported, rather than the same expression written twice: one place
+      # decides where the file went.
+      source: ${steps.write.output.target}
+      target: ${run.scratch}/readings-${params.day}-back.json
+      from: parquet
+      to: json
+
+  landed:
+    block: storage.read
+    depends_on: [read_back]
+    config:
+      source: ${steps.read_back.output.target}
+      max_size: 1mb
+
+  summary:
+    block: transform.jq
+    depends_on: [write, landed]
+    config:
+      input:
+        uri: ${steps.write.output.target}
+        file_bytes: ${steps.write.output.bytes_written}
+        records: ${steps.landed.output.value}
+      program: |
+        {uri, file_bytes,
+         rows: (.records | length),
+         columns: (.records[0] | keys),
+         records}

dirigent_examples/shelves/recipes/pagination-by-fan-out.yaml

@@ -0,0 +1,115 @@
+# Fetching several pages at once, with the page numbers as a literal list.
+#
+# In:  a list of page numbers, as a parameter.
+# Out: one HTTP call per page, each with its own status and retry budget, and a fan-in step
+#      that stitches the pages back into one list.
+#
+# for_each is expanded when the run is created, so it reads params.*, item and run.* -- and
+# never another step's output. That is the constraint this recipe is built around, and it
+# cuts both ways:
+#
+#   What you get. The item grid exists from the moment the run is visible, so a person
+#   watching sees six pages and their statuses rather than a step that might turn into six
+#   later. Items run concurrently, each with its own status, error and retry budget, so one
+#   slow page does not serialise the rest and one failed page is recorded against itself.
+#   What you give up. A fan-out whose cardinality depends on what an earlier call returned
+#   is not expressible: this is why the pages are a literal list. When the count really is
+#   unknown, either the first call reports it and a second pipeline is run with it as a
+#   parameter, or the pipeline walks the cursor one call at a time and gives up the
+#   concurrency.
+#
+# items decides what one bad page means. Under the default, fail_fast, any failed item fails
+# the step. Under continue, the step succeeds as long as one item did, the failures are
+# recorded against their own items, and the run reports completed_with_errors -- which is
+# the setting for a report that is worth producing from five pages out of six, and the wrong
+# setting for a load that must be whole.
+#
+# The fan-in reads ${steps.pages.output} -- the list of the items' outputs, in item order --
+# rather than .output.value, which is a field of one item's output and not of the list. Only
+# items that succeeded contribute.
+#
+# https://postman-echo.com/get echoes the query it was given, so each item's response
+# carries its own page number and the stitching is visibly per-page.
+#
+# To change it: -p pages='[1,2]' fetches fewer; -p page_size=10 changes what each call asks
+# for. In a real pipeline the pages usually come from an earlier run's count.
+#
+#   dg run --local examples/recipes/pagination-by-fan-out.yaml
+#   dg run --local examples/recipes/pagination-by-fan-out.yaml -p pages='[1,2]'
+
+format: dirigent/v1
+kind: pipeline
+code: pagination-by-fan-out
+name: Pagination as a fan-out
+description: Fetch a literal list of pages concurrently with for_each, each item its own attempt, then stitch the pages into one list.
+
+tags: [recipes, http, transform, graph]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    pages:
+      type: array
+      default: [1, 2, 3, 4]
+      items:
+        type: integer
+        minimum: 1
+      description: The page numbers to fetch; one item, and one call, per element.
+    page_size:
+      type: integer
+      minimum: 1
+      default: 25
+      description: What each call asks for.
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day every page is fetched for.
+
+steps:
+  pages:
+    block: http.request
+    for_each: ${params.pages}
+    # fail_fast: a partial read is not a page of results, it is a gap. A report that is
+    # worth producing from most of its pages says items: continue instead.
+    items: fail_fast
+    # Each item gets this budget of its own, so one flaky page is retried and the rest are
+    # untouched.
+    retry:
+      max_attempts: 3
+      backoff: 1s
+    config:
+      url: https://postman-echo.com/get
+      method: GET
+      query:
+        page: ${item}
+        page_size: ${params.page_size}
+        day: ${params.day}
+      timeout: 20s
+
+  stitch:
+    block: transform.jq
+    depends_on: [pages]
+    config:
+      # The fan-out's output is the list of its items' outputs, in item order.
+      input: ${steps.pages.output}
+      program: |
+        . as $items
+        | [$items[]
+           | {page: (.body.args.page | tonumber),
+              status: .status,
+              duration_ms: .duration_ms,
+              url: .body.url}] as $fetched
+        | {pages: ($fetched | length),
+           # In item order, which is the order the pages were asked for and not the order
+           # they came back in.
+           page_numbers: [$fetched[].page],
+           ordered: ([$fetched[].page] == ([$fetched[].page] | sort)),
+           slowest_ms: ([$fetched[].duration_ms] | max),
+           total_ms: ([$fetched[].duration_ms] | add),
+           fetched: $fetched}

dirigent_examples/shelves/recipes/parquet-round-trip-types.yaml

@@ -0,0 +1,163 @@
+# What survives a round trip through parquet, and what a csv would have flattened instead.
+#
+# In:  one record carrying every JSON scalar -- a string, an integer, a float, a boolean, a
+#      null, a numeric string, and a column whose values are integers and floats together.
+# Out: {before, after_parquet, after_csv, kinds, changed}, so the two formats can be
+#      compared column by column in one output.
+#
+# Parquet has a schema and csv does not, and that single difference is the whole recipe:
+#
+#   integer          survives as an int64.
+#   float            survives as a float64.
+#   boolean          survives as a boolean, not as "true".
+#   null             survives as a null, in a column that stays typed.
+#   numeric string   survives as a string. "007" comes back "007", where a csv reader is
+#                    left guessing and a spreadsheet would have eaten the leading zeros.
+#   mixed number     integers and floats unify to one float column, because JSON calls both
+#                    a number. The values still come back equal, since JSON has one number
+#                    spelling, so `changed.parquet` is empty -- the loss is in the column's
+#                    type, which a later reader sees and a jq comparison cannot.
+#
+# The csv leg is the control. Every cell in a csv is text on the way out and text on the way
+# back, so after it every value is a string and the booleans, the numbers and the nulls are
+# indistinguishable from the words that spell them.
+#
+# Both legs are four hops, and they are four rather than two because a conversion is a
+# storage-object operation: it reads one URI and writes another, and never carries a value.
+# The record is written once, converted out, converted back, and read into the run again --
+# so `before` is the value this document wrote and `after` is what came off a disk.
+#
+# The comparison is computed rather than described: `changed` lists the fields whose value
+# is not identical after the round trip, so this recipe stays honest if a codec ever changes
+# under it.
+#
+# A column whose rows disagree in a way that cannot unify is refused by name at the write
+# rather than coerced. Adding {"code": 7} to one of the rows below is the fastest way to see
+# it: "column 'code' holds both integer and string values, and a parquet column carries one
+# type; reshape it before converting".
+#
+#   dg run --local examples/recipes/parquet-round-trip-types.yaml
+
+format: dirigent/v1
+kind: pipeline
+code: parquet-round-trip-types
+name: What survives a parquet round trip
+description: Write one record of every JSON scalar, convert it out to parquet and back, and compare it against the same record round-tripped through csv.
+
+tags: [recipes, storage, transform, parquet]
+
+requires:
+  blocks:
+    - value.const
+    - storage.write
+    - convert.std
+    - convert.arrow
+    - storage.read
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the written file is named for.
+
+steps:
+  rows:
+    block: value.const
+    config:
+      value:
+        - text: Harbour
+          whole: 1440
+          fractional: 4.5
+          flag: true
+          absent: null
+          code: "007"
+          mixed: 1
+        - text: Ridge
+          whole: 1200
+          fractional: -1.25
+          flag: false
+          absent: null
+          code: "042"
+          # The other value in this column is an integer, so the column unifies to float.
+          mixed: 2.5
+
+  stored:
+    block: storage.write
+    depends_on: [rows]
+    config:
+      target: ${run.scratch}/types-${params.day}.json
+      value: ${steps.rows.output.value}
+
+  write_parquet:
+    block: convert.arrow
+    depends_on: [stored]
+    config:
+      source: ${steps.stored.output.uri}
+      target: ${run.scratch}/types-${params.day}.parquet
+      from: json
+      to: parquet
+
+  back_from_parquet:
+    block: convert.arrow
+    depends_on: [write_parquet]
+    config:
+      source: ${steps.write_parquet.output.target}
+      target: ${run.scratch}/types-${params.day}-from-parquet.json
+      from: parquet
+      to: json
+
+  read_parquet:
+    block: storage.read
+    depends_on: [back_from_parquet]
+    config:
+      source: ${steps.back_from_parquet.output.target}
+      max_size: 1mb
+
+  write_csv:
+    block: convert.std
+    depends_on: [stored]
+    config:
+      source: ${steps.stored.output.uri}
+      target: ${run.scratch}/types-${params.day}.csv
+      from: json
+      to: csv
+
+  back_from_csv:
+    block: convert.std
+    depends_on: [write_csv]
+    config:
+      source: ${steps.write_csv.output.target}
+      target: ${run.scratch}/types-${params.day}-from-csv.json
+      from: csv
+      to: json
+
+  read_csv:
+    block: storage.read
+    depends_on: [back_from_csv]
+    config:
+      source: ${steps.back_from_csv.output.target}
+      max_size: 1mb
+
+  compare:
+    block: transform.jq
+    depends_on: [rows, read_parquet, read_csv]
+    config:
+      input:
+        before: ${steps.rows.output.value}
+        after_parquet: ${steps.read_parquet.output.value}
+        after_csv: ${steps.read_csv.output.value}
+      program: |
+        def kinds: map_values(type);
+        . as {$before, $after_parquet, $after_csv}
+        | {before: $before[0], after_parquet: $after_parquet[0], after_csv: $after_csv[0],
+           kinds: {before: ($before[0] | kinds),
+                   after_parquet: ($after_parquet[0] | kinds),
+                   after_csv: ($after_csv[0] | kinds)},
+           changed: {parquet: [$before[0] | to_entries[]
+                               | select($after_parquet[0][.key] != .value) | .key],
+                     csv: [$before[0] | to_entries[]
+                           | select($after_csv[0][.key] != .value) | .key]}}