dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/patterns/fan-out-item-wise.yaml

@@ -0,0 +1,111 @@
+# Two fan-outs over one grid: item three of the second reads item three of the first.
+#
+# A fanned step's output is the list of its items' outputs, so a step that depends on a
+# fan-out sees the whole batch. That is right for a join and wrong for everything that is
+# still per item: writing one file per district, calling one endpoint per region, sending one
+# message per order. Those want the matching item, not the list.
+#
+# for_each: ${steps.<name>.items} is that. The step maps over the grid the named fan-out
+# already has instead of a grid of its own, and the two are paired by position: item 3 here is
+# item 3 there, ${item} is the same element in both, and ${steps.<name>.item.output} is the
+# matching item's output.
+#
+# Three things the pairing buys, and one it costs:
+#
+#   1. Cardinality is still fixed when the run is created. The grid comes from a step that was
+#      itself expanded then, so nothing here waits on execution to know how wide it is.
+#   2. The names must be a direct dependency: the adopted step goes in depends_on. Pairing
+#      across a step that is not waited for would read an item that has not run.
+#   3. Adoption chains. A third step may map over the second's items and still reach the
+#      first's, because they are all one grid.
+#   4. The cost: an item whose match did not succeed is SKIPPED, not failed and not run. Its
+#      input never existed, so there is nothing to attempt. The rest of the grid carries on.
+#
+# Hop by hop:
+#
+#   shape      transform.jq, once per region. Under items: continue a region that refuses is
+#              recorded against its own item and the others finish.
+#   write_one  storage.write, mapping over shape's grid rather than a list of its own. One
+#              file per region, named for ${item}, holding ${steps.shape.item.output.value} --
+#              that region's own object, not the batch.
+#   manifest   the join, with no for_each at all: one attempt over the whole list of files.
+#
+# EXPECT THIS RUN TO SUCCEED in about a second, with three files written and a manifest naming
+# all three.
+#
+# To change it: -p refuse=west makes shape's west item fail, which skips write_one's west item
+# rather than failing it, leaves the other two written, and ends the run
+# completed_with_errors with a manifest of two files.
+#
+#   dg run --local examples/patterns/fan-out-item-wise.yaml
+#   dg run --local examples/patterns/fan-out-item-wise.yaml -p refuse=west  # completed_with_errors
+
+format: dirigent/v1
+kind: pipeline
+code: fan-out-item-wise
+name: One item to one item
+description: |
+  `for_each: ${steps.<name>.items}` maps a step over another fan-out's grid, so the two are
+  paired by item position and `${steps.<name>.item.output}` is the matching item's output.
+
+  An item whose match did not succeed is skipped rather than failed, and the rest of the grid
+  carries on.
+
+tags: [patterns, storage, transform, fan-out, references, starter]
+
+requires:
+  blocks:
+    - storage.write
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    regions:
+      type: array
+      default: [east, west, north]
+      minItems: 1
+      maxItems: 16
+      items:
+        type: string
+      description: One item, one file, per element.
+    refuse:
+      type: string
+      default: ""
+      description: A region whose shape step fails, so its write is skipped rather than run.
+
+steps:
+  shape:
+    block: transform.jq
+    for_each: "${params.regions}"
+    items: continue
+    config:
+      input:
+        region: "${item}"
+        refuse: "${params.refuse}"
+      # error() is how a jq program fails one item deliberately; without the refuse knob this
+      # program is just the object-building half.
+      program: |
+        if .region == .refuse then error("region \(.region) refused the export")
+        else {region: .region, rows: [{station: "\(.region)-st-0", celsius: -4}]}
+        end
+
+  write_one:
+    block: storage.write
+    depends_on: [shape]
+    # The whole point of the file: this step has no grid of its own, it takes shape's.
+    for_each: "${steps.shape.items}"
+    config:
+      # ${item} is shape's element, so two items never write the same key.
+      target: "${run.scratch}/regions/${item}.json"
+      # ...and this is that element's own output, not the list of every item's.
+      value: "${steps.shape.item.output.value}"
+
+  manifest:
+    block: transform.jq
+    depends_on: [write_one]
+    # No for_each, so this is the join: one attempt, and the batch arrives as a list.
+    config:
+      input: "${steps.write_one.output}"
+      program: |
+        {files: length, uris: [.[].uri], total_bytes: ([.[].bytes_written] | add)}

dirigent_examples/shelves/patterns/fan-out-literal-list.yaml

@@ -0,0 +1,81 @@
+# for_each as a literal list: the smallest fan-out there is.
+#
+# for_each takes one of two things: a reference to a list, or a list written out in the
+# document. This file is the second, which is the right form when the elements are part of the
+# pipeline's shape rather than part of a run's input -- the three environments you deploy to,
+# the four quarters, the fixed set of feeds. Nobody should have to pass those in to get an
+# ordinary run.
+#
+# What a fan-out actually is: one step definition, N run items, one attempt each, all of them
+# claimable at once. The item grid exists from the moment the run is visible, because
+# for_each is expanded when the run is CREATED. That has a consequence worth knowing before
+# you reach for it: for_each may read params, run, and another fan-out's grid as
+# ${steps.<step>.items}, but never a step's output, because the cardinality has to be known
+# before anything executes. A fan-out over a list fetched at run time is not expressible, and
+# that is deliberate rather than missing.
+#
+# Hop by hop:
+#
+#   greet    transform.jq, once per name in the literal list. ${item} is the element, and
+#            here that is a plain string.
+#   collect  runs ONCE, not once per item, because it has no for_each of its own. It reads
+#            ${steps.greet.output}, which is the list of the items' outputs in item order.
+#
+# EXPECT THIS RUN TO SUCCEED in about a second. Nothing here touches the network or runs code
+# on the worker, so it needs no allowlist and no connection.
+#
+# To change it: fan-out-from-params.yaml is the same shape with the list supplied by the
+# caller, fan-out-nested-objects.yaml is the same shape with objects as elements, and
+# fan-out-item-wise.yaml is a second step mapping over this one's grid instead of joining it.
+#
+#   dg run --local examples/patterns/fan-out-literal-list.yaml
+
+format: dirigent/v1
+kind: pipeline
+code: fan-out-literal-list
+name: A fan-out over a literal list
+description: |
+  `for_each` written as a list in the document: one step definition, one run item per
+  element, all claimable at once.
+
+  A literal list is the right form when the elements are part of the pipeline's shape
+  rather than part of a run's input.
+
+tags: [patterns, transform, fan-out, graph]
+
+requires:
+  blocks:
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    greeting:
+      type: string
+      description: The word each item is greeted with; it is the same for every item.
+      default: hei
+
+steps:
+  greet:
+    block: transform.jq
+    # Written out here rather than referenced, because these three are what this pipeline is
+    # about. A list that changes per run belongs in params instead.
+    for_each: [oslo, bergen, tromso]
+    config:
+      input:
+        # ${item} is this run item's element. It is only in scope inside a step that fans
+        # out; anywhere else it is an unknown reference and the attempt is rejected.
+        city: "${item}"
+        greeting: "${params.greeting}"
+      program: |
+        {city, line: "\(.greeting), \(.city)"}
+
+  collect:
+    block: transform.jq
+    depends_on: [greet]
+    # One attempt, not three. A step joins a fan-out simply by having no for_each of its own.
+    config:
+      # The list of the items' outputs, in item order.
+      input: "${steps.greet.output}"
+      program: |
+        {lines: [.[].value.line], cities: length}

dirigent_examples/shelves/patterns/fan-out-nested-objects.yaml

@@ -0,0 +1,107 @@
+# An item that is an object, and the fields of it a step reaches into.
+#
+# An element of a for_each list is any JSON value, and once it is an object the item stops
+# being a label and becomes a small record. ${item.code} reaches a field, ${item.limits.max_ms}
+# reaches a nested one, and ${item} on its own is still the whole object -- so one element can
+# carry everything one item's work needs instead of forcing three parallel lists that have to
+# stay in the same order.
+#
+# The reference rule that makes this work: a reference that is the ENTIRE value resolves to
+# the typed value, so "${item.limits.max_ms}" is the integer 2000 downstream and not the text
+# "2000". A reference inside a larger string interpolates as text instead, which is why the
+# URL below reads as a URL and the threshold reads as a number.
+#
+# A missing field is not an empty string. ${item.nickname} on an element that has no nickname
+# raises, and the engine settles that attempt as rejected rather than quietly proceeding with
+# nothing -- so every element in the list must carry every field the step names. Give the
+# optional ones a default in the list itself.
+#
+# Hop by hop:
+#
+#   probe    one item per feed. Each item posts its own code and its own latency budget, so
+#            the three items do genuinely different work from one step definition. Postman
+#            Echo answers with what it was sent, which is how each item's element comes back
+#            attached to that item's output.
+#   verdict  a join over the batch, comparing each feed's measured round trip against the
+#            budget that travelled with it.
+#
+# EXPECT THIS RUN TO SUCCEED in about two seconds, with three feeds and a verdict each. The
+# items run side by side, so the run costs one round trip rather than three.
+#
+# To change it: add a feed to the list, or lower a budget below the real round trip and watch
+# that feed's verdict flip to slow. The budget lives on the element rather than in params
+# because it belongs to one feed and not to the run.
+#
+#   dg run --local examples/patterns/fan-out-nested-objects.yaml
+
+format: dirigent/v1
+kind: pipeline
+code: fan-out-nested-objects
+name: Items that are objects
+description: |
+  A `for_each` element is any JSON value. Once it is an object, `${item.code}` reaches a
+  field and `${item.limits.max_ms}` a nested one, so one element carries everything one item's
+  work needs.
+
+  A whole reference resolves to the typed value; a missing field is a rejected attempt, not
+  an empty string.
+
+tags: [patterns, http, transform, fan-out, references]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    tolerance_ms:
+      type: integer
+      description: Slack added to every feed's own threshold before the verdict is taken.
+      default: 500
+      minimum: 0
+      maximum: 5000
+
+steps:
+  probe:
+    block: http.request
+    # Three records, not three parallel lists. Nothing has to stay in step with anything.
+    for_each:
+      - {code: cases, path: post, limits: {max_ms: 2000}}
+      - {code: climate, path: post, limits: {max_ms: 2500}}
+      - {code: population, path: post, limits: {max_ms: 400}}
+    config:
+      # Interpolated into a larger string, so the field is rendered as text inside the URL.
+      url: "https://postman-echo.com/${item.path}"
+      method: POST
+      body:
+        feed: "${item.code}"
+        # The nested field, carried in the request so it comes back in the echo. That is how a
+        # per-item value reaches the join: the join sees outputs, not elements, so anything an
+        # element knows has to be inside an item's output to survive the fan-in. A whole
+        # reference stays typed, so this arrives as the integer it was written as.
+        max_ms: "${item.limits.max_ms}"
+
+  verdict:
+    block: transform.jq
+    depends_on: [probe]
+    config:
+      input:
+        measured: "${steps.probe.output}"
+        tolerance_ms: "${params.tolerance_ms}"
+      # json is the typed echo of the body that was posted, so max_ms is still a number here
+      # and needs no tonumber. A query string would have come back as text.
+      program: |
+        .tolerance_ms as $tolerance
+        | {
+            feeds: [
+              .measured[]
+              | {
+                  feed: .body.json.feed,
+                  duration_ms,
+                  budget_ms: (.body.json.max_ms + $tolerance)
+                }
+              | .verdict = (if .duration_ms <= .budget_ms then "within budget" else "slow" end)
+            ]
+          }

dirigent_examples/shelves/patterns/fan-out-then-join.yaml

@@ -0,0 +1,86 @@
+# Fan out, then reduce: N items in, one aggregate out, and no keyword for either.
+#
+# The join is not a construct. A step joins a fan-out by depending on it and NOT declaring a
+# for_each of its own -- that is the whole mechanism, and it is why nothing here says "join"
+# or "reduce". One step definition, one attempt, and the batch arrives as a list.
+#
+# Three facts about what the join sees:
+#
+#   1. ${steps.measure.output} is a JSON array of the items' outputs, in item order.
+#   2. It is a value like any other, so a jq program reduces it the way jq reduces anything.
+#   3. Under items: continue a failed item is absent from that array, so the join must compute
+#      from what is there rather than from what was asked for. add/length below is right
+#      whatever survived; dividing by a hard-coded four would not be.
+#
+# Hop by hop:
+#
+#   measure  a literal list of station readings, fanned out. Each item scales one reading and
+#            emits a small object. This stands in for a fetch, and it needs no network, so the
+#            aggregation is what you watch rather than the HTTP.
+#   summary  the join. One attempt, the whole array, count and min and max and mean out of it.
+#
+# EXPECT THIS RUN TO SUCCEED in about a second, with a summary over four stations.
+#
+# To change it: add a station to the literal list and nothing else needs editing -- the join is
+# already written for whatever arrives. fan-in.yaml on the graph/ shelf is the same shape
+# shipping the raw batch onward instead of reducing it.
+#
+#   dg run --local examples/patterns/fan-out-then-join.yaml
+#   dg run --local examples/patterns/fan-out-then-join.yaml -p scale=2
+
+format: dirigent/v1
+kind: pipeline
+code: fan-out-then-join
+name: Fan out, then reduce
+description: |
+  A step joins a fan-out by depending on it and having no `for_each` of its own: one attempt,
+  and the batch arrives as a JSON array of the items' outputs.
+
+  The aggregate is computed from what is in that array, never from the width that was asked
+  for.
+
+tags: [patterns, transform, fan-out, graph]
+
+requires:
+  blocks:
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    scale:
+      type: number
+      description: A factor every reading is multiplied by before the summary reduces them.
+      default: 1.0
+      minimum: 0.1
+      maximum: 10.0
+
+steps:
+  measure:
+    block: transform.jq
+    # Objects rather than strings, so one item carries a whole reading. ${item.station} and
+    # ${item.celsius} reach into it; fan-out-nested-objects.yaml is that on its own.
+    for_each:
+      - {station: fornebu, celsius: 4.5}
+      - {station: blindern, celsius: 3.1}
+      - {station: gardermoen, celsius: -1.2}
+      - {station: tryvann, celsius: -4.8}
+    config:
+      input:
+        station: "${item.station}"
+        # A whole reference resolves to the typed value, so this is a number downstream and
+        # not the text of one. That is what lets the program below multiply it.
+        celsius: "${item.celsius}"
+        scale: "${params.scale}"
+      program: |
+        {station, celsius: (.celsius * .scale)}
+
+  summary:
+    block: transform.jq
+    depends_on: [measure]
+    # No for_each: this is the join, and one attempt sees all four items.
+    config:
+      input: "${steps.measure.output}"
+      program: |
+        [.[].value.celsius] |
+        {stations: length, min: min, max: max, mean: (add / length)}

dirigent_examples/shelves/patterns/log-levels.yaml

@@ -0,0 +1,119 @@
+# Turning one block family's logs up, without turning the whole run's logs up.
+#
+# A run keeps info and above by default. --log-level changes that FOR ONE RUN, and it is a
+# property of the run rather than of the document -- which is the right place for it: how
+# loudly a pipeline logs is an operational question asked while something is being
+# investigated, not a fact about the pipeline that belongs in git and in a review.
+#
+# THE FLAG TAKES TWO FORMS, and it repeats:
+#
+#   --log-level debug              every block. The blunt instrument.
+#   --log-level PATTERN=LEVEL      one family. The one worth learning.
+#
+# PATTERN is an fnmatch over the BLOCK ID, and the most specific match wins -- the longest
+# pattern, with an exact id beating any wildcard. So:
+#
+#   --log-level 'http.*=debug'                 both HTTP blocks, nothing else
+#   --log-level 'http.request=debug'           one block, even alongside a looser http.* rule
+#   --log-level debug --log-level 'transform.*=warning'
+#                                              everything loud, except the jq steps
+#
+# WHY IT IS PER-FAMILY AND NOT PER-STEP: a level is about a block's own chattiness, and the
+# same block is usually several steps. Turning http.* up is asking one implementation to
+# explain itself, which is what an investigation actually wants.
+#
+# This document exists to give the flag something to bite on: three block families, several
+# steps in each, all of them cheap. Run it once plain and once loud and compare the log lines
+# -- and note that the RECORDS are identical either way. Only the logs change; a run's outcome
+# never depends on how much it said about itself.
+#
+# Hop by hop:
+#
+#   fetch_one, fetch_two   two http.request steps, so an http.* pattern has more than one
+#                          thing to affect.
+#   shape, tally           two transform.jq steps, the family to turn DOWN when the HTTP is
+#                          what is being investigated.
+#   settle                 a value.const, a third family, so a wildcard has a boundary.
+#
+# EXPECT THIS RUN TO SUCCEED in about two seconds, whichever levels are set.
+#
+#   dg run --local examples/patterns/log-levels.yaml
+#   dg run --local examples/patterns/log-levels.yaml --log-level 'http.*=debug'
+#   dg run --local examples/patterns/log-levels.yaml --log-level debug --log-level 'transform.*=warning'
+#   dg run --local examples/patterns/log-levels.yaml --log-level 'http.*=debug' | dg format
+#
+# A schedule carries its own log levels too, which is how a nightly firing can be made
+# permanently loud without every ad hoc run being loud with it.
+
+format: dirigent/v1
+kind: pipeline
+code: log-levels
+name: One family, turned up
+description: |
+  `--log-level` is a **run** setting, not a document one: `debug` for everything, or
+  `PATTERN=LEVEL` for one block family, repeatable.
+
+  The pattern is an fnmatch over the block id and the most specific match wins. The records
+  a run emits are the same either way; only the log lines change.
+
+tags: [patterns, http, transform, observability]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+    - value.const
+
+params:
+  type: object
+  properties:
+    dataset:
+      type: string
+      description: Echoed by both calls, so a debug line names which step made which request.
+      default: cases
+
+steps:
+  fetch_one:
+    block: http.request
+    config:
+      url: https://postman-echo.com/get
+      query:
+        dataset: "${params.dataset}"
+        part: "1"
+
+  fetch_two:
+    block: http.request
+    # No edge to fetch_one, so the two run side by side and their log lines interleave --
+    # which is exactly when knowing that every line carries its step name starts to matter.
+    config:
+      url: https://postman-echo.com/get
+      query:
+        dataset: "${params.dataset}"
+        part: "2"
+
+  shape:
+    block: transform.jq
+    depends_on: [fetch_one, fetch_two]
+    config:
+      input:
+        one: "${steps.fetch_one.output.body.args}"
+        two: "${steps.fetch_two.output.body.args}"
+      program: |
+        {parts: [.one.part, .two.part], dataset: .one.dataset}
+
+  tally:
+    block: transform.jq
+    depends_on: [shape]
+    config:
+      input: "${steps.shape.output.value}"
+      program: |
+        {dataset, parts: (.parts | length)}
+
+  settle:
+    block: value.const
+    depends_on: [tally]
+    # A third family, so 'http.*' and 'transform.*' each have something they plainly do not
+    # cover.
+    config:
+      value:
+        note: the run's records do not change with its log level

dirigent_examples/shelves/patterns/outputs-inline-vs-storage.yaml

@@ -0,0 +1,140 @@
+# Where a value lives: in a step's output, in the run's scratch space, or in storage.
+#
+# A value travels in a step's output. That is the whole of it: a step reads
+# ${steps.x.output.field} from a step it depends on, and no block reads or writes storage to
+# move a value around. The two ends of a run are the exception, and they are blocks of their
+# own -- storage.read is the only way a value comes in from storage, and storage.write the
+# only way one goes out.
+#
+# THE ENGINE'S OWN DECISION is separate from both, and it is about size rather than about
+# what a document said. Every successful attempt keeps its output whole, whatever its size,
+# so a reference always resolves. Alongside it the engine files a copy: at or below
+# inline_artifact_max (16384 bytes by default) the copy is inlined, and above it the
+# canonical JSON is streamed to the run's scratch prefix and the attempt records the URI it
+# went to. Nothing in this document chooses that, and nothing downstream has to know which
+# happened.
+#
+#   dg runs show <run-id>          the attempt says where its output's copy went
+#   DIRIGENT_INLINE_ARTIFACT_MAX   the instance setting that moves the line
+#
+# WHERE A URI MAY POINT. ${run.scratch} is this run's own directory under the instance's
+# artifact root. file:// URIs are refused outside that root, so a stored pipeline can never
+# be turned into an arbitrary-file read, and a run cannot write over another run's work. An
+# s3:// URI goes to whichever connection the instance binds that scheme to.
+#
+# Hop by hop:
+#
+#   fetch      an http.request. The answer is a few hundred bytes, so its copy is inlined,
+#              and the next step reads .body straight out of the output.
+#   bulk       jq builds four thousand rows from it. The serialised output is far over the
+#              default cap, so the engine spills that copy to the run's scratch.
+#   count      reads ${steps.bulk.output.value} anyway. A spilled copy changes nothing for a
+#              reference: the attempt kept the value whole.
+#   keep       storage.write, the one way out. The rows become an object with a URI.
+#   read_back  storage.read, the one way back in. The object becomes a value again.
+#   compare    the small answer and the round-tripped rows, side by side.
+#
+# EXPECT THIS RUN TO SUCCEED in about three seconds, with bulk's output spilled and every
+# reference to it resolving regardless.
+#
+#   dg run --local examples/patterns/outputs-inline-vs-storage.yaml
+
+format: dirigent/v1
+kind: pipeline
+code: outputs-inline-vs-storage
+name: A value, and the storage on either side of it
+description: |
+  A value travels in a step's output. `storage.read` is the only way one comes in from
+  storage and `storage.write` the only way one goes out.
+
+  Where the engine files its copy of an output is a separate question, decided by size:
+  at or below `inline_artifact_max` (16KB by default) it is inlined on the attempt, and
+  above it spilled to the run's scratch space. A reference resolves either way.
+
+tags: [patterns, http, storage, transform, outputs]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+    - storage.write
+    - storage.read
+
+params:
+  type: object
+  properties:
+    dataset:
+      type: string
+      description: Echoed back by the endpoint, so the answer has something recognisable in it.
+      default: cases
+    rows:
+      type: integer
+      description: How many rows the bulk step builds. Above about three hundred the output spills.
+      default: 4000
+      minimum: 1
+
+steps:
+  fetch:
+    block: http.request
+    config:
+      url: https://postman-echo.com/get
+      query:
+        dataset: "${params.dataset}"
+
+  bulk:
+    block: transform.jq
+    depends_on: [fetch]
+    config:
+      input:
+        dataset: "${steps.fetch.output.body.args.dataset}"
+        rows: "${params.rows}"
+      program: |
+        . as {$dataset, $rows} | [range($rows) | {row: ., dataset: $dataset, seen_at: "postman-echo"}]
+
+  count:
+    block: transform.jq
+    depends_on: [bulk]
+    config:
+      # The reference is the same reference it would be for a four-field output. Whether the
+      # engine inlined the copy or spilled it is not something a document can see.
+      input: "${steps.bulk.output.value}"
+      program: |
+        {rows: length, first: .[0]}
+
+  keep:
+    block: storage.write
+    depends_on: [bulk]
+    config:
+      target: "${run.scratch}/rows/${params.dataset}.json"
+      value: "${steps.bulk.output.value}"
+
+  read_back:
+    block: storage.read
+    depends_on: [keep]
+    config:
+      source: "${steps.keep.output.uri}"
+      # This step's own guard on how much it will hold. A humane size, and an object bigger
+      # than it fails the step rather than exhausting the worker; bytes nobody has to look
+      # at move with storage.copy instead, which this does not bound.
+      max_size: 8mb
+
+  compare:
+    block: transform.jq
+    depends_on: [fetch, count, keep, read_back]
+    config:
+      input:
+        inline_dataset: "${steps.fetch.output.body.args.dataset}"
+        counted: "${steps.count.output.value.rows}"
+        written_uri: "${steps.keep.output.uri}"
+        written_bytes: "${steps.keep.output.bytes_written}"
+        read_bytes: "${steps.read_back.output.bytes_read}"
+        round_tripped: "${steps.read_back.output.value}"
+      program: |
+        {
+          inline_dataset,
+          counted,
+          written_uri,
+          written_bytes,
+          read_bytes,
+          same: (.counted == (.round_tripped | length))
+        }