dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/recipes/filter-by-predicate.yaml

@@ -0,0 +1,116 @@
+# Keep the rows a predicate answers true for, and nothing else about them changes.
+#
+# In:  a list of stations with a status, a battery level, and a last-seen timestamp.
+# Out: the subset that is reporting and healthy, elements untouched, in input order.
+#
+# filter.jq is select as a contract. The program answers one question about one element, and
+# the frame appends the element it was given rather than anything the program returned -- so
+# a filter cannot edit a row on the way past, however the program is written. Order is the
+# input's, and the output is always a subset.
+#
+# The rule that catches everybody: jq's truthiness is not applied here. A program that
+# answers 0, "" or null is a mistake surfaced rather than an element quietly dropped, and
+# the step fails saying so:
+#
+#   element 0: the program answered 4, and a filter answers true or false; jq's truthiness
+#   is not applied, so a program meaning 'has readings' writes '.count > 0'
+#
+# So a predicate is written as a comparison, always. The three below are the shapes most
+# predicates take: a set membership with IN, a numeric threshold, and a presence check that
+# distinguishes absent from null.
+#
+# The count step afterwards is the other half of filtering: a filter that reports nothing
+# leaves "how many did it drop" to somebody reading two artifacts side by side.
+#
+# To change it: -p min_battery=90 tightens the threshold; -p statuses='["active","standby"]'
+# widens the set the membership test reads.
+#
+#   dg run --local examples/recipes/filter-by-predicate.yaml
+#   dg run --local examples/recipes/filter-by-predicate.yaml -p min_battery=90
+
+format: dirigent/v1
+kind: pipeline
+code: filter-by-predicate
+name: Filter with a predicate
+description: Keep the elements a true-or-false program accepts, unmodified and in order, and report how many were dropped.
+
+tags: [recipes, transform, filter]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+    - filter.jq
+
+params:
+  type: object
+  properties:
+    min_battery:
+      type: integer
+      minimum: 0
+      default: 20
+      description: The battery percentage below which a station is not considered healthy.
+    statuses:
+      type: array
+      default: [active]
+      items:
+        type: string
+      description: The statuses that count as reporting.
+
+steps:
+  stations:
+    block: value.const
+    config:
+      value:
+        - { id: st-1, status: active, battery: 96, last_seen: "2026-01-01T06:00:00Z" }
+        - { id: st-2, status: retired, battery: 100, last_seen: "2025-11-02T06:00:00Z" }
+        - { id: st-3, status: active, battery: 11, last_seen: "2026-01-01T05:00:00Z" }
+        - { id: st-4, status: standby, battery: 80, last_seen: "2026-01-01T04:00:00Z" }
+        # No battery reading at all: absent is not the same as flat.
+        - { id: st-5, status: active, last_seen: "2026-01-01T06:00:00Z" }
+
+  attach:
+    block: transform.jq
+    depends_on: [stations]
+    config:
+      # The predicate's thresholds are data, so they travel with each element; a filter
+      # program sees one element and nothing else.
+      input:
+        rows: ${steps.stations.output.value}
+        min_battery: ${params.min_battery}
+        statuses: ${params.statuses}
+      program: |
+        . as {$rows, $min_battery, $statuses}
+        | [$rows[] | . + {_min_battery: $min_battery, _statuses: $statuses}]
+
+  healthy:
+    block: filter.jq
+    depends_on: [attach]
+    config:
+      input: ${steps.attach.output.value}
+      # Every clause is a comparison: IN answers a boolean, the threshold answers a boolean,
+      # and the presence check answers a boolean. Nothing here relies on truthiness.
+      #
+      # The element is named first because IN evaluates its argument against whatever was
+      # piped into it -- a bare .status | IN(._statuses[]) would look for _statuses on the
+      # status string and fail the element.
+      program: |
+        . as $row
+        | ($row.status | IN($row._statuses[]))
+        and ($row.battery != null)
+        and ($row.battery >= $row._min_battery)
+
+  report:
+    block: transform.jq
+    depends_on: [attach, healthy]
+    config:
+      input:
+        seen: ${steps.attach.output.value}
+        kept: ${steps.healthy.output.value}
+      program: |
+        {seen: (.seen | length),
+         kept: (.kept | length),
+         dropped: [(.seen | map(.id)) - (.kept | map(.id)) | .[]],
+         # The bookkeeping fields the predicate needed are removed here rather than in the
+         # filter, which is not allowed to modify what it keeps.
+         rows: [.kept[] | del(._min_battery, ._statuses)]}

dirigent_examples/shelves/recipes/filter-then-map-then-reduce.yaml

@@ -0,0 +1,109 @@
+# The three verbs in the order a pipeline usually needs them: keep, convert, summarise.
+#
+# In:  raw meter readings, some of them from a decommissioned meter or flagged suspect.
+# Out: {kept, converted, summary} -- and a summary computed by reduce over the converted
+#      rows.
+#
+# Each step is the verb whose promise matches what it is doing, and the promise is what the
+# document tells a reader without their having to read the program:
+#
+#   filter.jq     the output is a subset of the input and the elements are untouched. A
+#                 reader knows no value was edited here without reading the predicate.
+#   map.jq        the output has one element per input element, in order. A reader knows the
+#                 batch did not change size here without reading the program.
+#   transform.jq  anything else. Summarising changes the shape from a list to an object, so
+#                 it is the only one of the three that can do it.
+#
+# Written as one transform.jq the result would be identical and the document would say
+# nothing. The cost of the split is two more steps; the payoff is that each step's failure
+# names the element it happened on, and that a run's step list reads as what was done.
+#
+# reduce is the summariser here rather than three passes of add, because the accumulator
+# needs to see each row once to keep a count, a total and a maximum together. add over three
+# mapped lists is fine and reads well; reduce is what to reach for once the accumulator
+# holds more than one number.
+#
+# To change it: -p suspect_is_kept=true keeps the flagged rows, which moves both the count
+# and the mean and is the fastest way to see how much a quality rule is worth.
+#
+#   dg run --local examples/recipes/filter-then-map-then-reduce.yaml
+#   dg run --local examples/recipes/filter-then-map-then-reduce.yaml -p suspect_is_kept=true
+
+format: dirigent/v1
+kind: pipeline
+code: filter-then-map-then-reduce
+name: Filter, then map, then reduce
+description: Keep the usable readings, convert each one, and fold the result into a summary with reduce.
+
+tags: [recipes, transform, filter, map]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+    - map.jq
+    - filter.jq
+
+params:
+  type: object
+  properties:
+    suspect_is_kept:
+      type: boolean
+      default: false
+      description: Keep the readings flagged suspect instead of dropping them.
+
+steps:
+  readings:
+    block: value.const
+    config:
+      value:
+        - { meter: m-1, at: "2026-01-01T06:00:00Z", kwh: 12.5, state: live, suspect: false }
+        - { meter: m-2, at: "2026-01-01T06:00:00Z", kwh: 0.0, state: decommissioned, suspect: false }
+        - { meter: m-3, at: "2026-01-01T06:00:00Z", kwh: 9.25, state: live, suspect: true }
+        - { meter: m-4, at: "2026-01-01T06:00:00Z", kwh: 31.0, state: live, suspect: false }
+        - { meter: m-5, at: "2026-01-01T06:00:00Z", kwh: 4.75, state: live, suspect: false }
+
+  attach:
+    block: transform.jq
+    depends_on: [readings]
+    config:
+      input:
+        rows: ${steps.readings.output.value}
+        keep_suspect: ${params.suspect_is_kept}
+      program: |
+        . as {$rows, $keep_suspect}
+        | [$rows[] | . + {_keep_suspect: $keep_suspect}]
+
+  kept:
+    block: filter.jq
+    depends_on: [attach]
+    config:
+      input: ${steps.attach.output.value}
+      program: |
+        .state == "live" and (._keep_suspect or (.suspect | not))
+
+  converted:
+    block: map.jq
+    depends_on: [kept]
+    config:
+      input: ${steps.kept.output.value}
+      # A megajoule is 3.6 kWh, so this is the unit conversion a sink asked for; the map is
+      # where it belongs because it touches one row at a time and changes no row count.
+      program: |
+        {meter, at, megajoules: (.kwh * 3.6 | . * 100 | round / 100)}
+
+  summary:
+    block: transform.jq
+    depends_on: [converted]
+    config:
+      input: ${steps.converted.output.value}
+      program: |
+        . as $rows
+        | reduce $rows[] as $row
+            ({meters: 0, megajoules: 0, peak: null};
+             {meters: (.meters + 1),
+              megajoules: (.megajoules + $row.megajoules),
+              peak: (if .peak == null or $row.megajoules > .peak.megajoules
+                     then $row else .peak end)})
+        | . + {mean_megajoules: (if .meters == 0 then null
+                                 else (.megajoules / .meters * 100 | round / 100) end)}

dirigent_examples/shelves/recipes/http-fetch-validate-post.yaml

@@ -0,0 +1,142 @@
+# Fetch, check what came back, reshape it, check what you made, and post it on.
+#
+# This is the shape most real pipelines have, and the reason it is written out in full here is
+# the second gate. One gate, at the fetch, is the common half-measure: it catches the day the
+# source changes and catches nothing else. The second gate catches the other failure, which is
+# the pipeline's own -- a jq program edited under time pressure, a field renamed in one place
+# and not the other -- and it catches it before the bad row leaves the building.
+#
+# In:  a station and a day, as parameters.
+# Out: one checked row, posted to a public echo service, and a receipt saying what it saw.
+#
+# Five hops, and what each one hands on:
+#
+#   fetch     http.request. A GET whose query carries the two parameters, answered by Postman
+#             Echo, a public request-and-response service, so this runs against something real
+#             with nothing to stand up first. Output: status, headers, body, body_bytes.
+#   checked   validate.schema against `echo-reading`: the source's shape, as this document
+#             understands it. The block passes the value through unchanged when it holds, so
+#             the next step reads the gate's output rather than reaching back past it -- which
+#             is what makes the gate impossible to skip by accident.
+#   reading   transform.jq. The reshaping: the query the echo service reflected back becomes
+#             the row a receiver wants, and the temperature becomes a number, because a query
+#             string carries text and a downstream average cannot add text.
+#   verified  validate.schema again, against `station-reading` this time: the shape this
+#             document promises, with additionalProperties false so a field the program leaks
+#             is refused rather than shipped.
+#   publish   http.request POST. The body is the value the gate passed, serialised by the
+#             block: nothing here builds JSON out of strings.
+#
+# BOTH SCHEMAS ARE NAMED, NOT CARRIED. `schema:` takes a code and never a shape, and a document
+# that carries its own top-level `schemas:` section is refused by a server, so the two live in
+# examples/schemas/ and `requires.schemas` below says the instance must hold them. That is the
+# preflight: an apply against an instance missing either one is refused up front, naming what
+# is missing, instead of failing on the first run.
+#
+#   dg schema create examples/schemas/echo-reading.json
+#   dg schema create examples/schemas/station-reading.json
+#   dg apply examples/recipes/http-fetch-validate-post.yaml
+#   dg run http-fetch-validate-post --watch
+#
+# Locally there is no instance holding them, so hand them over on the command line:
+#
+#   dg run --local examples/recipes/http-fetch-validate-post.yaml \
+#     --schema examples/schemas/echo-reading.json \
+#     --schema examples/schemas/station-reading.json
+#
+# TO MAKE IT YOURS: point `fetch` at your source, rewrite the two schemas for its shape and
+# yours, and point `publish` at your receiver. The five hops do not move; a real pipeline is
+# this with a longer jq program and a connection holding the credential.
+
+format: dirigent/v1
+kind: pipeline
+code: http-fetch-validate-post
+name: Fetch, validate, transform, validate, post
+description: |
+  The full shape: a GET from a public endpoint, a schema gate on what came back, a jq
+  reshaping, a second gate on what was made, and a POST of the result.
+
+  The second gate is the point. The first one catches the source changing; the second catches
+  this document's own program going wrong, before the bad row is delivered.
+
+tags: [recipes, http, transform, validate, starter]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+    - validate.schema
+  # The preflight that refuses this document on an instance holding neither shape.
+  schemas:
+    - echo-reading
+    - station-reading
+
+params:
+  type: object
+  properties:
+    station:
+      type: string
+      minLength: 1
+      default: st-1
+      description: The station the reading is attributed to.
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the reading reports for.
+    celsius:
+      type: string
+      default: "12.5"
+      description: The temperature, as the text a query string carries; the transform makes it a number.
+
+steps:
+  fetch:
+    block: http.request
+    config:
+      url: https://postman-echo.com/get
+      method: GET
+      query:
+        station: ${params.station}
+        day: ${params.day}
+        celsius: ${params.celsius}
+      timeout: 30s
+
+  checked:
+    block: validate.schema
+    depends_on: [fetch]
+    config:
+      input: ${steps.fetch.output.body}
+      # A code naming a schema the instance holds, never a shape written here.
+      schema: echo-reading
+
+  reading:
+    block: transform.jq
+    depends_on: [checked]
+    config:
+      input: ${steps.checked.output.value}
+      program: |
+        .args
+        | {station: .station,
+           day: .day,
+           # tonumber, because a query string carried this as text and a receiver that
+           # averages it cannot add text.
+           celsius: (.celsius | tonumber)}
+
+  verified:
+    block: validate.schema
+    depends_on: [reading]
+    config:
+      input: ${steps.reading.output.value}
+      schema: station-reading
+
+  publish:
+    block: http.request
+    depends_on: [verified]
+    config:
+      url: https://postman-echo.com/post
+      method: POST
+      # The value the gate passed through, serialised by the block.
+      body: ${steps.verified.output.value}
+      headers:
+        x-reading-day: ${params.day}
+      timeout: 30s

dirigent_examples/shelves/recipes/http-follow-redirects.yaml

@@ -0,0 +1,100 @@
+# Redirects are not followed unless a step says so, and both answers are useful.
+#
+# In:  a redirect status and a target, as parameters.
+# Out: the same call made twice -- once returning the 3xx itself, once following it to the
+#      end -- so the difference is one output rather than an argument.
+#
+# follow_redirects defaults to false, and the default is the interesting choice. A redirect
+# is information: it says the resource moved, or that a load balancer wants a different
+# host, or that an unauthenticated call is being sent to a login page. Following it
+# silently turns "your URL is wrong" into a 200 that a pipeline treats as data, and the
+# classic version of that is a 302 to an HTML sign-in page parsed as an empty result.
+#
+# So the block returns the 3xx as the answer, with its Location header, and a step that
+# genuinely wants the destination asks for it:
+#
+#   follow_redirects: false   status is 302 and the body is the redirect's own. Location
+#                             says where it points, which is a thing a pipeline can report
+#                             or refuse.
+#   follow_redirects: true    the client follows, and status is the destination's. Nothing
+#                             says a redirect happened, so this is the setting for an API
+#                             that is known to redirect as part of its normal working.
+#
+# A 3xx is not a 2xx, so the un-followed call needs the status listed as success or the
+# step fails -- which is success_status doing exactly what http-success-status-list.yaml
+# describes, on the one status that most deserves it.
+#
+# To change it: -p status=301 makes it a permanent move, which is the one worth reporting
+# rather than following, since the URL in the document is now wrong.
+#
+#   dg run --local examples/recipes/http-follow-redirects.yaml
+#   dg run --local examples/recipes/http-follow-redirects.yaml -p status=301
+
+format: dirigent/v1
+kind: pipeline
+code: http-follow-redirects
+name: Following a redirect, or not
+description: Make the same redirecting call twice, with and without follow_redirects, and compare the status and the Location header.
+
+tags: [recipes, http, transform]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    status:
+      type: integer
+      default: 302
+      description: The redirect status the endpoint answers with.
+    target:
+      type: string
+      default: https://postman-echo.com/get
+      description: Where the redirect points.
+
+steps:
+  unfollowed:
+    block: http.request
+    config:
+      url: https://postman-echo.com/redirect-to
+      method: GET
+      query:
+        url: ${params.target}
+        status_code: ${params.status}
+      # The default, written out because this document is about it.
+      follow_redirects: false
+      # A 3xx is not a 2xx, so the status the call is asking for has to be listed.
+      success_status: [301, 302, 307, 308]
+      timeout: 20s
+
+  followed:
+    block: http.request
+    config:
+      url: https://postman-echo.com/redirect-to
+      method: GET
+      query:
+        url: ${params.target}
+        status_code: ${params.status}
+      follow_redirects: true
+      timeout: 20s
+
+  compare:
+    block: transform.jq
+    depends_on: [unfollowed, followed]
+    config:
+      input:
+        unfollowed_status: ${steps.unfollowed.output.status}
+        unfollowed_headers: ${steps.unfollowed.output.headers}
+        followed_status: ${steps.followed.output.status}
+        followed_body: ${steps.followed.output.body}
+      program: |
+        {unfollowed: {status: .unfollowed_status,
+                      # Where it pointed: the thing a pipeline can report or refuse.
+                      location: .unfollowed_headers.location},
+         followed: {status: .followed_status,
+                    # The destination answered, and nothing in the response says a redirect
+                    # happened on the way.
+                    url: .followed_body.url}}

dirigent_examples/shelves/recipes/http-get-with-query.yaml

@@ -0,0 +1,96 @@
+# A GET with query parameters, and what to do with the answer.
+#
+# In:  a station code and a day, as parameters.
+# Out: the echo service's view of the request -- the query it received -- reshaped into a
+#      small record, plus the status and how long the call took.
+#
+# https://postman-echo.com/get answers with the request it was given, so a recipe about
+# building a request can show what actually arrived on the wire without standing anything
+# up. Against a real API the only lines that change are url or connection, and the path.
+#
+# The query is a map, not a string, and that is the whole point of the field: the block
+# encodes each value once, correctly, so a station code with a space or an ampersand in it
+# cannot break out of its parameter. A path with ?a=1&b=2 hand-written into it is where
+# injection and double-encoding both come from.
+#
+# Values may be strings, numbers or booleans -- paging=false goes over the wire as the text
+# `false`, because that is what a query string is -- and a reference resolves before the
+# encoding happens, so ${params.day} is a value and never a fragment of URL.
+#
+# The answer is one field. body carries the parsed document when the service answered JSON
+# and the text when it answered anything else, and body_bytes says how big it was.
+# max_response bounds what the step will hold: a body past it is refused rather than read
+# half way. A body worth keeping is handed to a storage.write step, which is what
+# http-save-body-to-storage.yaml does with this same call.
+#
+# duration_ms is on every response, so a pipeline can report what a call cost without
+# timing it itself.
+#
+# To change it: -p station=st-9, or point url at your own endpoint and keep the rest.
+#
+#   dg run --local examples/recipes/http-get-with-query.yaml
+#   dg run --local examples/recipes/http-get-with-query.yaml -p station=st-9 -p day=2026-02-01
+
+format: dirigent/v1
+kind: pipeline
+code: http-get-with-query
+name: A GET with query parameters
+description: Build a GET whose query is a map rather than a hand-written string, and reshape the JSON body it answers with.
+
+tags: [recipes, http, transform]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    station:
+      type: string
+      default: st-1
+      description: The station the request asks about.
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the request asks about.
+    page_size:
+      type: integer
+      minimum: 1
+      default: 50
+      description: A numeric query value, sent as its digits.
+
+steps:
+  fetch:
+    block: http.request
+    config:
+      # An absolute url, for the case where no connection is configured. A pipeline that
+      # runs against an instance names a connection instead, so moving from staging to
+      # production edits the connection and no document.
+      url: https://postman-echo.com/get
+      method: GET
+      query:
+        station: ${params.station}
+        day: ${params.day}
+        page_size: ${params.page_size}
+        # A boolean goes on the wire as the word, because a query string has no types.
+        paging: false
+      timeout: 20s
+
+  received:
+    block: transform.jq
+    depends_on: [fetch]
+    config:
+      input:
+        status: ${steps.fetch.output.status}
+        duration_ms: ${steps.fetch.output.duration_ms}
+        # The parsed body. The echo service puts what it received under .args.
+        body: ${steps.fetch.output.body}
+      program: |
+        {status, duration_ms,
+         url: .body.url,
+         # Every query value arrives back as text, because that is what a query string is.
+         query: .body.args,
+         page_size_is_text: (.body.args.page_size | type)}

dirigent_examples/shelves/recipes/http-headers-and-auth-connection.yaml

@@ -0,0 +1,110 @@
+# Credentials live on a connection, not in the document. Here is what that looks like.
+#
+# In:  nothing but parameters.
+# Out: {authenticated: true} from an endpoint that refuses an unauthenticated caller with a
+#      401, proving the connection's credentials were applied.
+#
+# A document is portable precisely because it names a credential rather than carrying one.
+# On a server the connection is created once and the document says only its code:
+#
+#   dg connection create http postman-echo \
+#     --set base_url=https://postman-echo.com \
+#     --set basic_username=postman --set basic_password=password
+#
+# basic_password is a SecretStr: sealed on the way in, encrypted at rest, and redacted in
+# every API response -- an applied connection reports which fields are set, never what they
+# are set to.
+#
+# This document carries its connection inline instead, which is the exception rather than
+# the pattern, and it is labelled below with why: the credentials are Postman's own public
+# demo pair, so the recipe runs standalone with nothing to hand it. Real credentials never
+# go in a file that lives in git. The instance will not store a document that carries a
+# connection, either, which is the format making that hard to get wrong.
+#
+# What the connection contributes, once named:
+#
+#   base_url          the step gives a path, and the connection decides which host it is
+#                     resolved against. Staging to production is a connection edit and no
+#                     document change.
+#   basic_username    the Authorization header, built by the block. Nothing in the document
+#   basic_password    ever spells out `Basic ...`, and no secret reaches a header map.
+#   timeout           the default deadline for every call on this connection, which a step
+#                     may override for one call.
+#
+# headers is for what the receiver wants that is not a credential: a routing key, an API
+# version, a correlation id. A token in there is a token in the document, which is the
+# mistake this whole surface exists to prevent.
+#
+# To change it: -p path=/get calls an endpoint that takes no credentials at all and echoes
+# the headers it saw, including the Authorization the connection added.
+#
+#   dg run --local examples/recipes/http-headers-and-auth-connection.yaml
+#   dg run --local examples/recipes/http-headers-and-auth-connection.yaml -p path=/get
+
+format: dirigent/v1
+kind: pipeline
+code: http-headers-and-auth-connection
+name: Auth on a connection, headers in the step
+description: Call a basic-auth endpoint through a connection that holds the credentials, with the step contributing only non-secret headers.
+
+tags: [recipes, http, transform]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+
+# Carried inline so this recipe runs standalone. The pair is Postman's own public demo
+# credential, documented at https://postman-echo.com; a real one is created on the instance
+# and named by code, and an instance refuses to store a document that carries a connection
+# at all.
+connections:
+  echo-basic:
+    kind: http
+    config:
+      base_url: https://postman-echo.com
+      basic_username: postman
+      basic_password: password
+      timeout: 30s
+      # What dg connection check calls to answer green or red.
+      health_path: /get
+
+params:
+  type: object
+  properties:
+    path:
+      type: string
+      default: /basic-auth
+      description: The path resolved against the connection's base URL.
+    correlation_id:
+      type: string
+      default: recipes-0001
+      description: A non-secret header the receiver wants, sent by the step.
+
+steps:
+  call:
+    block: http.request
+    config:
+      # A code, resolved by the instance. The host, the credentials and the default timeout
+      # all come from it.
+      connection: echo-basic
+      path: ${params.path}
+      method: GET
+      headers:
+        # Not a credential: a routing and tracing header, which is what this map is for.
+        x-correlation-id: ${params.correlation_id}
+        accept: application/json
+
+  result:
+    block: transform.jq
+    depends_on: [call]
+    config:
+      input:
+        status: ${steps.call.output.status}
+        body: ${steps.call.output.body}
+      program: |
+        # /basic-auth answers {"authenticated": true}; /get echoes the request instead, and
+        # its .headers shows the Authorization header the connection contributed.
+        {status,
+         authenticated: (.body.authenticated // null),
+         saw_authorization: ((.body.headers.authorization // null) != null)}