dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/recipes/reconcile-two-sources.yaml

@@ -0,0 +1,159 @@
+# Two systems that should agree, and a run that says exactly how they do not.
+#
+# In:  the same day's totals from a source of record and from a warehouse, each with rows
+#      the other does not have.
+# Out: {matched, only_in_source, only_in_target, differing, totals} and a run that fails
+#      when the disagreement is bigger than the tolerance.
+#
+# A reconciliation has exactly four buckets, and a report that does not name all four is not
+# a reconciliation:
+#
+#   in both, equal        the boring bucket, and the one worth counting so the others have a
+#                         denominator.
+#   in both, different    the interesting bucket. Each row carries both values and their
+#                         difference, because "17 rows differ" cannot be investigated.
+#   only in the source    something that has not arrived yet, or a load that dropped it.
+#   only in the target    something deleted upstream, or loaded twice under a changed key.
+#
+# The comparison is by key, so both sides are indexed first with INDEX and the key set is
+# the union of the two -- not the source's keys, which would make anything extra in the
+# target invisible.
+#
+# Floating point is the trap in the equality test. Two systems that both computed 0.1 + 0.2
+# do not agree on it, so the comparison is a tolerance rather than ==, and the tolerance is
+# a parameter because what counts as equal is a property of the data and not of the code.
+#
+# The verdict is the last decision: a difference that is within tolerance is not a
+# difference, a small number of differing rows is a warning somebody reads, and past the
+# threshold the run fails so that a schedule stops rather than reporting quietly for a week.
+#
+# To change it: -p max_differing=5 lets this run pass with its three differing rows; the
+# default of 2 is below that, so the default run fails on purpose.
+#
+#   dg run --local examples/recipes/reconcile-two-sources.yaml                  # fails, on purpose
+#   dg run --local examples/recipes/reconcile-two-sources.yaml -p max_differing=5
+
+format: dirigent/v1
+kind: pipeline
+code: reconcile-two-sources
+name: Reconcile two sources
+description: Compare two systems' totals by key with a tolerance, report the four buckets, and fail when too many rows disagree.
+
+tags: [recipes, transform]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day both sides are reporting on.
+    tolerance:
+      type: number
+      minimum: 0
+      default: 0.01
+      description: How far two values may differ and still count as equal.
+    max_differing:
+      type: integer
+      minimum: 0
+      default: 2
+      description: How many differing rows are tolerated before the run fails.
+
+steps:
+  source:
+    block: value.const
+    config:
+      value:
+        - { key: east/pump, total: 300.0 }
+        - { key: east/valve, total: 250.0 }
+        - { key: west/pump, total: 100.0 }
+        - { key: west/valve, total: 100.0 }
+        # In the source and not in the warehouse: not loaded yet, or dropped.
+        - { key: north/pump, total: 200.0 }
+        - { key: south/pump, total: 42.5 }
+
+  target:
+    block: value.const
+    config:
+      value:
+        - { key: east/pump, total: 300.0 }
+        # Differs by more than the tolerance.
+        - { key: east/valve, total: 247.5 }
+        # Differs by less: floating-point noise, not a disagreement.
+        - { key: west/pump, total: 100.000000001 }
+        - { key: west/valve, total: 90.0 }
+        - { key: south/pump, total: 41.0 }
+        # In the warehouse and not in the source: deleted upstream, or loaded twice.
+        - { key: east/gasket, total: 12.0 }
+
+  compare:
+    block: transform.jq
+    depends_on: [source, target]
+    config:
+      input:
+        source: ${steps.source.output.value}
+        target: ${steps.target.output.value}
+        tolerance: ${params.tolerance}
+        day: ${params.day}
+      program: |
+        . as {$source, $target, $tolerance, $day}
+        | ($source | INDEX(.key)) as $left
+        | ($target | INDEX(.key)) as $right
+        # The union of both key sets: the source's keys alone would hide anything extra in
+        # the target.
+        | (([$source[].key] + [$target[].key]) | unique) as $keys
+        | [$keys[]
+           | . as $key
+           | {key: $key,
+              source: ($left[$key].total // null),
+              target: ($right[$key].total // null)}
+           | . + {bucket: (if .source == null then "only_in_target"
+                           elif .target == null then "only_in_source"
+                           # A tolerance, not ==: two systems that both computed the same
+                           # sum in floating point do not agree on its last bits.
+                           elif ((.source - .target) | fabs) <= $tolerance then "matched"
+                           else "differing" end),
+                  difference: (if .source == null or .target == null then null
+                               else ((.source - .target) * 1000 | round / 1000) end)}]
+          as $rows
+        | {day: $day,
+           tolerance: $tolerance,
+           matched: [$rows[] | select(.bucket == "matched") | .key],
+           only_in_source: [$rows[] | select(.bucket == "only_in_source") | .key],
+           only_in_target: [$rows[] | select(.bucket == "only_in_target") | .key],
+           # Both values and the difference, because a count of differing rows cannot be
+           # investigated by anybody.
+           differing: [$rows[] | select(.bucket == "differing")
+                       | {key, source, target, difference}],
+           totals: {source: ([$source[].total] | add),
+                    target: ([$target[].total] | add),
+                    keys: ($keys | length)}}
+
+  verdict:
+    block: transform.jq
+    depends_on: [compare]
+    config:
+      input:
+        report: ${steps.compare.output.value}
+        max_differing: ${params.max_differing}
+      program: |
+        . as {$report, $max_differing}
+        | ($report.differing | length) as $differing
+        # Past the threshold the run fails, so a schedule stops instead of reporting a
+        # widening gap quietly every morning.
+        | if $differing > $max_differing
+          then error("\($differing) rows differ beyond the tolerance, and \($max_differing) "
+                     + "is the limit: \([$report.differing[].key] | join(", "))")
+          else {day: $report.day,
+                differing: $differing,
+                matched: ($report.matched | length),
+                only_in_source: $report.only_in_source,
+                only_in_target: $report.only_in_target,
+                gap: (($report.totals.source - $report.totals.target) * 100 | round / 100)}
+          end

dirigent_examples/shelves/recipes/report-built-in.yaml

@@ -0,0 +1,72 @@
+# Ask for a run's report and write no template at all.
+#
+# In:  five sales rows, inline.
+# Out: a total per region, and the run's own report document -- the built-in one, because
+#      the report: section is empty.
+#
+# `report: {}` is the whole of the opt-in. A document with no report: section renders
+# nothing; an empty section renders the page the engine ships:
+#
+#   a title    the pipeline's name, or its code when it has none, and the run's status
+#   the facts  run id, pipeline and version, trigger, started, finished, duration, when the
+#              page was rendered, and the data window when the run has one
+#   the steps  a row per step: outcome, attempts, how long it took, how large its output was,
+#              and how many warnings it logged, which a succeeded step can still have
+#   the items  the elements that failed, when a fan-out step had any, each with the step it
+#              failed at and the error
+#   the error  the run's own error, or a cancellation's reason, when it ended badly
+#
+# That page is written by the engine rather than by a step, which is the point of it: a run
+# that was cancelled, or whose worker died, has no step left to write anything, and those
+# are the runs whose account is worth the most. It is rendered in the transaction that
+# settles the run, and rewritten in place if a retry reopens the run and it settles again.
+#
+# The two steps are here to give the page something to tabulate, and they are the smallest
+# pair that does: a literal, and one jq program over it.
+#
+# To change it: write a `template:` under report: and the page is yours instead --
+# http-post-report.yaml is that document, and docs/reports.md lists every fact a template
+# may read.
+#
+#   dg run --local examples/recipes/report-built-in.yaml
+#   dg apply examples/recipes/report-built-in.yaml
+#   dg run report-built-in --watch
+#   dg runs report RUN_ID --markdown
+
+format: dirigent/v1
+kind: pipeline
+code: report-built-in
+name: The built-in report
+description: The smallest pipeline that renders a report document, with an empty report section and no template of its own.
+
+tags: [recipes, transform, report, starter]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+
+steps:
+  rows:
+    block: value.const
+    config:
+      value:
+        - { region: east, revenue: 300 }
+        - { region: east, revenue: 250 }
+        - { region: west, revenue: 100 }
+        - { region: west, revenue: 100 }
+        - { region: north, revenue: 200 }
+
+  totals:
+    block: transform.jq
+    depends_on: [rows]
+    config:
+      input: ${steps.rows.output.value}
+      program: |
+        {regions: (group_by(.region)
+                   | map({region: .[0].region,
+                          rows: length,
+                          revenue: (map(.revenue) | add)})),
+         revenue: (map(.revenue) | add)}
+
+report: {}

dirigent_examples/shelves/recipes/report-daily-digest.yaml

@@ -0,0 +1,186 @@
+# A daily digest: fetch, aggregate, write a csv somebody opens, and post the summary.
+#
+# In:  a day, as a parameter. The readings are generated so the recipe runs anywhere; the
+#      one HTTP call is the delivery at the end.
+# Out: a csv artifact per region in the run's scratch space, a markdown-ish digest line, and
+#      a delivered notification whose body carries the numbers.
+#
+# This is the shape most reporting pipelines end up with, and the reason each piece is
+# separate:
+#
+#   window     the day's boundaries are computed once, from the parameter and not from the
+#              clock, so a rerun for last Tuesday reports last Tuesday. A pipeline that
+#              reads `now` cannot be rerun, and being rerunnable is most of what makes an
+#              incident survivable.
+#   readings   stands in for the source read. Replacing it with an http.request whose body
+#              this step reads changes this step and nothing below it.
+#   aggregate  the numbers, computed once, in one place. Everything below reads them rather
+#              than recomputing, so the csv, the digest line and the notification can never
+#              disagree.
+#   rows_file  the per-region numbers as an object in the run's scratch space. A conversion
+#              reads a URI and writes a URI, so the value goes out through storage.write
+#              before the codec sees it.
+#   csv        the artifact somebody actually opens, converted from that object under a
+#              dated name.
+#   digest     the human sentence, derived from the aggregate.
+#   notify     the delivery, on rule all_done so a failed report still says something.
+#
+# A digest has to say what it does not know. The empty_regions field is there because a
+# region that reported nothing looks exactly like a region that reported zero once the
+# averages are taken, and only the pipeline knows the difference.
+#
+# To change it: -p day=2026-02-03 reruns for another day; -p regions='["east"]' narrows it.
+#
+#   dg run --local examples/recipes/report-daily-digest.yaml
+#   dg run --local examples/recipes/report-daily-digest.yaml -p day=2026-02-03
+
+format: dirigent/v1
+kind: pipeline
+code: report-daily-digest
+name: A daily digest report
+description: Compute a day's numbers once, write them to storage and convert them into a csv artifact, derive a digest line, and post the summary to a receiver.
+
+tags: [recipes, storage, transform, webhook]
+
+requires:
+  blocks:
+    - transform.jq
+    - storage.write
+    - convert.std
+    - webhook.post
+
+connections:
+  echo-webhook:
+    kind: http
+    config:
+      base_url: https://postman-echo.com
+      timeout: 30s
+      health_path: /get
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day being reported on; every date in the run comes from here.
+    regions:
+      type: array
+      default: [east, west, north]
+      items:
+        type: string
+      description: The regions the digest covers, including any that report nothing.
+
+steps:
+  window:
+    block: transform.jq
+    config:
+      input: ${params.day}
+      program: |
+        . as $day
+        | ($day + "T00:00:00Z" | fromdateiso8601) as $from
+        | {day: $day,
+           start: ($from | todate),
+           end: ($from + 86400 | todate),
+           iso_week: ($from | strftime("%G-W%V"))}
+
+  readings:
+    block: transform.jq
+    depends_on: [window]
+    config:
+      input: ${steps.window.output.value}
+      # Stands in for the source read: an http.request, or a storage.read of yesterday's drop.
+      # north is absent on purpose, so the digest has something to not know.
+      program: |
+        . as {$day}
+        | [range(24) as $hour
+           | ["east", "west"] as $regions
+           | $regions[]
+           | {region: ., day: $day,
+              hour: $hour,
+              celsius: ((($hour * 3) % 19) - 6),
+              station: "\(.)-st-\($hour % 3)"}]
+
+  aggregate:
+    block: transform.jq
+    depends_on: [readings, window]
+    config:
+      input:
+        rows: ${steps.readings.output.value}
+        window: ${steps.window.output.value}
+        regions: ${params.regions}
+      program: |
+        . as {$rows, $window, $regions}
+        | ($rows | group_by(.region)
+           | map({region: .[0].region,
+                  readings: length,
+                  stations: ([.[].station] | unique | length),
+                  mean_celsius: ((map(.celsius) | add) / length * 10 | round / 10),
+                  min_celsius: (map(.celsius) | min),
+                  max_celsius: (map(.celsius) | max)})) as $per_region
+        | {day: $window.day,
+           iso_week: $window.iso_week,
+           regions: $per_region,
+           total_readings: ($rows | length),
+           # A region that reported nothing is not a region that reported zero, and only
+           # this pipeline knows which happened.
+           empty_regions: ($regions - [$per_region[].region])}
+
+  rows_file:
+    block: storage.write
+    depends_on: [aggregate]
+    config:
+      target: ${run.scratch}/digest-${params.day}.json
+      value: ${steps.aggregate.output.value.regions}
+
+  csv:
+    block: convert.std
+    depends_on: [rows_file]
+    config:
+      source: ${steps.rows_file.output.uri}
+      target: ${run.scratch}/digest-${params.day}.csv
+      from: json
+      to: csv
+
+  digest:
+    block: transform.jq
+    depends_on: [aggregate, csv]
+    config:
+      input:
+        summary: ${steps.aggregate.output.value}
+        file: ${steps.csv.output.target}
+      program: |
+        .summary as $s
+        | {day: $s.day,
+           iso_week: $s.iso_week,
+           file: .file,
+           # Derived from the numbers, so it cannot disagree with them.
+           text: "\($s.day) (\($s.iso_week)): \($s.total_readings) readings across "
+                 + "\($s.regions | length) regions"
+                 + (if ($s.empty_regions | length) > 0
+                    then "; no data from \($s.empty_regions | join(", "))"
+                    else "" end),
+           lines: [$s.regions[]
+                   | "\(.region): \(.readings) readings, mean \(.mean_celsius)C "
+                     + "(\(.min_celsius) to \(.max_celsius))"]}
+
+  notify:
+    block: webhook.post
+    depends_on: [digest]
+    rule: all_done
+    config:
+      connection: echo-webhook
+      path: /post
+      body: ${steps.digest.output.value}
+      success_status: [200, 202, 204]
+
+  delivered:
+    block: transform.jq
+    depends_on: [notify]
+    config:
+      input:
+        status: ${steps.notify.output.status}
+        echoed: ${steps.notify.output.json_body}
+      program: |
+        {status, text: .echoed.json.text, lines: .echoed.json.lines, file: .echoed.json.file}

dirigent_examples/shelves/recipes/report-to-file.yaml

@@ -0,0 +1,131 @@
+# Render a page of markdown and put it in a file, then read the file back and log it.
+#
+# In:  three readings written inline.
+# Out: a markdown page at ${run.scratch}/report.md, and its text on the run's own timeline.
+#
+# report.render renders text and hands it on; it writes nothing itself. Sending it
+# somewhere is a step of its own, and which step that is decides where the page goes -- a
+# file here, a bucket in examples/s3/report-to-s3.yaml, a queue in the two under
+# examples/queues/. The template does not change between them.
+#
+# Six hops, and what each one hands on:
+#   readings  value.const, the rows a real pipeline would have fetched.
+#   summary   transform.jq, the numbers the page is about, computed before anything is
+#             worded. A template that does arithmetic is a template nobody can test.
+#   page      report.render, the markdown. Output: text, content_type, text_bytes.
+#   save      storage.write, the text to a URI. Output: uri, bytes_written, content_type.
+#   back      storage.read, the same URI. It is text/markdown, so the object comes back as
+#             `text` rather than as `value`.
+#   show      log.write, the page onto the run's timeline, which is the smallest sink there
+#             is and the easiest way to see that the round trip worked.
+#
+# THE TEMPLATE IS JINJA, and `values` is everything it may read: names it was not given
+# render as the empty string rather than failing the step, so a page missing a fact is still
+# a page. It is compiled when the document is applied, so a syntax error is refused there
+# rather than at the first run. docs/reports.md is the family's home; the same language
+# writes a run's own `report:` section.
+#
+# WHY THE CONTENT TYPE IS SAID TWICE. report.render says what the text is and storage.write
+# records it, so the object carries its type on a backend that can hold one and is guessed
+# from the `.md` extension on one that cannot. storage.read resolves it the same way round:
+# an override, else what the backend stored, else the extension, else bytes it refuses to
+# decode.
+#
+# TO MAKE IT YOURS: change the template, and change where `save` points. Nothing else moves.
+#
+#   dg run --local examples/recipes/report-to-file.yaml
+#   dg run --local examples/recipes/report-to-file.yaml -p day=2026-02-01
+
+format: dirigent/v1
+kind: pipeline
+code: report-to-file
+name: Render a report and write it to a file
+description: Render a markdown page from a Jinja template, write it to the run's scratch space, and read it back.
+
+tags: [recipes, report, storage, transform, starter]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+    - report.render
+    - storage.write
+    - storage.read
+    - log.write
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the report is about.
+
+steps:
+  readings:
+    block: value.const
+    config:
+      value:
+        - {station: st-1, region: east, celsius: 4.5}
+        - {station: st-3, region: west, celsius: 1.2}
+        - {station: st-4, region: north, celsius: -3.4}
+
+  summary:
+    block: transform.jq
+    depends_on: [readings]
+    config:
+      input: ${steps.readings.output.value}
+      # Every number the page shows is computed here, where it can be asserted on.
+      program: |
+        {stations: length,
+         coldest: (min_by(.celsius) | .station),
+         below_zero: [.[] | select(.celsius < 0) | .station],
+         rows: [.[] | {station, region, celsius}]}
+
+  page:
+    block: report.render
+    depends_on: [summary]
+    config:
+      # What the template may read, and the only thing it may read.
+      values:
+        day: ${params.day}
+        summary: ${steps.summary.output.value}
+      content_type: text/markdown
+      template: |
+        # Readings for {{ day }}
+
+        {{ summary.stations }} stations reported, {{ summary.below_zero | length }} below zero.
+        The coldest was {{ summary.coldest }}.
+
+        | Station | Region | Celsius |
+        | --- | --- | --- |
+        {% for row in summary.rows %}
+        | {{ row.station }} | {{ row.region }} | {{ row.celsius }} |
+        {% endfor %}
+
+  save:
+    block: storage.write
+    depends_on: [page]
+    config:
+      # ${run.scratch} is the run's own prefix in whatever storage the instance is
+      # configured with: a temporary directory under --local, the artifact root on a
+      # server, a bucket where an s3 connection serves the scheme.
+      target: ${run.scratch}/report.md
+      text: ${steps.page.output.text}
+      content_type: ${steps.page.output.content_type}
+
+  back:
+    block: storage.read
+    depends_on: [save]
+    config:
+      # Where the write said the object landed, not the same path typed twice.
+      source: ${steps.save.output.uri}
+      max_size: 1mb
+
+  show:
+    block: log.write
+    depends_on: [back]
+    config:
+      message: the report, read back out of storage
+      value: ${steps.back.output.text}

dirigent_examples/shelves/recipes/report-to-webhook.yaml

@@ -0,0 +1,136 @@
+# Render a page and POST it to a receiver, which is how a report reaches a chat client or a
+# ticket without anything on this side storing it.
+#
+# In:  a batch of load results, some of them rejected.
+# Out: a markdown page delivered to Postman Echo, and what the receiver echoed back.
+#
+# Three hops, and what each one hands on:
+#   summary   transform.jq, the counts and the identifiers, computed before anything is
+#             worded, so the prose can never disagree with the numbers.
+#   page      report.render, the markdown. Output: text, content_type, text_bytes.
+#   deliver   webhook.post, a JSON body carrying that text. Output: status, signed,
+#             duration_ms, and the receiver's answer.
+#
+# THE BODY IS AN OBJECT, NOT THE PAGE ALONE. webhook.post sends JSON, so the page travels as
+# one field of a small envelope beside the facts a receiver routes on: the pipeline, the day,
+# the severity, and the content type of the text. A receiver that renders markdown reads
+# `text`; one that only routes reads the fields beside it and never parses the page.
+#
+# WHY report.render RATHER THAN A jq STRING. A page built by string concatenation in jq is a
+# page nobody can read in the document. The template is text, indented as it will be
+# rendered, and the numbers in it come from `values` -- and a name that is not there renders
+# empty rather than failing the delivery.
+#
+# THE DELIVERY IS A STEP IN THE DAG, so a rule decides when it runs. This one is `all_done`,
+# so the report is sent whether the work above it succeeded or not: a report that only goes
+# out on success is one that goes quiet exactly when somebody needs it.
+#
+# TO CHANGE IT: -p failures=0 sends the green version, and the severity and the wording both
+# follow from the numbers without another branch in the document.
+#
+#   dg run --local examples/recipes/report-to-webhook.yaml
+#   dg run --local examples/recipes/report-to-webhook.yaml -p failures=0
+
+format: dirigent/v1
+kind: pipeline
+code: report-to-webhook
+name: Render a report and POST it
+description: Render a markdown page from a Jinja template and deliver it to a receiver as one field of a webhook body.
+
+tags: [recipes, report, transform, webhook]
+
+requires:
+  blocks:
+    - transform.jq
+    - report.render
+    - webhook.post
+
+connections:
+  echo-webhook:
+    kind: http
+    config:
+      base_url: https://postman-echo.com
+      timeout: 30s
+      health_path: /get
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the load reports on.
+    rows:
+      type: integer
+      minimum: 1
+      default: 12
+      description: How many records the load handled.
+    failures:
+      type: integer
+      minimum: 0
+      default: 3
+      description: How many of them were rejected; 0 sends the green version.
+
+steps:
+  summary:
+    block: transform.jq
+    config:
+      input:
+        rows: ${params.rows}
+        failures: ${params.failures}
+        day: ${params.day}
+      program: |
+        . as {$rows, $failures, $day}
+        | [range($rows)
+           | {key: "\($day)/rec-\(.)",
+              status: (if . < $failures then "rejected" else "loaded" end)}]
+        | {day: $day,
+           rows: length,
+           loaded: ([.[] | select(.status == "loaded")] | length),
+           rejected: ([.[] | select(.status == "rejected")] | length),
+           # The keys, so a reader can look them up; not a count of them, which nobody can act on.
+           rejected_keys: [.[] | select(.status == "rejected") | .key]}
+        | . + {severity: (if .rejected == 0 then "info"
+                          elif .rejected < (.rows / 2) then "warning"
+                          else "error" end)}
+
+  page:
+    block: report.render
+    depends_on: [summary]
+    config:
+      values:
+        summary: ${steps.summary.output.value}
+      content_type: text/markdown
+      template: |
+        # Load for {{ summary.day }}
+
+        {{ summary.loaded }} of {{ summary.rows }} records loaded, {{ summary.rejected }} rejected.
+
+        {% if summary.rejected_keys %}
+        Rejected:
+        {% for key in summary.rejected_keys %}
+        - `{{ key }}`
+        {% endfor %}
+        {% else %}
+        Nothing was rejected.
+        {% endif %}
+
+  deliver:
+    block: webhook.post
+    depends_on: [page]
+    # Sent whether the work above passed or failed.
+    rule: all_done
+    config:
+      connection: echo-webhook
+      path: /post
+      body:
+        pipeline: report-to-webhook
+        day: ${params.day}
+        severity: ${steps.summary.output.value.severity}
+        content_type: ${steps.page.output.content_type}
+        text: ${steps.page.output.text}
+      headers:
+        # Routing the receiver can dispatch on without reading the page.
+        x-severity: ${steps.summary.output.value.severity}
+      success_status: [200, 202, 204]