dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/recipes/http-post-file-from-storage.yaml

@@ -0,0 +1,117 @@
+# POST a file that lives in storage: read it, then send what the read brought back.
+#
+# In:  a table of readings generated inline and written to the run's scratch space.
+# Out: the echo service's view of the upload -- the size it received and the content type it
+#      was told -- beside the size the file actually is.
+#
+# A file is bytes at a URI, and a request sends a value, so the hop between them is a
+# storage.read step. Four hops:
+#
+#   build     transform.jq, the payload a real pipeline would have fetched.
+#   file      storage.write, the value to a URI. Output: uri, bytes_written, content_type.
+#   load      storage.read, the same URI. The object is application/json, so it comes back
+#             as `value`; max_size is how much of it this step will hold, and an object
+#             past it is refused rather than truncated.
+#   upload    http.request. body is that value, serialised and sent as JSON by the block, so
+#             nothing here builds a request out of strings.
+#
+# HOW BIG A FILE THIS SENDS. Everything above passes through the worker, so the file has to
+# fit in it twice over: once in the read, once in the request. That is the trade for being
+# able to look at what is sent. Bytes nobody has to look at move between URIs with
+# storage.copy instead, bounded by the storage rather than by the worker, and a body coming
+# back the other way is bounded by max_response.
+#
+# The file is addressed as ${steps.file.output.uri} rather than typed twice, so one step
+# decides where it went and a rename is a one-line change. A missing object is a rejected
+# read, not a retried one: calling again cannot make a file appear that was never written.
+#
+# To change it: point the write at s3://<bucket>/... and the read follows it, because the
+# scheme is the only thing that changes.
+#
+#   dg run --local examples/recipes/http-post-file-from-storage.yaml
+#   dg run --local examples/recipes/http-post-file-from-storage.yaml -p stations=500
+
+format: dirigent/v1
+kind: pipeline
+code: http-post-file-from-storage
+name: POST a file read out of storage
+description: Write a file into the run's scratch space, read it back with storage.read, and POST what the read brought back.
+
+tags: [recipes, http, storage, transform]
+
+requires:
+  blocks:
+    - transform.jq
+    - storage.write
+    - storage.read
+    - http.request
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the uploaded file is named for.
+    stations:
+      type: integer
+      minimum: 1
+      default: 40
+      description: How many rows the file holds, which is what makes it big enough to matter.
+
+steps:
+  build:
+    block: transform.jq
+    config:
+      input:
+        count: ${params.stations}
+        day: ${params.day}
+      program: |
+        . as {$count, $day}
+        | {day: $day,
+           readings: [range($count)
+                      | {station: "st-\(.)", celsius: (. % 17) - 5}]}
+
+  file:
+    block: storage.write
+    depends_on: [build]
+    config:
+      target: ${run.scratch}/readings-${params.day}.json
+      value: ${steps.build.output.value}
+
+  load:
+    block: storage.read
+    depends_on: [file]
+    config:
+      source: ${steps.file.output.uri}
+      max_size: 8mb
+
+  upload:
+    block: http.request
+    depends_on: [load]
+    config:
+      url: https://postman-echo.com/post
+      method: POST
+      # A value, so the block serialises it and sends it as application/json. An endpoint
+      # that wants some other type is told with content_type.
+      body: ${steps.load.output.value}
+      timeout: 30s
+
+  receipt:
+    block: transform.jq
+    depends_on: [file, load, upload]
+    config:
+      input:
+        written: ${steps.file.output.bytes_written}
+        read: ${steps.load.output.bytes_read}
+        echoed: ${steps.upload.output.body}
+      program: |
+        {written, read,
+         # What the receiver was told the body was, and how much of it arrived.
+         content_type: .echoed.headers["content-type"],
+         received: (.echoed.headers["content-length"] | tonumber),
+         rows: (.echoed.json.readings | length),
+         # The file and the read agree on one number, and the receiver counted the same
+         # bytes back out of the request.
+         intact: (.written == .read and .read == (.echoed.headers["content-length"] | tonumber))}

dirigent_examples/shelves/recipes/http-post-json-echo.yaml

@@ -0,0 +1,105 @@
+# POST a JSON body built from upstream work, and read the receipt.
+#
+# In:  a batch of readings built inline.
+# Out: the echo service's view of the POST -- the body it parsed and the headers it saw --
+#      alongside the status and the payload's own row count.
+#
+# body is a value, not a string. The block serialises it, sets the content type, and sends
+# it, so a payload assembled by a jq step upstream travels as structure the whole way and
+# nothing hand-builds JSON with string interpolation. That is what keeps a quote in a
+# station name from becoming a syntax error on somebody else's parser.
+#
+# https://postman-echo.com/post answers with what it received, so the .json field of the
+# response is this pipeline's own body coming back -- which is why the last step can compare
+# what was sent with what arrived and report them side by side.
+#
+# Two headers are worth knowing about. The block sets content-type itself, so a header map
+# is for what the receiver wants on top: a routing key, an idempotency key, an API version.
+# A secret does not go here -- it goes on a connection, which is the whole point of
+# http-headers-and-auth-connection.yaml.
+#
+# A POST is not idempotent, and the block declares itself so: the engine will spend a retry
+# budget on one, and it is on the author to know whether the receiver deduplicates. An
+# idempotency key in the headers is what makes that safe, and it belongs in the payload's
+# own identity rather than in a random value minted per attempt.
+#
+# To change it: -p batch=3 sends a bigger payload; the shape of the document does not move.
+#
+#   dg run --local examples/recipes/http-post-json-echo.yaml
+#   dg run --local examples/recipes/http-post-json-echo.yaml -p batch=3
+
+format: dirigent/v1
+kind: pipeline
+code: http-post-json-echo
+name: POST a JSON body
+description: Build a payload upstream, POST it as structure rather than as text, and read the receipt the endpoint answers with.
+
+tags: [recipes, http, transform]
+
+requires:
+  blocks:
+    - transform.jq
+    - http.request
+
+params:
+  type: object
+  properties:
+    batch:
+      type: integer
+      minimum: 1
+      default: 5
+      description: How many readings the payload carries.
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the payload reports.
+
+steps:
+  payload:
+    block: transform.jq
+    config:
+      input:
+        batch: ${params.batch}
+        day: ${params.day}
+      program: |
+        . as {$batch, $day}
+        | {day: $day,
+           source: "dirigent recipes",
+           readings: [range($batch)
+                      | {station: "st-\(.)", celsius: (. * 2) - 4, note: "it's fine"}]}
+
+  publish:
+    block: http.request
+    depends_on: [payload]
+    config:
+      url: https://postman-echo.com/post
+      method: POST
+      # A value, serialised by the block. Nothing here builds JSON out of strings.
+      body: ${steps.payload.output.value}
+      headers:
+        # What the receiver asked for, on top of the content type the block sets itself.
+        x-batch-day: ${params.day}
+        # Derived from what is being sent rather than minted per attempt, so a retry of the
+        # same payload presents the same key.
+        idempotency-key: readings-${params.day}
+      timeout: 30s
+
+  receipt:
+    block: transform.jq
+    depends_on: [payload, publish]
+    config:
+      input:
+        sent: ${steps.payload.output.value}
+        status: ${steps.publish.output.status}
+        duration_ms: ${steps.publish.output.duration_ms}
+        echoed: ${steps.publish.output.body}
+      program: |
+        {status, duration_ms,
+         sent_rows: (.sent.readings | length),
+         # The echo service returns the parsed body under .json, so this is the payload
+         # having made the round trip.
+         received_rows: (.echoed.json.readings | length),
+         identical: (.echoed.json == .sent),
+         content_type: .echoed.headers["content-type"],
+         batch_day_header: .echoed.headers["x-batch-day"]}

dirigent_examples/shelves/recipes/http-post-report.yaml

@@ -0,0 +1,183 @@
+# POST a batch, summarise the receipt, and let the run write the page a person reads.
+#
+# In:  a count of stations and a count of days, as parameters. The readings are generated,
+#      so the payload is genuinely large without forty rows being typed out.
+# Out: forty readings POSTed, a small summary of what came back, and the run's own report
+#      document: a markdown page rendered when the run settles, stored with the run, and
+#      read back with `dg runs report ID --markdown`.
+#
+# The four hops:
+#
+#   readings  a jq program builds the batch. range() twice is a cross product, so
+#             stations x days rows come out of five lines rather than out of a literal, and
+#             the arithmetic is fixed rather than random: the same parameters always POST
+#             the same bytes, which is what makes a rerun comparable to the first run.
+#   publish   the batch goes out as structure. body is a value the block serialises, so a
+#             quote in a station name is not somebody else's parse error. The URL is here
+#             rather than on a connection because Postman Echo is public and this recipe
+#             should run with nothing applied first.
+#   summary   postman-echo answers with what it received: the parsed body under .json and
+#             the headers it saw under .headers. This hop keeps a page's worth of it -- a
+#             row count, min, max and mean, the first few rows, and the headers as a list
+#             of {name, value} pairs, because a template loops over a list without asking
+#             anything of the sandbox.
+#   report    the section at the bottom. It renders no data of its own: everything in it is
+#             the run's facts and the outputs the three steps left behind.
+#
+# Where the template's facts come from:
+#
+#   run.*       the engine's account of the run -- id, status, trigger, the two moments and
+#               the duration between them. Half of them are legitimately null depending on
+#               how a run ended, and a null renders as nothing rather than failing the page.
+#   pipeline.*  the pipeline version this run pinned, so the page names what actually ran.
+#   steps[]     one entry per step, in the order a person watched them, carrying outcome,
+#               attempts, duration_ms, output and output_bytes. output is the step's *last*
+#               attempt's value, and it is there only when the step ran: a skipped or
+#               never-reached step has none, which is why the page guards on it before
+#               reading into it.
+#   rendered_at when this page was rendered, which is the moment the run settled.
+#
+# steps is a list, in the order a person watched the run. step is the same facts by name --
+# `step.summary.output.value` is the summary hop's output -- which is how this page reads one
+# hop without walking the list, and it is why a step a template reads should be named for what
+# it holds.
+#
+# The three filters are the ones that make a machine's number readable: `duration` turns
+# milliseconds into 1m30s, `iso` renders a moment, `bytes` a size. `round` is Jinja's own.
+#
+# To change it: -p stations=20 -p days=10 POSTs two hundred rows and the page reports two
+# hundred; -p rows_shown=10 lengthens the table without touching the payload. Deleting the
+# report: section stops the document being rendered at all, and `report: {}` swaps this page
+# for the built-in one -- report-built-in.yaml is that document.
+#
+#   dg run --local examples/recipes/http-post-report.yaml
+#   dg apply examples/recipes/http-post-report.yaml
+#   dg run http-post-report --watch
+#   dg runs report RUN_ID --markdown
+
+format: dirigent/v1
+kind: pipeline
+code: http-post-report
+name: POST a batch and report on it
+description: POST a generated batch to an echo service, summarise the receipt, and render the run's own markdown report from the run's facts and its steps' outputs.
+
+tags: [recipes, http, transform, report, starter]
+
+requires:
+  blocks:
+    - transform.jq
+    - http.request
+
+params:
+  type: object
+  properties:
+    stations:
+      type: integer
+      minimum: 1
+      default: 8
+      description: How many stations report.
+    days:
+      type: integer
+      minimum: 1
+      default: 5
+      description: How many days each station reports for.
+    rows_shown:
+      type: integer
+      minimum: 1
+      default: 5
+      description: How many rows the report's table carries.
+
+steps:
+  readings:
+    block: transform.jq
+    config:
+      input:
+        stations: ${params.stations}
+        days: ${params.days}
+      program: |
+        . as {$stations, $days}
+        | {source: "dirigent recipes",
+           readings: [range($days) as $day
+                      | range($stations) as $station
+                      | {station: "st-\($station)",
+                         day: ("2026-01-0" + ($day + 1 | tostring)),
+                         # Fixed arithmetic, so the same parameters POST the same bytes.
+                         celsius: ((($station * 7) + ($day * 3)) % 19) - 4}]}
+
+  publish:
+    block: http.request
+    depends_on: [readings]
+    config:
+      url: https://postman-echo.com/post
+      method: POST
+      # A value the block serialises. Nothing here builds JSON out of strings.
+      body: ${steps.readings.output.value}
+      headers:
+        # A header the report's own table has something to show. Headers are strings, so
+        # a count is interpolated into one rather than passed as a number.
+        x-report-batch: ${params.stations}-stations-${params.days}-days
+      timeout: 30s
+
+  summary:
+    block: transform.jq
+    depends_on: [publish]
+    config:
+      input:
+        shown: ${params.rows_shown}
+        status: ${steps.publish.output.status}
+        echoed: ${steps.publish.output.body}
+      program: |
+        . as {$shown, $status, $echoed}
+        | ($echoed.json.readings | map(.celsius)) as $values
+        | {status: $status,
+           rows: ($echoed.json.readings | length),
+           min: ($values | min),
+           max: ($values | max),
+           mean: (($values | add) / ($values | length)),
+           first_rows: ($echoed.json.readings[:$shown]),
+           # A list rather than a map: a template loops over a list with no filter at all.
+           headers: ($echoed.headers | to_entries | map({name: .key, value: (.value | tostring)}))}
+
+report:
+  template: |
+    # {{ pipeline.name or pipeline.code }}: {{ run.status }}
+
+    - run: `{{ run.id }}`
+    - pipeline: `{{ pipeline.code }}` version {{ pipeline.version }}
+    - trigger: {{ run.trigger }}
+    - started: {{ run.started_at | iso }}
+    - finished: {{ run.finished_at | iso }}
+    - duration: {{ run.duration_ms | duration }}
+    - rendered: {{ rendered_at | iso }}
+
+    ## Steps
+
+    | step | outcome | attempts | took | output |
+    | --- | --- | --- | --- | --- |
+    {% for one in steps %}
+    | {{ one.step }} | {{ one.outcome }} | {{ one.attempts }} | {{ one.duration_ms | duration }} | {{ one.output_bytes | bytes }} |
+    {% endfor %}
+
+    ## What came back
+
+    {% set echoed = step.summary.output.value %}
+    {% if echoed %}
+    The echo service answered {{ echoed.status }} and parsed {{ echoed.rows }} readings back,
+    between {{ echoed.min }}C and {{ echoed.max }}C, averaging {{ echoed.mean | round(1) }}C.
+
+    | station | day | celsius |
+    | --- | --- | --- |
+    {% for row in echoed.first_rows %}
+    | {{ row.station }} | {{ row.day }} | {{ row.celsius }} |
+    {% endfor %}
+
+    ### Headers it saw
+
+    | header | value |
+    | --- | --- |
+    {% for header in echoed.headers %}
+    | {{ header.name }} | {{ header.value }} |
+    {% endfor %}
+    {% else %}
+    The summary step left no output, so this run never got as far as a receipt.
+    {% endif %}

dirigent_examples/shelves/recipes/http-save-body-to-storage.yaml

@@ -0,0 +1,106 @@
+# Call a service, put the answer in a file, and read the file back.
+#
+# In:  a query the echo service reflects back, named by parameters.
+# Out: the body as a file in the run's scratch space, and a small summary read back out of
+#      that file.
+#
+# An http.request step hands its answer on as `body`, and a body worth keeping travels to
+# storage the way every other value does: through a storage.write step. Four hops, and what
+# each one hands on:
+#
+#   download  http.request. Output: status, headers, body, body_bytes, duration_ms. body is
+#             the parsed document here, because the echo service answers JSON.
+#   save      storage.write, the value to a URI. Output: uri, bytes_written, content_type.
+#             The bytes are staged beside the target and renamed on close, so the object
+#             becomes visible only once it is whole -- which is what makes a storage.exists
+#             sensor on the other side a gate rather than a race.
+#   back      storage.read, the same URI. It is application/json, so the object comes back
+#             as `value` rather than as `text`.
+#   summary   transform.jq, which is where a pipeline decides how much of a large body it
+#             actually needs: this one keeps four fields.
+#
+# HOW BIG A BODY THIS HOLDS. max_response bounds what the call reads into the worker, and a
+# body past it is refused rather than read half way; max_size bounds what the read brings
+# back. Both are whole values a step can look at. Bytes nobody has to look at never become a
+# step output at all: storage.copy moves them between URIs, bounded by the storage rather
+# than by the worker's memory.
+#
+# To change it: point save at s3://<bucket>/incoming/... and the same three steps become a
+# download into an object store, because the scheme is the only thing that changes.
+#
+#   dg run --local examples/recipes/http-save-body-to-storage.yaml
+#   dg run --local examples/recipes/http-save-body-to-storage.yaml -p day=2026-02-01
+
+format: dirigent/v1
+kind: pipeline
+code: http-save-body-to-storage
+name: A response body saved to storage
+description: Call an endpoint, write the body it answered with to a storage URI, and read the file back for a summary.
+
+tags: [recipes, http, storage, transform, starter]
+
+requires:
+  blocks:
+    - http.request
+    - storage.write
+    - storage.read
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the downloaded file is named for.
+    station:
+      type: string
+      default: st-1
+      description: A query value, echoed back inside the downloaded body.
+
+steps:
+  download:
+    block: http.request
+    config:
+      url: https://postman-echo.com/get
+      method: GET
+      query:
+        station: ${params.station}
+        day: ${params.day}
+      max_response: 8mb
+      timeout: 30s
+
+  save:
+    block: storage.write
+    depends_on: [download]
+    config:
+      # ${run.scratch} is the run's own prefix in whatever storage the instance is
+      # configured with: a temporary directory under --local, the artifact root on a
+      # server, a bucket where an s3 connection serves the scheme.
+      target: ${run.scratch}/incoming/${params.day}.json
+      value: ${steps.download.output.body}
+
+  back:
+    block: storage.read
+    depends_on: [save]
+    config:
+      # Where the write said the object landed, not the same path typed twice.
+      source: ${steps.save.output.uri}
+      max_size: 8mb
+
+  summary:
+    block: transform.jq
+    depends_on: [download, save, back]
+    config:
+      input:
+        status: ${steps.download.output.status}
+        uri: ${steps.save.output.uri}
+        bytes: ${steps.save.output.bytes_written}
+        body: ${steps.back.output.value}
+      program: |
+        {status, uri, file_bytes: .bytes,
+         summary: {url: .body.url,
+                   station: .body.args.station,
+                   day: .body.args.day,
+                   host: .body.headers.host}}

dirigent_examples/shelves/recipes/http-success-status-list.yaml

@@ -0,0 +1,80 @@
+# When a non-2xx is the right answer: success_status, and what it does not mean.
+#
+# In:  a status code, as a parameter, defaulting to 418.
+# Out: that status treated as success, with the run green and the body available to the
+#      steps below.
+#
+# By default any 2xx is success and everything else fails the step. success_status replaces
+# that list for one step: writing [200, 418] says exactly those two are success, and 200
+# alone no longer passes unless it is in the list. It is a replacement, not an addition,
+# which is the field's one sharp edge.
+#
+# It is worth using where an API genuinely answers a non-2xx for an outcome that is not an
+# error:
+#
+#   404 on a delete or a lookup, where "it is not there" is the state being asked about.
+#   409 on a create, where "it already exists" is the state being asked about.
+#   204 for an accepted write with nothing to return.
+#
+# It is not a way to silence a failure. A step that lists 500 as success has not made the
+# server healthy; it has removed the run's only way of saying otherwise. The rule of thumb
+# is that a listed status must be one the steps below can act on, and the branch below here
+# does exactly that: it reads the status and reports what it means.
+#
+# https://postman-echo.com/status/<code> answers with the code it is given, which is what
+# makes this runnable without arranging a real 409.
+#
+# To change it: -p status=404 with -p expected=404 is the "not there" case; -p status=500
+# with the default expectations fails the run, which is the check that this recipe is not
+# just swallowing everything.
+#
+#   dg run --local examples/recipes/http-success-status-list.yaml
+#   dg run --local examples/recipes/http-success-status-list.yaml -p status=200
+#   dg run --local examples/recipes/http-success-status-list.yaml -p status=500   # fails, on purpose
+
+format: dirigent/v1
+kind: pipeline
+code: http-success-status-list
+name: Statuses that count as success
+description: Accept a specific non-2xx as success for one call with success_status, and branch on what the status actually meant.
+
+tags: [recipes, http, transform]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    status:
+      type: integer
+      default: 418
+      description: The status the endpoint is asked to answer with.
+
+steps:
+  call:
+    block: http.request
+    config:
+      url: https://postman-echo.com/status/${params.status}
+      method: GET
+      # Replaces the default, so 200 is listed explicitly: leaving it out would fail a call
+      # that succeeded ordinarily.
+      success_status: [200, 404, 418]
+      timeout: 20s
+
+  interpret:
+    block: transform.jq
+    depends_on: [call]
+    config:
+      input:
+        status: ${steps.call.output.status}
+      program: |
+        # A listed status has to be one the steps below can act on; that is what makes
+        # listing it different from ignoring a failure.
+        {status,
+         outcome: (if .status == 200 then "present"
+                   elif .status == 404 then "absent, which is an answer"
+                   else "answered, and unusual" end),
+         acted_on: (.status != 200)}