dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/patterns/pipeline-run-wait.yaml

@@ -0,0 +1,107 @@
+# pipeline.run with wait: true -- a parent step that is a parked row, not a held worker.
+#
+# wait is the default, and it is what makes composition useful: the parent's step finishes
+# when the CHILD finishes, so everything downstream of it genuinely runs after the child's
+# work. The child's final status becomes this step's outcome -- succeeded is succeeded, failed
+# is failed -- and the parent's step output carries the child's code, its run id and its
+# status.
+#
+# HOW IT WAITS MATTERS. pipeline.run is submit-then-probe: the child run is created, a handle
+# is stored, and the parent's attempt is parked with a wake-up time. So a parent waiting on a
+# three-hour child holds a row for three hours and a worker for none of it, and the parent
+# does not have to be running anywhere in particular for the child to progress.
+#
+# poll is the cadence of that probing, and it is a step field like any other. The block's own
+# default is five seconds; 1s below is so the example finishes while you watch it. A real
+# parent waiting on an hour-long child leaves it out.
+#
+# A dirigent document holds exactly one pipeline, so composition is two files, and the apply
+# order matters: the child has to exist before the parent applies at all. requires.pipelines
+# is what says so, and an apply against an instance without the child is refused up front with
+# the code to apply first -- rather than failing at the step, at five in the morning.
+#
+# Hop by hop:
+#
+#   load_east  runs the child for one region and waits. Its output is the child's identity and
+#              outcome, not the child's data: a child's outputs stay in the child's run.
+#   archive    reads that output back. It runs only because the child finished, which is the
+#              guarantee wait: true buys.
+#
+# EXPECT THIS RUN TO SUCCEED in about three seconds, with load_east reporting the child's
+# status as succeeded.
+#
+# Locally there is no instance to have applied the child to, so hand it over:
+#
+#   dg run --local examples/patterns/pipeline-run-wait.yaml \
+#     --also-apply examples/patterns/pipeline-run-child.yaml
+#
+# Against a server it is two applies and a run:
+#
+#   dg apply examples/patterns/pipeline-run-child.yaml
+#   dg apply examples/patterns/pipeline-run-wait.yaml
+#   dg run pipeline-run-wait --watch
+
+format: dirigent/v1
+kind: pipeline
+code: pipeline-run-wait
+name: A parent that waits
+description: |
+  `pipeline.run` with the default `wait: true` finishes when the child finishes, and the
+  child's status becomes the step's outcome.
+
+  It is submit-then-probe, so a parent waiting on a three-hour child holds a database row
+  and no worker at all.
+
+tags: [patterns, pipeline, transform, composition]
+
+requires:
+  blocks:
+    - pipeline.run
+    - transform.jq
+  # The child, by code. An apply against an instance that does not hold it is refused here
+  # rather than at the step.
+  pipelines:
+    - pipeline-run-child
+
+params:
+  type: object
+  properties:
+    region:
+      type: string
+      description: Which region the child is asked to load.
+      default: east
+      pattern: "^[a-z][a-z0-9-]*$"
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+
+steps:
+  load_east:
+    block: pipeline.run
+    # Faster than the block's five-second default so the example settles while you watch. A
+    # real parent leaves this out: probing a long child every second buys nothing.
+    poll: 1s
+    config:
+      pipeline: pipeline-run-child
+      # Written out rather than forwarded. The child's schema is its interface, and a parent
+      # that passed its own parameters through would be coupled to whatever it was run with.
+      params:
+        region: "${params.region}"
+        day: "${params.day}"
+      # The default, written once so the pair with pipeline-run-fire-and-forget.yaml reads as
+      # a choice rather than as an omission.
+      wait: true
+
+  archive:
+    block: transform.jq
+    depends_on: [load_east]
+    config:
+      input:
+        # The child's identity and outcome. Its data is not here: a child's outputs belong to
+        # the child's run, and a parent that needs a value reads it from a store both can see.
+        pipeline: "${steps.load_east.output.pipeline}"
+        run_id: "${steps.load_east.output.run_id}"
+        status: "${steps.load_east.output.status}"
+      program: |
+        {archived: .pipeline, child_run: .run_id, ended_as: .status}

dirigent_examples/shelves/patterns/pipeline-run-with-params.yaml

@@ -0,0 +1,124 @@
+# Passing parameters to a child, and the two things the engine checks about them and when.
+#
+# A parent writes the child's parameters out. It does not forward its own, and there is no
+# syntax for "pass everything through" -- because the child's schema is the contract, and a
+# parent that forwarded whatever it was run with would break the moment either side grew a
+# parameter.
+#
+# WHEN THE CHECKING HAPPENS, in two stages:
+#
+#   at apply    requires.pipelines is checked, so a parent naming a child the instance does
+#               not hold is refused with the code to apply first.
+#   at execute  the params map below is validated against the CHILD's schema, by the child, as
+#               the step runs. So a wrong parameter is a failed step with the schema's own
+#               message, not a refused apply -- the parent has no way to know at apply time
+#               what a ${params.x} will resolve to.
+#
+# max_depth is the other guard, and it counts the chain reaching this run rather than the
+# calls this document makes. The default is 5, so a parent calling a child that calls a child
+# is fine and a cycle is not: a pipeline that eventually calls itself fails at the depth limit
+# instead of filling the instance with runs.
+#
+# A FAN-OUT OF pipeline.run IS THE USEFUL SHAPE. One item per region, each starting its own
+# child run with its own parameters, all of them parked rows rather than held workers. That is
+# how a parent runs eleven children in parallel without eleven of anything.
+#
+# Hop by hop:
+#
+#   plan       builds the list of regions to load, so the fan-out's width is one value the
+#              rest of the document reads.
+#   load       one child run per region, waited on, each with its own parameters. Note the
+#              day is the same for all of them and the region is not: an element carries what
+#              differs, and params carries what does not.
+#   summarise  the join. It reads the batch of child outcomes -- codes, run ids and statuses,
+#              which is everything a parent gets back from a child.
+#
+# EXPECT THIS RUN TO SUCCEED in about five seconds, with three child runs and three statuses
+# of succeeded.
+#
+# To change it: -p regions='["east"]' runs one child; a region that does not match the child's
+# pattern fails that item at execute time with the child's own message, which is the two-stage
+# checking above made visible.
+#
+#   dg run --local examples/patterns/pipeline-run-with-params.yaml \
+#     --also-apply examples/patterns/pipeline-run-child.yaml
+#   dg run --local examples/patterns/pipeline-run-with-params.yaml \
+#     --also-apply examples/patterns/pipeline-run-child.yaml -p regions='["east","west"]'
+
+format: dirigent/v1
+kind: pipeline
+code: pipeline-run-with-params
+name: Children, one per region
+description: |
+  A parent writes its child's parameters out explicitly; they are validated against the
+  **child's** schema when the step executes, not when the parent is applied.
+
+  A fan-out of `pipeline.run` is how one parent runs many children in parallel, each a
+  parked row rather than a held worker.
+
+tags: [patterns, pipeline, transform, composition, fan-out]
+
+requires:
+  blocks:
+    - pipeline.run
+    - transform.jq
+  pipelines:
+    - pipeline-run-child
+
+params:
+  type: object
+  properties:
+    regions:
+      type: array
+      description: One child run per element.
+      default: [east, west, north]
+      minItems: 1
+      maxItems: 16
+      items:
+        type: string
+        # The same pattern the child declares. Duplicating it here is what turns a child-side
+        # failure at execute time into a parent-side refusal before the run exists.
+        pattern: "^[a-z][a-z0-9-]*$"
+    day:
+      type: string
+      format: date
+      description: The day every child loads; the same for all of them.
+      default: "2026-01-01"
+
+steps:
+  plan:
+    block: transform.jq
+    config:
+      input: "${params.regions}"
+      program: |
+        {regions: ., count: length}
+
+  load:
+    block: pipeline.run
+    depends_on: [plan]
+    # The fan-out reads params, not the plan step's output: cardinality is fixed when the run
+    # is created, so a for_each names params, run, or another fan-out's grid -- never a step's
+    # output.
+    for_each: "${params.regions}"
+    # One region refusing is not a reason to abandon the others; the join below counts what
+    # actually landed.
+    items: continue
+    poll: 1s
+    config:
+      pipeline: pipeline-run-child
+      params:
+        # What differs per child comes from the element.
+        region: "${item}"
+        # What is the same for every child comes from the parent's parameters.
+        day: "${params.day}"
+
+  summarise:
+    block: transform.jq
+    depends_on: [load]
+    config:
+      input: "${steps.load.output}"
+      program: |
+        {
+          children: [.[] | {pipeline, run_id, status}],
+          started: length
+        }

dirigent_examples/shelves/patterns/poll-cadence.yaml

@@ -0,0 +1,102 @@
+# poll: what the cadence buys, and what it costs, measured on the same wait twice.
+#
+# poll is how often a sensor looks. It is not how long the wait is, and it is not accuracy for
+# free: a sensor learns nothing between pokes, so the answer arrives up to one whole poll
+# interval late. Coarse polling is cheap and blunt; fine polling is precise and chatty. That
+# is the entire trade, and this file measures it rather than asserting it.
+#
+# Two time.sleep sensors, both asked for exactly the same wait, differing only in cadence.
+# time.sleep reports waited_ms, which is the configured duration plus however much of a poll
+# interval it had to sit through after the wait was already over -- so the overshoot is
+# readable in the output instead of having to be timed by hand.
+#
+# NOTE WHERE THE CADENCE IS WRITTEN. poll, deadline, timeout and retry are step fields, and
+# the reference language reaches config and for_each only. So the two cadences below are
+# literals: a run's shape is not something a caller talks the engine into at run time. Only
+# the wait itself, which is block config, takes a parameter.
+#
+# Hop by hop:
+#
+#   fine     poll: 1s over the wait. It pokes once a second, and waited_ms lands close to the
+#            configured duration.
+#   coarse   poll: 4s over the same wait. The wait being over is not noticed until the next
+#            poke, so waited_ms lands up to four seconds higher.
+#   compare  reads both and subtracts. The difference is what the coarse cadence cost in
+#            latency, and it is bounded by the poll interval, which is the rule of thumb.
+#
+# The two sensors have no edge between them, so they wait side by side and the run takes about
+# as long as the slower one rather than the sum. Width in a DAG is not a keyword: it is what
+# two steps that do not name each other already are.
+#
+# EXPECT THIS RUN TO SUCCEED, in about eight to nine seconds, with a coarse_cost_ms of
+# somewhere between zero and four seconds depending on where the wait fell between pokes.
+#
+# To change it: -p wait=20 makes the run longer without changing the gap, because the
+# overshoot depends on the cadence and not on the length of the wait -- which is exactly the
+# thing worth internalising before choosing a poll interval for a twelve-hour sensor.
+#
+#   dg run --local examples/patterns/poll-cadence.yaml
+#   dg run --local examples/patterns/poll-cadence.yaml -p wait=20
+
+format: dirigent/v1
+kind: pipeline
+code: poll-cadence
+name: What a poll interval costs
+description: |
+  The same wait polled every second and every four seconds, with the difference in
+  `waited_ms` reported.
+
+  A sensor learns nothing between pokes, so its answer is late by up to one poll interval.
+  Cheap and blunt, or precise and chatty -- measured here rather than asserted.
+
+tags: [patterns, sensor, transform, timeout]
+
+requires:
+  blocks:
+    - time.sleep
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    wait:
+      type: integer
+      description: How many seconds both sensors are asked to wait for.
+      default: 6
+      minimum: 1
+      maximum: 60
+
+steps:
+  fine:
+    block: time.sleep
+    # A literal, because a step field is not interpolated. One second is a chatty cadence for
+    # anything real; it is here so the tight end of the trade is visible.
+    poll: 1s
+    config:
+      # Config is interpolated, and a configured duration is written humanely rather than as
+      # a bare number of seconds, so the parameter goes inside a string that ends in s.
+      for: "${params.wait}s"
+
+  coarse:
+    block: time.sleep
+    # The same wait, looked at a quarter as often. Nothing else differs, which is what makes
+    # the two outputs comparable.
+    poll: 4s
+    config:
+      for: "${params.wait}s"
+
+  compare:
+    block: transform.jq
+    depends_on: [fine, coarse]
+    # A join: no rule written, so all_success, and it waits for both sensors.
+    config:
+      input:
+        fine_ms: "${steps.fine.output.waited_ms}"
+        coarse_ms: "${steps.coarse.output.waited_ms}"
+      program: |
+        {
+          fine_ms,
+          coarse_ms,
+          coarse_cost_ms: (.coarse_ms - .fine_ms),
+          rule_of_thumb: "a sensor's answer is late by up to one poll interval"
+        }

dirigent_examples/shelves/patterns/priority-layered.yaml

@@ -0,0 +1,120 @@
+# One pipeline, three answers to "how urgently", each overriding the one before it.
+#
+# Every run carries a priority -- low, normal, or high -- and the claim takes a higher one
+# first. It is one word, and it LAYERS the way parameters already do:
+#
+#   the document       priority: low          what this pipeline is by default
+#   a trigger          priority: normal       what a schedule or webhook fires it at
+#   an ad hoc run      --priority high        what a person asks for, once
+#
+# The specific one wins, and the answer is PINNED ON THE RUN when it is created. Editing this
+# document tomorrow does not reorder a run that started today, which is the same promise the
+# pinned pipeline version makes about everything else a run executes.
+#
+# WHAT THE THREE WORDS ARE FOR, in the case this document is drawn from -- a bulk reload that
+# takes hours and matters to nobody in particular:
+#
+#   low       its default. It is enormous, it is not urgent, and it should be behind
+#             whatever a person is waiting on. That is what low means: last.
+#   normal    what the nightly schedule fires it at, because a nightly run that never
+#             finishes is a nightly run that failed. Ordinary, not deferred.
+#   high      what an incident asks for by hand. The reload is now the thing being waited
+#             on, so it goes ahead of everything queued.
+#
+# THE CLAIM'S ORDER IS PRIORITY, THEN FAIRNESS, THEN DUE TIME. Fairness is round-robin
+# between runs: the claim ranks each run's due attempts within that run and takes one from
+# each in turn, so this document's fan-out over four regions interleaves with a two-step run
+# queued beside it rather than holding every worker slot until it drains. Priority sorts
+# ahead of that, so a high run's attempts still come first.
+#
+# WHAT PRIORITY CANNOT DO: it never takes a slot that is already busy. An attempt that is
+# running is never cancelled to make room for an urgent one, because that means killing work
+# with side effects nobody can take back. A high run is claimed first the moment a slot frees
+# -- so with every slot held by a long step, it still waits for one to finish.
+#
+# Hop by hop:
+#
+#   plan       turns the parameters into the regions this reload covers.
+#   reload     a fan-out, one item per region -- the many small units that make a run worth
+#              being fair about in the first place.
+#   receipt    joins them back and reports what was covered.
+#
+# EXPECT THIS RUN TO SUCCEED in about a second, with no network. A local run is alone in a
+# throwaway instance, so --priority is refused there: there is nothing to be ahead of.
+#
+#   dg run --local examples/patterns/priority-layered.yaml
+#   dg apply examples/patterns/priority-layered.yaml
+#   dg run priority-layered                     # low: the document's own
+#   dg run priority-layered --priority high     # the incident case
+#   dg runs list                                # ! marks high, a muted low marks low
+
+format: dirigent/v1
+kind: pipeline
+code: priority-layered
+name: Low by default, high by hand
+description: |
+  `priority` is one word -- `low`, `normal` or `high` -- and it layers: the document declares
+  the default, a schedule or webhook overrides it for what it triggers, and `dg run
+  --priority` overrides it once more. The resolved word is pinned on the run at creation.
+
+  The claim orders by priority, then round-robin fairness between runs, then due time.
+  Nothing is preempted: an attempt already running is never cancelled for an urgent one.
+
+tags: [patterns, transform, priority, schedule]
+
+# Low, because a bulk reload should be behind whatever somebody is waiting on.
+priority: low
+
+requires:
+  blocks:
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    regions:
+      type: array
+      description: The regions this reload covers; one fan-out item each.
+      items:
+        type: string
+      default: [nordics, nepal, sahel, andes]
+
+steps:
+  plan:
+    block: transform.jq
+    config:
+      input:
+        regions: "${params.regions}"
+      program: |
+        {regions, total: (.regions | length)}
+
+  reload:
+    block: transform.jq
+    depends_on: [plan]
+    # A fan-out is where fairness earns its keep: four items here, four hundred in the real
+    # thing, and every one of them is a unit the claim interleaves with other runs' work.
+    for_each: "${params.regions}"
+    config:
+      input:
+        region: "${item}"
+      program: |
+        {region, rows: 1200}
+
+  receipt:
+    block: transform.jq
+    depends_on: [reload]
+    config:
+      input: "${steps.reload.output}"
+      program: |
+        {regions: length, rows: (map(.value.rows) | add)}
+
+triggers:
+  schedules:
+    - code: nightly-reload
+      name: Nightly, at ordinary urgency
+      description: A nightly run that never finishes is a nightly run that failed.
+      cron: "0 2 * * *"
+      timezone: UTC
+      # The schedule overrides the document's low for what IT fires, and nothing else. An
+      # ad hoc run of the same pipeline is still low unless it says otherwise.
+      priority: normal

dirigent_examples/shelves/patterns/references-cheat-sheet.yaml

@@ -0,0 +1,186 @@
+# Every ${...} form the reference language allows, each used once, in one document.
+#
+# THERE ARE FOUR NAMESPACES AND NOTHING ELSE. No expressions, no arithmetic, no conditionals,
+# no functions, no defaults, no coalescing. A reference names a value or it does not resolve.
+#
+#   params.<name>                a run parameter
+#   params.<name>.<path>         a leaf inside an object parameter
+#   params.<name>.<index>        an element of an array parameter
+#   steps.<key>.output           a step's whole output
+#   steps.<key>.output.<field>   one field of it
+#   steps.<key>.output.<index>   one element of a fan-out step's output list
+#   steps.<key>.items            the grid a fan-out maps over, and only for_each reads it
+#   steps.<key>.item.output      the matching item's output, in a step that shares its grid
+#   item                         this run item's element, inside a step that fans out
+#   item.<path>                  a field of it, when the element is an object
+#   run.id                       this run's uuid
+#   run.scratch                  this run's own directory under the artifact root
+#   run.window.start             the start of the interval this run covers
+#   run.window.end               the end of it, exclusive
+#
+# TWO RULES ABOUT WHAT A REFERENCE BECOMES:
+#
+#   A reference that is the WHOLE value resolves to the typed value. "${params.count}" is the
+#   integer 4 downstream, "${params.window}" is an object, "${steps.x.output}" is an array.
+#   A reference INSIDE a larger string interpolates as text: "day-${params.day}" is a string,
+#   null renders as empty, and a boolean renders as true or false.
+#
+# AND ONE ABOUT WHAT DOES NOT RESOLVE. An unknown reference is not an empty string: it raises,
+# and the engine settles that attempt as rejected, naming what was actually available. That
+# applies to a misspelled parameter, a step that produced no output, a field that is not
+# there, ${item} outside a fan-out, and ${run.window.start} on a run that carries no window.
+#
+# WHERE REFERENCES ARE RESOLVED: in a step's config, and in for_each. Nowhere else. poll,
+# deadline, timeout, retry, rule and concurrency are the pipeline's shape, and a caller does
+# not get to talk the engine into a different one at run time.
+#
+# One more rule this document cannot show without the allowlist: in a config field a block
+# marked as a shell string -- shell.run's `command` -- every substituted value is shell-quoted,
+# so a parameter that arrived in a webhook payload becomes exactly one word and the
+# metacharacters the author typed keep their meaning.
+#
+# THIS FILE NEEDS A WINDOW. run.window.* is a property of the run, so an ad hoc run has to be
+# given one; without it this document is refused at the first step, naming the reference. That
+# refusal is the demonstration too:
+#
+#   dg run --local examples/patterns/references-cheat-sheet.yaml --window 2026-06-01..2026-06-02
+#   dg run --local examples/patterns/references-cheat-sheet.yaml   # refused: this run carries no window
+
+format: dirigent/v1
+kind: pipeline
+code: references-cheat-sheet
+name: The whole reference language
+description: |
+  Four namespaces -- `params`, `steps`, `item`, `run` -- and no expressions, conditionals or
+  functions anywhere.
+
+  A whole reference resolves to the typed value; one inside a larger string interpolates as
+  text; an unknown one is rejected rather than resolved to empty. Needs `--window`.
+
+tags: [patterns, transform, fan-out, references]
+
+requires:
+  blocks:
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      default: "2026-06-01"
+    count:
+      type: integer
+      description: An integer, so the typed-value rule has something to be visible on.
+      default: 4
+      minimum: 1
+    regions:
+      type: array
+      description: An array, indexed once below and fanned out over once.
+      default: [east, west, north]
+      minItems: 1
+      items:
+        type: string
+    window:
+      type: object
+      description: An object parameter, so a nested leaf has somewhere to be.
+      default: {days: 7, align: midnight}
+      properties:
+        days: {type: integer}
+        align: {type: string}
+
+steps:
+  namespaces:
+    block: transform.jq
+    config:
+      input:
+        # params, three ways: whole, a nested leaf, and an element by index.
+        day: "${params.day}"
+        align: "${params.window.align}"
+        first_region: "${params.regions.0}"
+        # An integer stays an integer, because the reference is the entire value.
+        typed_count: "${params.count}"
+        # The same reference inside a larger string is text, and reads as text downstream.
+        interpolated: "loading ${params.count} regions for ${params.day}"
+        # run: the two values every run has.
+        run_id: "${run.id}"
+        scratch: "${run.scratch}"
+        # run.window: only on a run that carries one. Half-open, so start is included and
+        # end is not.
+        window_start: "${run.window.start}"
+        window_end: "${run.window.end}"
+      program: |
+        .
+
+  per_region:
+    block: transform.jq
+    # for_each is the one place outside config where references resolve.
+    for_each: "${params.regions}"
+    depends_on: [namespaces]
+    config:
+      input:
+        # item: this run item's element. Outside a fan-out this reference does not resolve.
+        region: "${item}"
+        # A step's whole output, and one field of it, from inside a fan-out.
+        day: "${steps.namespaces.output.value.day}"
+      program: |
+        {region, day}
+
+  paired_region:
+    block: transform.jq
+    depends_on: [per_region]
+    # steps.<key>.items is the grid per_region maps over, so this step gets that grid rather
+    # than one of its own and the two are paired by item position.
+    for_each: "${steps.per_region.items}"
+    config:
+      input:
+        # ...which is what lets this read the MATCHING item's output instead of the whole
+        # list. fan-out-item-wise.yaml is this on its own.
+        region: "${steps.per_region.item.output.value.region}"
+        # ${item} is the same element it was in per_region.
+        element: "${item}"
+      program: |
+        {region, element}
+
+  per_feed:
+    block: transform.jq
+    # A literal list of objects, so item.<path> has somewhere to reach.
+    for_each:
+      - {code: cases, weight: 2}
+      - {code: climate, weight: 1}
+    depends_on: [namespaces]
+    config:
+      input:
+        # item.<path>: a field of an element that is an object. A field the element does not
+        # carry raises here rather than resolving to an empty string.
+        feed: "${item.code}"
+        weight: "${item.weight}"
+        # ${item} on its own is still the whole element.
+        element: "${item}"
+      program: |
+        {feed, weight, element}
+
+  collect:
+    block: transform.jq
+    depends_on: [per_region, paired_region, per_feed, namespaces]
+    config:
+      input:
+        # A fan-out step's whole output: a JSON array of the items' outputs, in item order.
+        every_item: "${steps.per_region.output}"
+        # One element of it, by index. Under items: continue a failed item is absent from the
+        # list, so index 0 is the first item that WORKED rather than the first one asked for.
+        first_item: "${steps.per_region.output.0.value.region}"
+        paired: "${steps.paired_region.output}"
+        feeds: "${steps.per_feed.output}"
+        # A whole step output, unindexed, so the object arrives intact.
+        namespaces: "${steps.namespaces.output.value}"
+      program: |
+        {
+          regions: [.every_item[].value.region],
+          paired: [.paired[].value.region],
+          feeds: [.feeds[].value.feed],
+          first_item,
+          covered: "\(.namespaces.window_start)..\(.namespaces.window_end)",
+          typed_count_is_a_number: (.namespaces.typed_count | type)
+        }