dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/patterns/concurrency-skip.yaml

@@ -0,0 +1,97 @@
+# concurrency: skip -- a second trigger while one run is going creates no run at all.
+#
+# The policy is on the PIPELINE, not on a step and not on a schedule, because the question it
+# answers is about the pipeline: may two runs of this be in flight at once? Four answers:
+#
+#   allow    (the default) start it; two runs of this can safely overlap.
+#   skip     do not start it; there is nothing to catch up on. THIS FILE.
+#   queue    hold it until the slot is free, then release the oldest held one.
+#   replace  cancel what is running and start this instead.
+#
+# WHAT A SECOND TRIGGER DOES HERE: no run is created. Not queued, not deferred, not started
+# and cancelled -- nothing exists. The trigger is recorded as skipped by the concurrency
+# policy and the clock moves on to its next slot. Nothing is retried later, because skip's
+# claim is that the missed firing had no work of its own: the run already going will refresh
+# whatever the skipped one would have.
+#
+# That claim is what makes skip the right policy for a REFRESH and the wrong one for an
+# INCREMENT. Rebuilding a rolling view, warming a cache, re-reading the last 24 hours: any of
+# those can lose a firing without losing data. Consuming a queue, appending yesterday's rows,
+# advancing a cursor: each of those loses work permanently, and wants queue instead.
+#
+# The decision is serialised. Two triggers arriving in the same instant on different processes
+# take a lock first, so exactly one of them wins the slot -- otherwise both would read
+# "nothing is running" and skip would skip nothing.
+#
+# Hop by hop:
+#
+#   slow_scan  a sleep long enough that a second trigger while it runs is easy to arrange by
+#              hand. This is what stands in for the work.
+#   refresh    rebuilds the view. Idempotent by construction, which is the property skip
+#              depends on.
+#
+# EXPECT THIS RUN TO SUCCEED in about six seconds. A single local run has nothing to overlap
+# with, so the policy is exercised against a server -- start one run, then start another while
+# it is going and watch the second not appear:
+#
+#   dg run --local examples/patterns/concurrency-skip.yaml
+#   dg apply examples/patterns/concurrency-skip.yaml
+#   dg run concurrency-skip &
+#   dg run concurrency-skip          # no run created; the policy skipped it
+#   dg runs list concurrency-skip    # one run, not two
+
+format: dirigent/v1
+kind: pipeline
+code: concurrency-skip
+name: A second run that never starts
+description: |
+  `concurrency: skip` creates no run at all when one is already in flight: not queued, not
+  deferred, nothing.
+
+  Right for a refresh, whose next firing subsumes the one that was skipped. Wrong for an
+  increment, which loses that work permanently -- use `queue` there.
+
+tags: [patterns, sensor, transform, concurrency]
+
+# The one line this file is about. It is a property of the pipeline, because "may two of
+# these overlap" is a question about the pipeline and not about any one trigger.
+concurrency: skip
+
+requires:
+  blocks:
+    - time.sleep
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    seconds:
+      type: integer
+      description: How long the scan takes; long enough to trigger a second run by hand.
+      default: 5
+      minimum: 1
+      maximum: 120
+    view:
+      type: string
+      description: Which materialised view the refresh rebuilds.
+      default: cases-rolling-24h
+
+steps:
+  slow_scan:
+    block: time.sleep
+    # A sensor, so the wait is a parked row rather than a held worker -- the slot is occupied
+    # by the RUN, not by a worker, which is why a long run can block a short one.
+    poll: 1s
+    config:
+      for: "${params.seconds}s"
+
+  refresh:
+    block: transform.jq
+    depends_on: [slow_scan]
+    # A full rebuild, not an append. That is the property skip rests on: a firing that never
+    # happened costs nothing, because the next one does all of the work again.
+    config:
+      input:
+        view: "${params.view}"
+      program: |
+        {rebuilt: .view, incremental: false}

dirigent_examples/shelves/patterns/connections-referenced-vs-carried.yaml

@@ -0,0 +1,140 @@
+# Naming a connection, and carrying one, in the same document -- and when each is right.
+#
+# A document is portable precisely because it names credentials rather than holding them. A
+# step says connection: postman-echo, and the base URL, the auth, the TLS settings and the
+# timeout live on the instance, encrypted at rest and redacted in every API response. Moving
+# from staging to production edits the connection and not one pipeline.
+#
+# THE TWO FORMS:
+#
+#   connection: <code>   the ordinary one. The instance holds it, requires.connections says so,
+#                        and an apply against an instance without it is refused up front with
+#                        the code to create.
+#   connections: {...}   carried in the document. For a document that has to run with no
+#                        instance to create a connection on first: a published example, a
+#                        reproduction, a --local run. A SERVER REFUSES a document that carries
+#                        one, because applying it would store the credential in every version
+#                        of the pipeline.
+#
+# Both appear below, which is why this document runs locally and would not apply to a server
+# as it stands. That is not a trick: it is the honest way to show the two side by side.
+#
+# WHAT A CONNECTION BUYS BEYOND SECRECY: the base URL. A step that names one writes a path and
+# not a URL, so the same document points at a test instance and a production instance by
+# naming a different connection -- there is no URL to edit. It also carries the timeout and
+# the health path, which is what dg connection check calls, so a connection can be tested
+# without running a pipeline.
+#
+# A LOCAL RUN HAS NO SERVER to hold the named one, so it is handed over:
+#
+#   dg run --local examples/patterns/connections-referenced-vs-carried.yaml \
+#     --connections examples/connections.yaml
+#
+# Without --connections the run is refused at the step that names it, because a connection
+# that is not there is not something to guess at.
+#
+# Hop by hop:
+#
+#   named    http.request through the named connection. It writes a PATH; the base URL comes
+#            from the connection, and so do the credentials.
+#   carried  the same call through the connection this document carries, so both forms are
+#            exercised in one run.
+#   absolute the third form, for completeness: an absolute url with no connection at all.
+#            Fine for a public endpoint, wrong the moment anything needs a credential --
+#            because there is nowhere in a document to put one.
+#   compare  reads all three back. They answer identically, which is the point: a connection
+#            changes where a credential lives, not what a step does.
+#
+# EXPECT THIS RUN TO SUCCEED in about three seconds, given --connections.
+
+format: dirigent/v1
+kind: pipeline
+code: connections-referenced-vs-carried
+name: A connection named, and one carried
+description: |
+  A step names a connection by code and the instance holds the credential, encrypted at rest
+  and redacted in every response. That is what makes a document portable.
+
+  A document may instead **carry** one, for a run with no instance to create it on -- and a
+  server refuses such a document, because applying it would store the secret in every
+  version.
+
+tags: [patterns, http, transform, credential]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+  # Named here, so an apply against an instance that does not hold it is refused with the
+  # code to create rather than failing at the step.
+  connections:
+    - postman-echo
+
+# Carried, for the case with no instance to name one on. The credentials below are Postman's
+# own public demo pair. basic_password is a SecretStr: sealed on the way in, and an applied
+# connection reports which fields are set and never what they are set to.
+connections:
+  echo-carried:
+    kind: http
+    config:
+      base_url: https://postman-echo.com
+      basic_username: postman
+      basic_password: password
+      timeout: 30s
+      # What dg connection check calls. A connection with no health path can only be tested
+      # by running a pipeline through it.
+      health_path: /get
+
+params:
+  type: object
+  properties:
+    dataset:
+      type: string
+      description: Echoed by all three calls, so the three answers are comparable.
+      default: cases
+
+steps:
+  named:
+    block: http.request
+    config:
+      # The code, not a URL. examples/connections.yaml is where a --local run gets it from.
+      connection: postman-echo
+      # A path, resolved against the connection's base URL. This is the line that does not
+      # change when the instance moves from staging to production.
+      path: /get
+      query:
+        dataset: "${params.dataset}"
+        via: named
+
+  carried:
+    block: http.request
+    config:
+      connection: echo-carried
+      path: /get
+      query:
+        dataset: "${params.dataset}"
+        via: carried
+
+  absolute:
+    block: http.request
+    config:
+      # No connection at all. Legitimate for a public endpoint with no credential and no
+      # environment to move between, and nothing more than that.
+      url: https://postman-echo.com/get
+      query:
+        dataset: "${params.dataset}"
+        via: absolute
+      # A per-call override of whatever timeout applies. On the two steps above the
+      # connection's own timeout would have applied instead.
+      timeout: 20s
+
+  compare:
+    block: transform.jq
+    depends_on: [named, carried, absolute]
+    config:
+      input:
+        named: "${steps.named.output.body.args.via}"
+        carried: "${steps.carried.output.body.args.via}"
+        absolute: "${steps.absolute.output.body.args.via}"
+      program: |
+        {named, carried, absolute, note: "three ways to say where; one thing done"}

dirigent_examples/shelves/patterns/deadline-on-a-sensor.yaml

@@ -0,0 +1,106 @@
+# deadline: how long a step may keep waiting, and what it costs to wait that long.
+#
+# A sensor does not hold a worker. Each poke is one short, read-only observation -- here, one
+# head request against a URI -- and between pokes the attempt is a parked row with a wake-up
+# time on it. So waiting twelve hours costs a few hundred rows, not twelve hours of a worker,
+# and the deadline can honestly be as long as the business answer requires.
+#
+# The three fields that shape a wait, and none of them lives in the block's config:
+#
+#   poll       how often to look. Cost per hour of waiting.
+#   deadline   how long to keep looking. When to give up.
+#   on_timeout what giving up MEANS: fail (the default, this file) or skip
+#              (timeout-skips-the-step.yaml).
+#
+# This one takes the default, because a drop that has not landed by its deadline is a real
+# problem here: somebody promised a file and did not send it, and the run being red is how
+# that is noticed.
+#
+# Hop by hop:
+#
+#   wait_for_drop  storage.exists pokes a path under this run's own scratch space every two
+#                  seconds. Nothing ever writes there, so it never appears, and after eight
+#                  seconds the deadline expires and the step fails.
+#   load           the default all_success edge, so it is skipped.
+#   page_someone   one_failed, so it runs: the absence is reported rather than merely logged.
+#
+# EXPECT THIS RUN TO FAIL, in about eight seconds, with wait_for_drop failed, load skipped and
+# page_someone green. The attempt's error class is rejected, not transient: an expired
+# deadline is not something another attempt could fix, so a retry budget on this step would
+# buy nothing. An expired `timeout` is classified the other way, and
+# timeout-fails-the-step.yaml says why that matters.
+#
+# Note where the drop is looked for. ${run.scratch} is this run's own directory under the
+# instance's artifact root, and file:// URIs are refused outside that root -- so a stored
+# pipeline can never be turned into an arbitrary-file read. A real drop is an s3:// URI or a
+# path under that root, never an absolute path typed into the document.
+#
+# To change it: on_timeout: skip turns this exact run green, which is the same document
+# saying "no drop today is fine". min_size is the other guard worth knowing -- a producer
+# uploading a large object makes the key visible before the bytes are all there, and a floor
+# stops the load reading a half-written file.
+#
+#   dg run --local examples/patterns/deadline-on-a-sensor.yaml   # fails in ~8s, by design
+
+format: dirigent/v1
+kind: pipeline
+code: deadline-on-a-sensor
+name: A deadline on a wait
+description: |
+  `poll`, `deadline` and `on_timeout` are step-level engine semantics, uniform across every
+  sensor and never buried in a block's config.
+
+  A parked sensor costs rows, not workers. As written the drop never lands, the deadline
+  expires with the default `on_timeout: fail`, and the run reports `failed`.
+
+tags: [patterns, http, sensor, storage, timeout]
+
+requires:
+  blocks:
+    - storage.exists
+    - storage.copy
+    - http.request
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      description: The day whose drop is expected; it is part of the object's name.
+      default: "2026-01-01"
+
+steps:
+  wait_for_drop:
+    block: storage.exists
+    # Chosen for how soon the drop matters, not for what it costs to look: each poke is one
+    # head request and one row. A daily drop is polled in minutes, not seconds.
+    poll: 2s
+    # Short enough to watch. A real one is the hour by which somebody promised the file.
+    deadline: 8s
+    # on_timeout is left unwritten, so it is fail: a promised file that never arrived is a
+    # problem somebody has to hear about.
+    config:
+      uri: "${run.scratch}/drops/${params.day}.parquet"
+      # A floor, so a key that exists but is still being written is not treated as a landing.
+      min_size: 1kb
+
+  load:
+    block: storage.copy
+    depends_on: [wait_for_drop]
+    config:
+      # Reading the sensor's own output rather than rebuilding the URI: the sensor may have
+      # matched a glob, and its answer is which object actually landed.
+      source: "${steps.wait_for_drop.output.uri}"
+      target: "${run.scratch}/loaded/${params.day}.parquet"
+
+  page_someone:
+    block: http.request
+    depends_on: [wait_for_drop]
+    rule: one_failed
+    config:
+      url: https://postman-echo.com/post
+      method: POST
+      body:
+        text: the drop did not land inside its deadline
+        day: "${params.day}"

dirigent_examples/shelves/patterns/fan-out-continue.yaml

@@ -0,0 +1,88 @@
+# items: continue -- one bad element, and the other nine carrying on without it.
+#
+# This is fan-out-fail-fast.yaml with one word replaced, and the pair is worth running back to
+# back. continue says the elements are independent: a region that refused is a region that
+# refused, not a reason to abandon the other three.
+#
+# Four consequences, and they are easy to blur into one:
+#
+#   1. The failed item settles as failed. Its attempt is red and its error is stored.
+#   2. The other items run to completion regardless of when the failure happened.
+#   3. The STEP counts as succeeded, so an ordinary all_success edge below it fires.
+#   4. The run reports completed_with_errors -- a third status, not a shade of failed.
+#
+# And the sharp edge, which is rule 3's real cost: the step's output is the list of its items'
+# outputs, and a failed item is ABSENT from it. Not null, absent. So the list is shorter than
+# the input, index 0 is the first item that worked rather than the first one asked for, and a
+# downstream step that counts rows is counting the ones that survived. Read the whole list;
+# be careful with indices.
+#
+# If EVERY item fails the step itself failed, and a downstream step forced to run anyway with
+# rule: all_done is refused rather than handed an empty list.
+#
+# Hop by hop:
+#
+#   push       one item per status. Item two answers 500 and fails; items one and three
+#              answer 200 and do not.
+#   surviving  runs once over the whole batch, and counts what is actually there. Two, not
+#              three, and the document says so out loud rather than assuming three.
+#
+# EXPECT THIS RUN TO END completed_with_errors, with two succeeded items, one failed item, a
+# succeeded step, and a surviving.count of 2.
+#
+# To change it: -p statuses='[500,500,500]' makes every item fail, which fails the step, which
+# skips the join. That is the boundary between this file and fan-out-fail-fast.yaml.
+#
+#   dg run --local examples/patterns/fan-out-continue.yaml                              # completed_with_errors
+#   dg run --local examples/patterns/fan-out-continue.yaml -p statuses='[500,500,500]'  # failed
+
+format: dirigent/v1
+kind: pipeline
+code: fan-out-continue
+name: A batch that tolerates one bad item
+description: |
+  `items: continue` lets the surviving elements finish. The failed item stays red, the step
+  counts as succeeded, and the run reports `completed_with_errors`.
+
+  A failed item is **absent** from the step's output list rather than null, so the batch a
+  downstream step reads can be shorter than the one that was asked for.
+
+tags: [patterns, http, transform, failure, fan-out]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    statuses:
+      type: array
+      description: One item per element; each item asks the endpoint for that status.
+      default: [200, 500, 200]
+      minItems: 1
+      maxItems: 16
+      items:
+        type: integer
+        enum: [200, 404, 500]
+
+steps:
+  push:
+    block: http.request
+    for_each: "${params.statuses}"
+    # The one word this file is about.
+    items: continue
+    config:
+      url: "https://postman-echo.com/status/${item}"
+      method: GET
+
+  surviving:
+    block: transform.jq
+    depends_on: [push]
+    config:
+      input: "${steps.push.output}"
+      # length rather than a hard-coded three: under continue the batch is whatever survived,
+      # and a program that assumes otherwise reports a number nobody can trust.
+      program: |
+        {statuses: [.[].status], count: length}

dirigent_examples/shelves/patterns/fan-out-fail-fast.yaml

@@ -0,0 +1,80 @@
+# items: fail_fast -- the default item policy, and what one bad element costs the batch.
+#
+# A fan-out has its own failure question, separate from every other one in the engine: when
+# one of N items fails, is that a bad element or a bad batch? fail_fast is the default answer
+# and it says bad batch. The step fails, and it fails as a whole -- the successful items do
+# not leave a partial result behind for anything downstream to read.
+#
+# That is the right default. A fan-out is usually N slices of one job, and half a job silently
+# succeeding is worse than a red run: it is a load somebody believes happened.
+#
+# The list below is statuses rather than regions, because the element is what decides whether
+# the item fails, and making that the visible content of the item is the shortest way to show
+# it. The second element is a 500.
+#
+# Hop by hop:
+#
+#   push     one item per status, each calling /status/<item>. Items one and three answer 200;
+#            item two answers 500 and fails. Under fail_fast that is the end of the step.
+#   receipt  the default all_success edge on a failed step, so it is skipped.
+#
+# EXPECT THIS RUN TO FAIL. Look at the item grid rather than the step: some items are
+# succeeded, one is failed, and depending on timing a sibling may not have run at all. The
+# step is failed regardless, which is the policy.
+#
+# To change it: fan-out-continue.yaml is this file with one word replaced, and it is worth
+# running both back to back. -p statuses='[200,200,200]' makes this one green.
+#
+#   dg run --local examples/patterns/fan-out-fail-fast.yaml                          # fails, by design
+#   dg run --local examples/patterns/fan-out-fail-fast.yaml -p statuses='[200,200]'  # succeeds
+
+format: dirigent/v1
+kind: pipeline
+code: fan-out-fail-fast
+name: One bad item sinks the batch
+description: |
+  `items: fail_fast` is the default: one failed element fails the whole fan-out step, and
+  nothing downstream reads a partial batch.
+
+  The right default, because a fan-out is usually N slices of one job, and half a job
+  silently succeeding is worse than a red run.
+
+tags: [patterns, http, failure, fan-out]
+
+requires:
+  blocks:
+    - http.request
+
+params:
+  type: object
+  properties:
+    statuses:
+      type: array
+      description: One item per element; each item asks the endpoint for that status.
+      default: [200, 500, 200]
+      minItems: 1
+      maxItems: 16
+      items:
+        type: integer
+        enum: [200, 404, 500]
+
+steps:
+  push:
+    block: http.request
+    for_each: "${params.statuses}"
+    # items is deliberately not written. Omitting it is what makes this file about the
+    # default rather than about a setting somebody chose.
+    config:
+      url: "https://postman-echo.com/status/${item}"
+      method: GET
+
+  receipt:
+    block: http.request
+    depends_on: [push]
+    # Skipped. Under fail_fast there is no half-batch to hand on, so the edge behaves exactly
+    # as it would below any other failed step.
+    config:
+      url: https://postman-echo.com/post
+      method: POST
+      body:
+        pushed: "${steps.push.output}"

dirigent_examples/shelves/patterns/fan-out-from-params.yaml

@@ -0,0 +1,84 @@
+# A fan-out whose width comes from the run's parameters.
+#
+# for_each: "${params.regions}" is the form to reach for when the caller decides how much work
+# there is: a backfill over the months somebody names, a load over the regions this tenant
+# has, a re-index of the datasets that moved. The pipeline is one document; the run is three
+# items today and eleven tomorrow.
+#
+# The parameter schema is what keeps that honest. regions below is declared as an array of
+# strings with a minimum length, so a run started with a bare string, an empty list, or a
+# number is refused at the door with the reason -- before a run exists, before an item grid
+# is built, before anything is called. A fan-out over an unvalidated parameter is how a
+# pipeline ends up making one request per character of a string.
+#
+# Expansion happens when the run is CREATED, so the item grid is complete and visible from the
+# start, and the reference may read params, run and item but never an upstream step's output.
+#
+# Hop by hop:
+#
+#   fetch    http.request once per region, with the region in the query string so each item's
+#            call is distinguishable in the log. Postman Echo answers with what it was sent,
+#            so each item's output carries its own region back.
+#   tally    one attempt over the whole list, pulling each item's echoed region out again.
+#
+# EXPECT THIS RUN TO SUCCEED with four items, in a couple of seconds.
+#
+# To change it: pass your own list. A JSON array on the command line is the way a list-typed
+# parameter is written, and dotted keys address nested leaves rather than array indices --
+# writing into a list by index is refused, because a dotted path cannot say how long the array
+# is meant to be.
+#
+#   dg run --local examples/patterns/fan-out-from-params.yaml
+#   dg run --local examples/patterns/fan-out-from-params.yaml -p regions='["east","west"]'
+
+format: dirigent/v1
+kind: pipeline
+code: fan-out-from-params
+name: A fan-out sized by its parameters
+description: |
+  `for_each: "${params.regions}"` lets the caller decide how wide the run is, while the
+  parameter schema keeps that from meaning anything at all.
+
+  Expansion happens when the run is created, so the item grid is complete before the first
+  attempt runs.
+
+tags: [patterns, http, transform, fan-out, params]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    regions:
+      type: array
+      description: The regions this run loads; one run item per element.
+      default: [east, west, north, south]
+      # The guard that matters. Without items and minItems, -p regions=east would be a
+      # string, and a fan-out over a string is a fan-out over nothing good.
+      minItems: 1
+      maxItems: 32
+      items:
+        type: string
+        minLength: 1
+
+steps:
+  fetch:
+    block: http.request
+    for_each: "${params.regions}"
+    config:
+      url: https://postman-echo.com/get
+      query:
+        # Each item sends its own region, so the log line for one item names the work that
+        # item did rather than the work the step did.
+        region: "${item}"
+
+  tally:
+    block: transform.jq
+    depends_on: [fetch]
+    config:
+      input: "${steps.fetch.output}"
+      program: |
+        {loaded: [.[].body.args.region], count: length}