dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/docker/docker-run-failing-teardown.yaml

@@ -0,0 +1,94 @@
+# A drive step that fails, and the teardown that runs anyway.
+#
+# THIS PIPELINE FAILS ON PURPOSE. It is here to show what the run looks like afterwards, so
+# the expected outcome is a failed run with a succeeded teardown in it and an empty daemon.
+#
+# REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
+# tcp+TLS, or the local socket in dev). docs/docker.md is the family's home.
+#
+# Every block here declares local_execution, so the engine refuses them unless the instance
+# allowlists their ids:
+#   dg run --local examples/docker/docker-run-failing-teardown.yaml \
+#     --enable-unsafe docker.compose.up,docker.compose.down,docker.run
+#   export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up","docker.compose.down","docker.run"]'
+#
+# Three hops:
+#   stack     one nginx service, up and healthy, persisting for the rest of the run.
+#   drive     a container that reaches the service, prints what it got, and then exits 1.
+#             The container's non-zero exit is the step's failure, and its stderr is what the
+#             failure message quotes.
+#   teardown  the same project down again, under rule: all_done.
+#
+# WHAT THE RUN'S STEP LIST READS LIKE AFTERWARDS: stack succeeded, drive failed, teardown
+# succeeded, run failed. A teardown is not an apology for a failure and does not hide one --
+# the run's status is failed because a step of it failed, whatever ran after. That is the
+# whole reason down is its own step rather than something up does at the end of itself:
+# all_done is an ordinary engine rule, evaluated the same way every other rule is, so no
+# "finally" primitive is needed and no block has to guess when a stack is finished with.
+#
+# WHAT THE DAEMON HOLDS AFTERWARDS: nothing of this run.
+#   docker ps -a --filter label=com.docker.compose.project
+# lists nothing from it, because the teardown ran. Take rule: all_done off the down step and
+# the same run leaves nginx running until somebody notices.
+#
+# TO MAKE IT YOURS: the drive step is standing in for whatever real work a stack is brought up
+# for -- a migration, a test suite, a load. Give it your command; keep the shape.
+
+format: dirigent/v1
+kind: pipeline
+code: docker-run-failing-teardown
+name: A failing drive step over a stack
+description: A container that reaches the stack and then exits 1, with the teardown running on any outcome.
+
+tags: [docker, execute]
+
+requires:
+  blocks:
+    - docker.compose.up
+    - docker.compose.down
+    - docker.run
+
+steps:
+  stack:
+    block: docker.compose.up
+    deadline: 10m
+    config:
+      wait: true
+      wait_timeout: 2m
+      pull: always
+      content: |
+        services:
+          api:
+            image: nginx:alpine
+            healthcheck:
+              test: ["CMD-SHELL", "wget -qO- http://localhost/ >/dev/null 2>&1 || exit 1"]
+              interval: 2s
+              timeout: 2s
+              retries: 5
+
+  drive:
+    block: docker.run
+    depends_on: [stack]
+    deadline: 5m
+    config:
+      image: alpine:3
+      pull: true
+      network: "${steps.stack.output.default_network}"
+      # The work succeeds and the step still fails: the exit code is the whole verdict, and
+      # the last line on stderr is what the step's failure message quotes back.
+      argv:
+        - sh
+        - -c
+        - 'wget -qO- http://api/ >/dev/null && echo "the service answered"; echo "and now failing on purpose" >&2; exit 1'
+      memory: 64mb
+      pids_limit: 64
+
+  teardown:
+    block: docker.compose.down
+    depends_on: [drive]
+    # Without this the teardown would be skipped the moment drive failed, and the stack would
+    # outlive the run.
+    rule: all_done
+    deadline: 5m
+    config:
+      down_volumes: true

dirigent_examples/shelves/docker/docker-ticker.yaml

@@ -0,0 +1,49 @@
+# A container that talks while it works, for watching the log stream live.
+#
+# The block is asynchronous: execute starts the container detached and the engine probes it
+# on a clock. Every probe reads what the container printed since the last one and appends it
+# to the run's log, so the run's screen and `dg logs` both move while the container runs --
+# a hundred ticks over three and a half minutes, arriving a couple at a time.
+#
+# REQUIRES A DOCKER SOCKET, like every docker.run step: without a reachable daemon the
+# document applies and the step fails saying so. The id is unsafe and must be allowlisted:
+#   dg run --local examples/docker/docker-ticker.yaml --enable-unsafe docker.run
+#   export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.run"]'   # for the instance
+
+format: dirigent/v1
+kind: pipeline
+code: docker-ticker
+name: Docker ticker
+description: A container printing a line every two seconds, streamed into the run as it goes.
+
+tags: [docker, execute]
+
+requires:
+  blocks:
+    - docker.run
+
+params:
+  type: object
+  additionalProperties: false
+  properties:
+    ticks:
+      type: integer
+      default: 100
+      minimum: 1
+      maximum: 1000
+      description: How many lines the container prints, two seconds apart.
+
+steps:
+  tick:
+    block: docker.run
+    deadline: 40m
+    config:
+      image: alpine:3
+      pull: true
+      network: none
+      argv:
+        - sh
+        - -c
+        - 'i=1; while [ "$i" -le "${params.ticks}" ]; do echo "tick $i of ${params.ticks}"; i=$((i+1)); sleep 2; done'
+      memory: 64mb
+      pids_limit: 64

dirigent_examples/shelves/execute/README.md

@@ -0,0 +1,16 @@
+# Execute examples
+
+A shell step that runs code on the worker, which is why it needs its entry in
+`DIRIGENT_ENABLED_UNSAFE_BLOCKS` before an instance will run it. The container family --
+`docker.run`, `docker.compose.*` and `docker.build` -- now lives on its own shelf,
+[docker/](../docker).
+
+```bash
+dg run --local examples/execute/long-log.yaml --enable-unsafe shell.run
+```
+
+## Pipelines
+
+| File | What it teaches |
+| --- | --- |
+| [long-log.yaml](long-log.yaml) | A step that logs enough to scroll, for exercising the terminal pane rather than reading about it. |

dirigent_examples/shelves/execute/long-log.yaml

@@ -0,0 +1,89 @@
+# A step that prints far more than fits anywhere, to see what the terminal and the UI do.
+#
+# Most examples print one line. Real steps print thousands, and a log viewer that is only
+# ever tested against "hello from dirigent" is not tested. This one emits a few hundred lines
+# of plausible batch output on purpose, so there is something to scroll, search, and wrap.
+#
+# It is also the example that shows what shell.run does with a stream it cannot carry inline.
+# The block returns nine output fields for the two streams, and they answer different
+# questions:
+#
+#   stdout            the HEAD of the stream, cut at the instance's inline capture size
+#   stdout_bytes      how much the command printed altogether, cut or not
+#   stdout_truncated  whether the field above is short of the stream
+#   stdout_uri        where the WHOLE of it was written, never truncated
+#
+# So a downstream step that needs the full output reads stdout_uri and never stdout: the
+# inline copy is a convenience for looking at, and it is the one that lies about length. The
+# same four exist for stderr, which is captured separately -- a run that prints progress to
+# stderr and results to stdout keeps them apart, and this one does exactly that.
+#
+# Lower the instance's inline capture size and the same run starts reporting truncated: true
+# with an unchanged stdout_bytes, which is the pair worth watching.
+#
+# This one uses shell.run, which executes code on the worker and is refused unless the
+# instance allowlists it: export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["shell.run"]', or pass
+# --enable-unsafe shell.run to a local run.
+#
+#   dg run --local examples/execute/long-log.yaml --enable-unsafe shell.run
+#   dg run --local examples/execute/long-log.yaml --enable-unsafe shell.run -p lines=2000
+
+format: dirigent/v1
+kind: pipeline
+code: long-log
+name: Long log
+description: |
+  A step that prints a few hundred lines, so a log viewer has something to do.
+
+  The point is the difference between the four `stdout_*` fields: `stdout` is the **head**
+  of the stream, `stdout_bytes` is its true length, `stdout_truncated` says whether those
+  two disagree, and `stdout_uri` is where the whole of it was written.
+
+  A downstream step that needs all of it reads the URI. The inline copy is for looking at.
+
+tags: [execute]
+
+requires:
+  blocks:
+    - shell.run
+
+params:
+  type: object
+  properties:
+    lines:
+      type: integer
+      description: How many rows the batch pretends to process.
+      default: 400
+      minimum: 1
+      maximum: 100000
+
+steps:
+  process_batch:
+    block: shell.run
+    config:
+      # Progress goes to stderr and results go to stdout, because the two streams are
+      # captured separately and this is what makes that visible rather than merely true.
+      command: |
+        echo "batch starting: ${params.lines} rows" >&2
+        i=1
+        while [ "$i" -le "${params.lines}" ]; do
+          printf 'row %05d  station=st-%03d  celsius=%d.%d  status=accepted\n' \
+            "$i" "$((i % 250))" "$((i % 40 - 10))" "$((i % 10))"
+          if [ "$((i % 100))" -eq 0 ]; then
+            echo "progress: $i rows" >&2
+          fi
+          i=$((i + 1))
+        done
+        echo "batch finished" >&2
+
+  # Reads the measurements rather than the text, which is the honest way to depend on a
+  # stream that may have been cut. stdout_bytes is the whole length even when stdout is not.
+  report_volume:
+    block: shell.run
+    depends_on: [process_batch]
+    config:
+      argv:
+        - echo
+        - "wrote ${steps.process_batch.output.stdout_bytes} bytes to stdout
+          (truncated inline: ${steps.process_batch.output.stdout_truncated}),
+          whole stream at ${steps.process_batch.output.stdout_uri}"

dirigent_examples/shelves/failure/README.md

@@ -0,0 +1,20 @@
+# Failure examples
+
+How a run goes wrong on purpose: retries and the budget they spend, a timeout that ends an
+attempt, a step allowed to fail without failing the run, and the always-runs cleanup edge.
+Seeing these settle red is the point; a corpus where everything is green teaches nothing
+about reading a failure.
+
+```bash
+dg run --local examples/failure/retries.yaml
+```
+
+## Pipelines
+
+| File | What it teaches |
+| --- | --- |
+| [retries.yaml](retries.yaml) | The retry policy: attempts, backoff, and an error class that says whether trying again could ever help. |
+| [retry-budget.yaml](retry-budget.yaml) | The budget behind the policy: an unknown failure (a non-zero exit) is retried too, and a hopeless step spends every attempt before it fails. |
+| [step-timeout.yaml](step-timeout.yaml) | A timeout ends the attempt, and the policy decides whether another one starts. |
+| [optional-step.yaml](optional-step.yaml) | `completed_with_errors`: a step that may fail without failing the run, and the third status that says so. |
+| [error-handler.yaml](error-handler.yaml) | The cleanup edge: alert on failure, and always release the lock, however the load went. |

dirigent_examples/shelves/failure/error-handler.yaml

@@ -0,0 +1,89 @@
+# Error handling drawn in the DAG rather than coded inside a block.
+#
+# rule is the edge condition on a step's depends_on. Four values, and they are the whole
+# vocabulary for branching on an outcome:
+#
+#   all_success  (the default) every prerequisite succeeded
+#   one_failed   at least one prerequisite failed; skipped when none did
+#   all_done     every prerequisite settled, whatever it settled as
+#   always       runs once every prerequisite has settled, whatever the outcome -- the
+#                engine still waits for upstream to finish, it does not fire mid-flight
+#
+# one_failed is what makes a step an error-handler branch. all_done is where cleanup that
+# must happen either way belongs. Both are visible in the graph, which is the point: the
+# failure path is drawn in the DAG rather than coded inside a block.
+#
+# There is one other failure knob, and it is a different question. rule decides whether a
+# step runs; continue_on_failure decides whether a step that failed sinks the run. A step
+# marked continue_on_failure: true settles as failed, but its dependents are shown a success,
+# so their all_success edges fire and the branch carries on, while the run ends
+# completed_with_errors instead of failed. That is also why a one_failed handler below a
+# tolerated step never runs: the rules are shown no failure to handle. optional-step.yaml is
+# that flag on its own. Use it for the branch nobody should be paged about; use rule for the
+# branch that exists to handle the failure.
+# For a fan-out the equivalent is items: continue, which tolerates one bad element rather
+# than one bad step.
+#
+# As written the load fails, so the notify branch runs and the success branch is skipped.
+#
+# This one uses shell.run, which executes code on the worker and is refused unless the
+# instance allowlists it: export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["shell.run"]', or pass
+# --enable-unsafe shell.run to a local run.
+#
+#   dg run --local examples/failure/error-handler.yaml
+#   dg run --local examples/failure/error-handler.yaml -p status=200   # the other branch
+
+format: dirigent/v1
+kind: pipeline
+code: error-handler
+name: Handle a failing step
+description: Load a batch, alert on failure, and always release the lock afterwards.
+
+tags: [failure, execute, http, graph]
+
+params:
+  type: object
+  properties:
+    status:
+      type: integer
+      description: The status the load endpoint is asked to return.
+      default: 500
+
+steps:
+  load_batch:
+    block: http.request
+    retry:
+      max_attempts: 2
+      backoff: 1s
+    config:
+      url: "https://postman-echo.com/status/${params.status}"
+      method: GET
+
+  notify_failure:
+    block: http.request
+    depends_on: [load_batch]
+    # one_failed is what makes this a handler: it runs only on the path where the load
+    # failed, and skips silently on the path where it did not.
+    rule: one_failed
+    config:
+      url: https://postman-echo.com/post
+      method: POST
+      body:
+        text: the nightly load failed
+
+  publish_success:
+    block: shell.run
+    depends_on: [load_batch]
+    rule: all_success
+    config:
+      argv: [echo, "load succeeded"]
+
+  release_lock:
+    block: http.request
+    depends_on: [notify_failure, publish_success]
+    # Exactly one of the two branches above runs and the other skips, so all_done is what
+    # releases the lock either way. all_success would wait for a step that never runs.
+    rule: all_done
+    config:
+      url: https://postman-echo.com/delete
+      method: DELETE

dirigent_examples/shelves/failure/optional-step.yaml

@@ -0,0 +1,82 @@
+# A step that is allowed to fail, and what that does to the run it is part of.
+#
+# continue_on_failure is one narrow claim: this step failing is not a reason to stop. The
+# engine holds it as "tolerated", and tolerated has three consequences worth separating,
+# because they are easy to blur into one:
+#
+#   1. The step itself still settles as FAILED. Its attempt is red, its error is stored, and
+#      its logs are there to read. Tolerating a failure never hides one.
+#   2. Its dependents see SUCCEEDED. An ordinary all_success edge below a tolerated failure
+#      fires, so the branch carries on -- which is the entire point of the flag.
+#   3. The run ends completed_with_errors, not succeeded. A run with a tolerated failure in
+#      it is not a clean run, and it does not get to claim it was.
+#
+# Consequence 2 has a sharp edge. Because dependents see SUCCEEDED, a rule: one_failed handler
+# hung below a tolerated step never fires: there is, as far as the rules are concerned, no
+# failure to handle. Tolerating a failure and handling one are opposite instructions, so a
+# step should not be asked for both.
+#
+# The failure here is local and instant -- a command that exits non-zero -- so the run settles
+# in about a second and needs no network. What it costs to fail is the point, not what fails.
+#
+# This one uses shell.run, which executes code on the worker and is refused unless the
+# instance allowlists it: export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["shell.run"]', or pass
+# --enable-unsafe shell.run to a local run.
+#
+#   dg run --local examples/failure/optional-step.yaml --enable-unsafe shell.run   # completed_with_errors
+#   dg run --local examples/failure/optional-step.yaml --enable-unsafe shell.run -p exit_code=0  # succeeded
+
+format: dirigent/v1
+kind: pipeline
+code: optional-step
+name: Optional step
+description: |
+  Import a batch, push metrics on a best-effort basis, and publish either way.
+
+  `continue_on_failure: true` makes the metrics push **optional**: it settles as failed,
+  its dependents see it as succeeded and carry on, and the run reports
+  `completed_with_errors` rather than `succeeded`.
+
+  Nothing is hidden. The red attempt is still there to open.
+
+tags: [failure, execute]
+
+requires:
+  blocks:
+    - shell.run
+
+params:
+  type: object
+  properties:
+    exit_code:
+      type: integer
+      description: What the metrics push exits with; 0 makes the whole run green.
+      default: 3
+      minimum: 0
+      maximum: 125
+
+steps:
+  import_batch:
+    block: shell.run
+    config:
+      argv: [echo, "imported 412 rows"]
+
+  push_metrics:
+    block: shell.run
+    depends_on: [import_batch]
+    # The one flag this example is about. Nobody should be paged because a metrics endpoint
+    # was unhappy, and nobody should be told the run was clean either.
+    continue_on_failure: true
+    # No retry policy: one attempt, and the exit code is deterministic. retry-budget.yaml is
+    # where a failure is spent against a budget instead.
+    config:
+      command: "echo 'metrics endpoint refused' >&2; exit ${params.exit_code}"
+
+  publish:
+    block: shell.run
+    depends_on: [push_metrics]
+    # The default rule, left unwritten, is all_success -- and it fires, because a tolerated
+    # failure reads as succeeded to whatever depends on it. Without the flag above this step
+    # would be skipped and the run would be failed.
+    config:
+      argv: [echo, "published, with or without metrics"]

dirigent_examples/shelves/failure/retries.yaml

@@ -0,0 +1,75 @@
+# Retry policy per step, and the difference between a retry and a tolerated failure.
+#
+# A retry is the attempt-level question: should we try this again? The delay is data, so no
+# worker sleeps through a backoff -- a new attempt row is written with available_at in the
+# future and picked up when it comes due. Whether a retry happens at all also depends on
+# the block's own error classification: a 5xx is transient and retried, a 4xx is rejected
+# and never is, however much budget is left.
+#
+# continue_on_failure is the step-level question: what does this failure mean downstream?
+# A tolerated failure reads as succeeded to its dependents, so the branch carries on, while
+# the run itself reports completed_with_errors rather than succeeded.
+#
+# As written, publish hits a deliberate 503 three times over six seconds and then gives up,
+# which is what makes the retry visible. Pass a different status to watch the publish
+# succeed -- the optional cache still times out, so the run still ends completed_with_errors:
+#
+# This one uses shell.run, which executes code on the worker and is refused unless the
+# instance allowlists it: export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["shell.run"]', or pass
+# --enable-unsafe shell.run to a local run.
+#
+#   dg run --local examples/failure/retries.yaml              # completed_with_errors
+#   dg run --local examples/failure/retries.yaml -p status=200  # publish succeeds; completed_with_errors
+
+format: dirigent/v1
+kind: pipeline
+code: retries
+name: Retry behavior
+description: Publish to a service that refuses, then warm an optional cache that is slow.
+
+tags: [failure, execute, http, retry]
+
+params:
+  type: object
+  properties:
+    status:
+      type: integer
+      description: The status the publish endpoint is asked to return.
+      default: 503
+
+steps:
+  publish:
+    block: http.request
+    continue_on_failure: true
+    retry:
+      # 2s, then 4s, then give up: multiplier doubles the wait and max_backoff caps it, so a
+      # long budget cannot turn into a long sleep. jitter spreads the retries of a fan-out
+      # that all failed at once, instead of firing them again in the same instant.
+      max_attempts: 3
+      backoff: 2s
+      max_backoff: 10s
+      multiplier: 2.0
+      jitter: 0.2
+    config:
+      url: "https://postman-echo.com/status/${params.status}"
+      method: GET
+
+  warm_cache:
+    block: http.request
+    depends_on: [publish]
+    continue_on_failure: true
+    # The endpoint sleeps five seconds and the timeout is one, so this fails on time rather
+    # than on the service: an optional step is not allowed to hold the run open.
+    timeout: 1s
+    retry:
+      max_attempts: 2
+      backoff: 1s
+    config:
+      url: https://postman-echo.com/delay/5
+      method: GET
+
+  finish:
+    block: shell.run
+    depends_on: [warm_cache]
+    config:
+      argv: [echo, "published if it was accepted, cache warmed if it was willing"]

dirigent_examples/shelves/failure/retry-budget.yaml

@@ -0,0 +1,82 @@
+# A budget of three attempts, spent in full on a failure nobody can explain.
+#
+# retries.yaml shows a retry against a service that answers 503, where the block itself calls
+# the failure transient. This is the other half of the rule, and the one that surprises
+# people: a failure the engine cannot classify is retried too.
+#
+# There are exactly three classes, and only one of them ends a step immediately:
+#
+#   transient  network, a 5xx, a timeout          retried while the budget lasts
+#   rejected   validation, auth, a 4xx            never retried, however much budget is left
+#   unknown    anything else                      retried while the budget lasts
+#
+# A command that exits non-zero is unknown. shell.run cannot know whether exit 1 meant a
+# corrupt input file or a database that was briefly away, so it declines to guess, and the
+# engine spends the budget rather than assuming the worst. That is what makes max_attempts
+# worth writing on a step whose failures are ordinary program failures rather than HTTP.
+#
+# The consequence is visible here and worth sitting with: this step is hopeless, and it is
+# retried anyway, three times, over roughly three seconds. Nothing knows it is hopeless.
+# A retry budget is a bet that the failure was luck, and the bet is paid whether or not it
+# was -- which is why a step that fails deterministically wants max_attempts: 1, and why
+# step-timeout.yaml has no retry at all.
+#
+# The delay is data rather than a sleep. Each failure writes the next attempt's available_at
+# into the future and the worker moves on, so nothing holds a worker slot for the backoff.
+# Watch the attempts on the step: there are three, a second or so apart, each with its own
+# stderr, and the third one settles the step as failed and the run with it.
+#
+# This one uses shell.run, which executes code on the worker and is refused unless the
+# instance allowlists it: export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["shell.run"]', or pass
+# --enable-unsafe shell.run to a local run.
+#
+# Only config is interpolated, so max_attempts below is a literal: a retry budget is part of
+# the pipeline's shape, not something a caller talks the engine into at run time.
+#
+#   dg run --local examples/failure/retry-budget.yaml --enable-unsafe shell.run   # fails after 3 attempts
+
+format: dirigent/v1
+kind: pipeline
+code: retry-budget
+name: Retry budget
+description: |
+  A local command that always exits 1, given three attempts to prove it.
+
+  A non-zero exit is classified `unknown`, and **unknown is retried**: only `rejected`
+  ends a step on its first failure. The budget is spent in full, about a second apart,
+  and then the step fails.
+
+  A retry budget is a bet that the failure was luck. This one loses that bet three times,
+  which is what makes it worth reading.
+
+tags: [failure, execute, retry]
+
+requires:
+  blocks:
+    - shell.run
+
+steps:
+  reconcile:
+    block: shell.run
+    retry:
+      # The budget counts the first try, so 3 here means one attempt and two retries.
+      max_attempts: 3
+      # A second between attempts, doubling, so the whole thing settles in about three
+      # seconds instead of the thirty the default backoff would take.
+      backoff: 1s
+      max_backoff: 4s
+      multiplier: 2.0
+      jitter: 0.1
+    config:
+      # Deterministic: there is no exit code where this succeeds, and the engine has no way
+      # to know that. It prints which attempt it is on, so the three are told apart in logs.
+      command: "echo 'reconcile failed: ledger is short 3 rows' >&2; exit 1"
+
+  # Never reached. The default all_success edge is not satisfied by a step that spent its
+  # budget and failed, and there is no continue_on_failure here to tolerate it -- that is
+  # optional-step.yaml, where the failure is allowed and the branch carries on.
+  settle:
+    block: shell.run
+    depends_on: [reconcile]
+    config:
+      argv: [echo, "only reached if a retry ever worked"]