dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/patterns/sensor-storage-exists.yaml

@@ -0,0 +1,134 @@
+# storage.exists: waiting for an object, with a glob, a size floor, and a drop that arrives.
+#
+# deadline-on-a-sensor.yaml is this sensor missing its deadline. This one is the sensor doing
+# its job: something writes the object while the run is going, the sensor sees it, and the
+# pipeline carries on. Everything below is offline apart from one echo call, and the drop is
+# written by the run itself so there is nothing to arrange.
+#
+# THE TWO CONFIG FIELDS, and both earn their place:
+#
+#   uri       may contain a GLOB in its final segments. Waiting for
+#             drops/2026-01-01/*.json is how a pipeline waits for "whatever the producer
+#             called it today" instead of demanding a name nobody agreed to. The sensor's
+#             output says which object actually matched, which is why every step downstream
+#             reads ${steps.<sensor>.output.uri} rather than rebuilding the pattern.
+#   min_size  ignores an object until it is at least this large. A producer uploading a large
+#             file makes the key visible before the bytes are all there, so without a floor
+#             the sensor fires on a half-written object and the load reads a truncated file.
+#             This is the single most common way a drop-watching pipeline goes wrong.
+#
+# The output carries uri, size and modified_at -- what landed, how big it is, and when it was
+# last written -- which is everything a downstream step needs to decide whether to trust it.
+#
+# Hop by hop:
+#
+#   producer  fetches the payload the drop is made of. It stands in for the upstream system.
+#   drop      writes that payload to a URI under this run's scratch, which is the moment the
+#             sensor is waiting for: storage.write is the only way a value leaves a run.
+#   dropped   the sensor. It matches on a glob, insists on a minimum size, and fires as soon
+#             as the object is there.
+#   archive   copies what landed to its resting place, reading the sensor's answer rather than
+#             the pattern.
+#   receipt   reports what was seen: the object, its size, and when it was written.
+#
+# WHERE A URI MAY POINT. ${run.scratch} is this run's own directory under the instance's
+# artifact root, and file:// URIs are refused outside that root -- a stored pipeline cannot be
+# turned into an arbitrary-file read. A real drop is an s3:// URI, or a path under that root;
+# never an absolute path typed into a document.
+#
+# EXPECT THIS RUN TO SUCCEED in about two seconds, with the sensor firing on its first poke.
+#
+# To change it: raise -p min_size and the object is never big enough, so the sensor misses its
+# deadline and the run fails -- which is the same shape deadline-on-a-sensor.yaml shows
+# deliberately.
+#
+#   dg run --local examples/patterns/sensor-storage-exists.yaml
+#   dg run --local examples/patterns/sensor-storage-exists.yaml -p min_size=10mb   # fails at the deadline
+
+format: dirigent/v1
+kind: pipeline
+code: sensor-storage-exists
+name: Waiting for a drop
+description: |
+  `storage.exists` waits for an object at a URI, which may end in a **glob**, and ignores it
+  until it is at least `min_size` -- the guard against opening a file a producer is still
+  uploading.
+
+  Its output says which object matched, how big it is and when it was written, so downstream
+  reads the answer rather than rebuilding the pattern.
+
+tags: [patterns, http, sensor, storage, transform]
+
+requires:
+  blocks:
+    - http.request
+    - storage.write
+    - storage.exists
+    - storage.copy
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      description: The day the drop belongs to; it is a directory in the URI.
+      default: "2026-01-01"
+    min_size:
+      type: string
+      description: The floor the object has to clear. A humane size, so 1kb and 10mb both read as sizes.
+      default: 100b
+
+steps:
+  producer:
+    block: http.request
+    # Stands in for the upstream system: it only fetches, and what it fetches is a value in
+    # its output like any other.
+    config:
+      url: https://postman-echo.com/get
+      query:
+        day: "${params.day}"
+
+  drop:
+    block: storage.write
+    depends_on: [producer]
+    # The write is what the sensor below is waiting for. The producer names the file, which
+    # is why the sensor matches a pattern rather than this exact target.
+    config:
+      target: "${run.scratch}/drops/${params.day}/export-0001.json"
+      value: "${steps.producer.output.body}"
+
+  dropped:
+    block: storage.exists
+    depends_on: [drop]
+    poll: 1s
+    deadline: 15s
+    config:
+      # A glob in the final segment: the producer names the file, and the pipeline does not
+      # have to know what it will be called.
+      uri: "${run.scratch}/drops/${params.day}/*.json"
+      # A configured size is humane and carries no _bytes suffix: the type says the unit.
+      min_size: "${params.min_size}"
+
+  archive:
+    block: storage.copy
+    depends_on: [dropped]
+    config:
+      # The sensor's answer, not the pattern. A glob may have matched anything; only the
+      # sensor knows what it actually matched.
+      source: "${steps.dropped.output.uri}"
+      target: "${run.scratch}/archive/${params.day}.json"
+
+  receipt:
+    block: transform.jq
+    depends_on: [archive]
+    config:
+      input:
+        landed: "${steps.dropped.output.uri}"
+        size: "${steps.dropped.output.size}"
+        modified_at: "${steps.dropped.output.modified_at}"
+        archived: "${steps.archive.output.target}"
+        copied_bytes: "${steps.archive.output.bytes_copied}"
+      program: |
+        {landed, size, modified_at, archived, copied_bytes}

dirigent_examples/shelves/patterns/step-names-and-keys.yaml

@@ -0,0 +1,99 @@
+# The map key is the identity; name is a title nothing may ever reference.
+#
+# Every addressable thing in dirigent carries the same quartet, and each field has exactly one
+# job:
+#
+#   id           the uuid a machine holds. Never written in a document.
+#   code         the addressable key: constrained, unique, and what appears in a URL, in a
+#                document, and in every reference. A step's map key IS its code.
+#   name         an optional human title. Free-form, with no identity semantics at all --
+#                nothing may ever be referenced by it, and two things may share one.
+#   description  long-form and markdown-capable; the body a screen renders. A STEP carries no
+#                description -- only name and its key -- because a step's explanation is the
+#                comment above it, which travels in the document a person reads.
+#
+# So a step is referenced by its map key and by nothing else: ${steps.load_batch.output.status}
+# names load_batch, and renaming the STEP NAME below to anything at all leaves every reference
+# in this document working. Renaming the KEY breaks them, and breaks them at apply time, which
+# is the trade -- an identifier that is checkable is an identifier that cannot quietly drift.
+#
+# Two steps in this file deliberately share a name. That is legal, because a name identifies
+# nothing; the keys tell them apart. A screen shows the name where there is one and the key
+# otherwise, with the key always on the page in mono and never drawn twice.
+#
+# The same quartet, in the same shape, everywhere: the pipeline's code is its filename here
+# (the corpus test insists on it), a schedule's code addresses it, a webhook's code addresses
+# it, and a connection's code is what a document names instead of carrying a credential.
+#
+# Hop by hop:
+#
+#   load_batch   a key in snake_case, and a name that reads like a sentence.
+#   verify_rows  depends on the KEY above, not on its name, and reads its output the same way.
+#   publish      carries the same name as verify_rows, which nothing anywhere minds.
+#
+# EXPECT THIS RUN TO SUCCEED in about a second, with no network and no allowlist.
+#
+# To change it: edit any name below and re-run -- everything still works. Edit a key and the
+# document stops validating, with the dangling reference named.
+#
+#   dg run --local examples/patterns/step-names-and-keys.yaml
+
+format: dirigent/v1
+kind: pipeline
+code: step-names-and-keys
+name: Keys address, names describe
+description: |
+  A step's map **key** is its code: the only thing anything references, and what
+  `${steps.<key>.output...}` names.
+
+  `name` is a free-form title with no identity semantics -- two steps may share one, and
+  renaming it breaks nothing.
+
+tags: [patterns, transform, conventions, references]
+
+requires:
+  blocks:
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    batch:
+      type: string
+      description: Which batch this run covers.
+      default: 2026-01-01-cases
+
+steps:
+  load_batch:
+    # A title, for a person reading a run. Change it freely: nothing in this document, in the
+    # API, or in a URL refers to it.
+    name: Load the nightly batch
+    block: transform.jq
+    config:
+      input:
+        batch: "${params.batch}"
+      program: |
+        {batch, rows: 4120}
+
+  verify_rows:
+    name: Check the batch
+    block: transform.jq
+    # The key, never the name. A dependency on "Load the nightly batch" is not expressible,
+    # which is the point.
+    depends_on: [load_batch]
+    config:
+      # Same rule in a reference: steps.<key>.output.<field>.
+      input: "${steps.load_batch.output.value}"
+      program: |
+        {batch, rows, verified: (.rows > 0)}
+
+  publish:
+    # Deliberately the same name as the step above. Two rows may share a name because a name
+    # identifies nothing; publish and verify_rows are still two different steps.
+    name: Check the batch
+    block: transform.jq
+    depends_on: [verify_rows]
+    config:
+      input: "${steps.verify_rows.output.value}"
+      program: |
+        {published: .batch, rows}

dirigent_examples/shelves/patterns/timeout-fails-the-step.yaml

@@ -0,0 +1,94 @@
+# timeout: how long ONE block call may take, and what an expired one is classified as.
+#
+# Three different clocks can stop a step, and they answer three different questions:
+#
+#   timeout          (a step field)   how long one block call may take. This file.
+#   deadline         (a step field)   how long a step may keep WAITING on a sensor or a
+#                                     remote job. deadline-on-a-sensor.yaml.
+#   timeout          (block config)  the block's own limit on its own work, such as the
+#                                     HTTP client's socket timeout.
+#
+# The two on slow_read below are set deliberately far apart: the endpoint is asked to take ten
+# seconds, the HTTP client would allow thirty, and the engine allows two. So the engine is
+# plainly the clock that fires, and the attempt says so.
+#
+# An expired timeout is a TRANSIENT failure, which matters more than it looks: a retry budget
+# would be spent on it, three attempts against a call that is always too slow costing three
+# full timeouts of waiting. There is no retry policy here for exactly that reason -- a
+# deadline that cannot be met is not a flake. And note what on_timeout does NOT do: it governs
+# an expired deadline, never an expired timeout, so there is no spelling of this step that
+# skips instead of failing. timeout-skips-the-step.yaml is where that distinction lives.
+#
+# Hop by hop:
+#
+#   start      a local value, so the run has a step that plainly succeeded to compare against.
+#   slow_read  /delay/10 against a two-second budget. It loses, in about two seconds.
+#   parse      the default all_success edge, so it is skipped.
+#
+# EXPECT THIS RUN TO FAIL, in about two seconds, with slow_read failed and parse skipped.
+# It fails ON TIME, not on work: the endpoint was healthy and would have answered.
+#
+# To change it: -p seconds=1 finishes inside the budget and the run goes green. Widen the
+# step's timeout past ten and the same happens. Narrow the block's own timeout below two and
+# the HTTP client gives up first -- same red step, different reason, and the attempt's message
+# says which.
+#
+#   dg run --local examples/patterns/timeout-fails-the-step.yaml               # fails in ~2s
+#   dg run --local examples/patterns/timeout-fails-the-step.yaml -p seconds=1  # succeeds
+
+format: dirigent/v1
+kind: pipeline
+code: timeout-fails-the-step
+name: A step that runs out of time
+description: |
+  `timeout` bounds one block call. An expired one is a **transient** failure, so a retry
+  budget would be spent on it -- which is why this step has none.
+
+  As written a ten-second endpoint is given two seconds, the step fails in about two, and
+  the run reports `failed`.
+
+tags: [patterns, http, transform, failure, timeout]
+
+requires:
+  blocks:
+    - http.request
+    - value.const
+
+params:
+  type: object
+  properties:
+    seconds:
+      type: integer
+      description: How long the endpoint is asked to take. Anything above the two-second budget loses.
+      default: 10
+      minimum: 0
+      maximum: 30
+
+steps:
+  start:
+    block: value.const
+    config:
+      value:
+        note: everything above the failing step is ordinary
+
+  slow_read:
+    block: http.request
+    depends_on: [start]
+    # The budget the engine holds the call to. It is the shortest of the three clocks in this
+    # file, which is what makes it the one that fires.
+    timeout: 2s
+    config:
+      url: "https://postman-echo.com/delay/${params.seconds}"
+      method: GET
+      # The block's own clock, left generous so it is plainly not the one that stops this.
+      timeout: 30s
+
+  parse:
+    block: transform.jq
+    depends_on: [slow_read]
+    # Skipped. An abandoned attempt is a failure like any other as far as the edge is
+    # concerned; there is no separate "timed out" outcome for a rule to branch on.
+    config:
+      input: "${steps.slow_read.output.body}"
+      program: |
+        {delayed: .}

dirigent_examples/shelves/patterns/timeout-skips-the-step.yaml

@@ -0,0 +1,102 @@
+# on_timeout: skip -- the wait that ran out and did not make the run red.
+#
+# WHICH CLOCK on_timeout GOVERNS IS THE WHOLE POINT OF THIS FILE. It governs an expired
+# DEADLINE, which is how long a step may keep waiting. It has nothing to say about an expired
+# `timeout`, which bounds one block call and always fails the step as a transient failure --
+# timeout-fails-the-step.yaml is that one, and there is no spelling of it that skips.
+#
+# So this is the pair to read together. The same waiting step, one word apart:
+#
+#   on_timeout: fail   (the default)  the deadline passing is a broken pipeline.
+#                                     deadline-on-a-sensor.yaml.
+#   on_timeout: skip                  the deadline passing is a quiet day. This file.
+#
+# skip is the right answer whenever "it never showed up" is an ordinary outcome: no export
+# was published today, no drop landed, the upstream had nothing to say. Failing there trains
+# people to ignore red runs, which is worse than the day with no data.
+#
+# Hop by hop:
+#
+#   upstream_ready  http.ready is a sensor: each poke is one cheap GET, "not yet" is the
+#                   expected answer, and the run parks between pokes rather than holding a
+#                   worker. It pokes /status/503, which never answers 2xx, so it never becomes
+#                   ready and the six-second deadline expires.
+#   ingest          the default all_success edge, and a skipped prerequisite does not satisfy
+#                   it, so the skip carries down the branch.
+#   close_out       all_done, so the run still reports what it did with the day.
+#
+# EXPECT THIS RUN TO SUCCEED, in about six seconds, with upstream_ready and ingest both
+# skipped and close_out green. A run that did nothing is not a run that went wrong.
+#
+# To change it: -p status=200 makes the endpoint ready on the first poke and the whole branch
+# runs. Drop the on_timeout line and the identical run fails instead.
+#
+#   dg run --local examples/patterns/timeout-skips-the-step.yaml                 # succeeds, having skipped
+#   dg run --local examples/patterns/timeout-skips-the-step.yaml -p status=200   # succeeds, having worked
+
+format: dirigent/v1
+kind: pipeline
+code: timeout-skips-the-step
+name: A wait that skips instead of failing
+description: |
+  `on_timeout: skip` turns an expired **deadline** into a skipped step rather than a failed
+  one, and the skip carries down the branch through the default `all_success` edges.
+
+  It governs the deadline only: an expired `timeout` always fails. As written the sensor
+  never sees a ready endpoint, everything below it skips, and the run reports `succeeded`.
+
+tags: [patterns, http, sensor, timeout]
+
+requires:
+  blocks:
+    - http.ready
+    - http.request
+    - value.const
+
+params:
+  type: object
+  properties:
+    status:
+      type: integer
+      description: What the readiness endpoint answers; 503 is never ready, 200 is ready at once.
+      default: 503
+      enum: [200, 503]
+
+steps:
+  upstream_ready:
+    block: http.ready
+    # Two seconds between pokes, so three of them fit inside the deadline and the parking
+    # between them is visible in the log. A real readiness check on an hourly export uses
+    # minutes here, because each poke costs a scheduled row and nothing else.
+    poll: 2s
+    # Short so the example finishes while you watch it. This is the clock on_timeout is about.
+    deadline: 6s
+    # The one line this file is about.
+    on_timeout: skip
+    config:
+      # A sensor has no method: it looks, and looking is a GET. Readiness means a 2xx unless
+      # expect_status says otherwise, so a 503 is "not yet" rather than an error -- a sensor
+      # observing an unready service has observed correctly.
+      url: "https://postman-echo.com/status/${params.status}"
+
+  ingest:
+    block: http.request
+    depends_on: [upstream_ready]
+    # No rule written, so all_success, which a skipped prerequisite does not satisfy. That
+    # propagation is what makes on_timeout: skip a branch-level decision rather than a
+    # step-level one.
+    config:
+      url: https://postman-echo.com/post
+      method: POST
+      body:
+        ingested_after_status: "${steps.upstream_ready.output.status}"
+
+  close_out:
+    block: value.const
+    depends_on: [ingest]
+    # all_done, so the run has an outcome either way: a skipped day is a fact worth recording,
+    # not an absence.
+    rule: all_done
+    config:
+      value:
+        note: nothing to ingest is a normal day

dirigent_examples/shelves/patterns/webhook-mapping-nested-payload.yaml

@@ -0,0 +1,125 @@
+# An inbound webhook, and the strict mapping that is the whole of its interface.
+#
+# A webhook declares one thing: which JSONPath in the delivered payload becomes which
+# parameter. That mapping is the entire attack surface, and it is strict in both directions:
+#
+#   * Anything the payload carries that is not named here is IGNORED. A caller who adds a
+#     field, or fifty, changes nothing about the run.
+#   * A parameter the mapping does not name cannot be reached at all, whatever is POSTed. So
+#     environment below is unreachable from outside and keeps its default -- which is how a
+#     webhook is exposed without letting the internet choose which environment to write to.
+#   * What is mapped is validated against params like any other run's parameters. A payload
+#     with a batch_size of 10000000 is refused by the schema, not by a step.
+#
+# The paths are JSONPath into the delivered body, so a nested payload needs no unwrapping step:
+# $.published.date reaches two levels down, and $.source.system.name three. Reshaping the
+# payload inside the pipeline would be doing at run time what the mapping already does at
+# intake, one layer later and with worse errors.
+#
+# WHAT STAYS ON THE INSTANCE. The token is minted server-side and shown exactly once. It is
+# not in this file, it will not be in an export, and there is nowhere in dirigent/v1 to write
+# one. The same is true of whether the endpoint is enabled and of the delivery history: those
+# are facts about one instance, so an apply never touches them.
+#
+# Hop by hop:
+#
+#   ingest   shapes the mapped parameters into the record a loader would be handed.
+#   receipt  reads it back, standing in for the work.
+#
+# EXPECT THIS RUN TO SUCCEED in about a second, on the pipeline's defaults -- the webhook
+# accepts deliveries only on an instance the document has been applied to.
+#
+#   dg run --local examples/patterns/webhook-mapping-nested-payload.yaml
+#   dg apply examples/patterns/webhook-mapping-nested-payload.yaml
+#   dg webhook list webhook-mapping-nested-payload
+#   dg webhook rotate-token webhook-mapping-nested-payload upstream-publish   # shows a token once
+#   curl -X POST "$DG_URL/hooks/$TOKEN" -H 'content-type: application/json' -d '{
+#     "published": {"date": "2026-01-01", "dataset": {"code": "cases"}},
+#     "source": {"system": {"name": "warehouse"}},
+#     "counts": {"rows": 4120},
+#     "anything_else": "ignored"
+#   }'
+#   dg webhook deliveries webhook-mapping-nested-payload upstream-publish
+
+format: dirigent/v1
+kind: pipeline
+code: webhook-mapping-nested-payload
+name: A webhook mapped from a nested payload
+description: |
+  `params_from_payload` maps JSONPaths in the delivered body onto parameters, and that
+  mapping is the whole interface: unmapped fields are ignored, unmapped parameters are
+  unreachable, and what is mapped is validated like any other run's parameters.
+
+  The token, the enabled flag and the delivery history stay on the instance; none of them
+  belongs in git.
+
+tags: [patterns, transform, webhook]
+
+requires:
+  blocks:
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      description: The day the upstream published; mapped from the payload.
+      default: "2026-01-01"
+    dataset:
+      type: string
+      description: Which dataset was published; mapped from two levels down.
+      default: cases
+      enum: [cases, climate, population]
+    source:
+      type: string
+      description: Which upstream system published it; mapped from three levels down.
+      default: warehouse
+      minLength: 1
+      maxLength: 64
+    rows:
+      type: integer
+      description: How many rows the upstream says it published; bounded, so a delivery cannot claim anything.
+      default: 0
+      minimum: 0
+      maximum: 10000000
+    environment:
+      type: string
+      description: Deliberately absent from the mapping, so no delivery can choose it.
+      default: staging
+      enum: [staging, production]
+
+steps:
+  ingest:
+    block: transform.jq
+    config:
+      input:
+        day: "${params.day}"
+        dataset: "${params.dataset}"
+        source: "${params.source}"
+        rows: "${params.rows}"
+        environment: "${params.environment}"
+      program: |
+        {day, dataset, source, rows, environment}
+
+  receipt:
+    block: transform.jq
+    depends_on: [ingest]
+    config:
+      input: "${steps.ingest.output.value}"
+      program: |
+        {ingested: "\(.dataset) for \(.day) from \(.source)", rows, environment}
+
+triggers:
+  webhooks:
+    - code: upstream-publish
+      name: Upstream publish notification
+      description: Fired by the warehouse when a dataset finishes publishing.
+      params_from_payload:
+        # One line per parameter, each a JSONPath into the delivered body. Depth costs
+        # nothing; a field the mapping does not name is not reachable at any depth.
+        day: "$.published.date"
+        dataset: "$.published.dataset.code"
+        source: "$.source.system.name"
+        rows: "$.counts.rows"