dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/recipes/schema-carried.yaml

@@ -0,0 +1,112 @@
+# A shape written into the document itself, so the pipeline runs anywhere with nothing to
+# hand it.
+#
+# In:  a payload shaped like an API read -- an envelope around a list of org units.
+# Out: the same payload, passed through the gate unchanged, and a count taken from the
+#      gate's output rather than from the step before it.
+#
+# A validate.schema gate never writes a shape inline in its config: its schema field is a
+# code, and the code resolves either to a schema the instance holds or to one this document
+# carries in its top-level schemas: section. This recipe is the carried half; its pair,
+# schema-referenced.yaml, is the same check written the other way.
+#
+# Carried is the right choice when the shape belongs to this pipeline and nothing else, and
+# when the document has to run somewhere that holds nothing -- a --local run, a colleague's
+# checkout, a CI lane. The cost is that two documents carrying the same shape are two copies
+# that drift.
+#
+# What the shape says, and why each keyword is there:
+#
+#   required            presence, which is a different question from type. A field named in
+#                       properties but not required may be absent; one that is present is
+#                       still held to its type.
+#   additionalProperties: false
+#                       closes the door. JSON Schema is open by default, so an unexpected
+#                       field passes silently unless the schema says otherwise. Closed is
+#                       right for a payload this pipeline fully owns and wrong for one an
+#                       upstream is free to extend.
+#   minItems            an empty list is a shape that fits every item schema, so "there is
+#                       at least one" has to be said out loud.
+#   pattern, minimum    the value-level narrowing that turns a type check into a real gate.
+#
+# The gate is also a waypoint. Steps below it read ${steps.gate.output.value}, which is the
+# input unchanged, and that reference is the document's own proof that nothing bypassed the
+# check.
+#
+# To change it: -p level=0 makes the payload violate `minimum: 1`, and that run fails at the
+# gate with the failing instance path and keyword named.
+#
+# The schema is carried in the document, so no server stores this one: it runs with
+# `dg run --local`, and on an instance the same shape is created once with `dg schema create`.
+#
+#   dg run --local examples/recipes/schema-carried.yaml
+#   dg run --local examples/recipes/schema-carried.yaml -p level=0
+
+format: dirigent/v1
+kind: pipeline
+code: schema-carried
+name: A schema carried in the document
+description: Validate a payload against a shape the document carries, and read the gate's output downstream as proof of the check.
+
+tags: [recipes, transform, validate]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+    - validate.schema
+
+schemas:
+  recipe-org-units:
+    title: Organisation units read
+    description: The envelope an org-unit read answers with, and the fields a loader depends on.
+    type: object
+    required: [organisationUnits]
+    additionalProperties: false
+    properties:
+      organisationUnits:
+        type: array
+        minItems: 1
+        items:
+          type: object
+          required: [id, displayName, level]
+          additionalProperties: false
+          properties:
+            id: { type: string, pattern: "^[A-Za-z][A-Za-z0-9]{10}$" }
+            displayName: { type: string, minLength: 1 }
+            level: { type: integer, minimum: 1 }
+
+params:
+  type: object
+  properties:
+    level:
+      type: integer
+      default: 2
+      description: The level the second org unit reports; 0 makes the payload fail the gate.
+
+steps:
+  payload:
+    block: value.const
+    config:
+      value:
+        organisationUnits:
+          - { id: ImspTQPwCqd, displayName: Sierra Leone, level: 1 }
+          # The level is a parameter so the gate can be made to refuse without editing the
+          # document: value.const emits what it is given, references included.
+          - { id: O6uvpzGd5pu, displayName: Bo, level: "${params.level}" }
+
+  gate:
+    block: validate.schema
+    depends_on: [payload]
+    config:
+      input: ${steps.payload.output.value}
+      schema: recipe-org-units
+
+  count:
+    block: transform.jq
+    depends_on: [gate]
+    config:
+      input: ${steps.gate.output.value}
+      program: |
+        {units: (.organisationUnits | length),
+         levels: [.organisationUnits[].level] | unique}

dirigent_examples/shelves/recipes/schema-formats.yaml

@@ -0,0 +1,107 @@
+# format: the keyword that says what kind of string a string is, and here actually asserts.
+#
+# In:  one record of strings -- a date, a timestamp, an email, a uri, a uuid, a hex digest,
+#      and one string carrying a format no pack on this instance contributes.
+# Out: the record, through a gate that checked every one of them.
+#
+# In JSON Schema, format is an annotation by default: a validator only asserts it when it is
+# handed a format checker, and plenty are not. Dirigent's engine always hands its validator
+# one, so a format written here is enforced rather than decorative. That is the difference
+# worth knowing before writing a schema for this engine and reusing it somewhere else.
+#
+# The standard formats are all available -- date, date-time, time, email, uri, hostname,
+# ipv4, ipv6, regex, and the rest -- and dirigent registers the ones its own data speaks:
+# ulid, uuid4, uuid7, md5, sha1, sha256, sha512, base64.
+#
+# A format only ever narrows a string. A value of the wrong type is caught by `type`, and
+# format speaks only once the value is already a string, so `{type: integer, format: date}`
+# asserts nothing about anything.
+#
+# The last field is the interesting one. acme-site-id is a format a plugin pack
+# contributes; on an instance with that pack installed it asserts, and on one without it the
+# keyword is a passing annotation and the value is simply unchecked. That is what keeps a
+# schema portable: the same file asserts more on an instance that knows more, and refuses
+# nothing it cannot check. This recipe runs green either way, and on an instance with the
+# pack a site id of the wrong shape there would fail it.
+#
+# To change it: -p email=not-an-address, -p when=last%20Tuesday, or -p digest=abc all fail
+# the gate, each naming the field and the format it did not satisfy.
+#
+# The schema is carried in the document, so no server stores this one: it runs with
+# `dg run --local`, and on an instance the same shape is created once with `dg schema create`.
+#
+#   dg run --local examples/recipes/schema-formats.yaml
+#   dg run --local examples/recipes/schema-formats.yaml -p email=not-an-address
+
+format: dirigent/v1
+kind: pipeline
+code: schema-formats
+name: Formats that actually assert
+description: Hold a record of strings to date, date-time, email, uri, uuid4 and digest formats, and show what a pack-contributed format does.
+
+tags: [recipes, validate]
+
+requires:
+  blocks:
+    - value.const
+    - validate.schema
+
+schemas:
+  recipe-formatted-record:
+    type: object
+    required: [day, when, email, endpoint, request_id, digest, site]
+    additionalProperties: false
+    properties:
+      day: { type: string, format: date }
+      when: { type: string, format: date-time }
+      email: { type: string, format: email }
+      endpoint: { type: string, format: uri }
+      request_id: { type: string, format: uuid4 }
+      digest: { type: string, format: sha256 }
+      # Contributed by a plugin pack. Where that pack is installed this asserts; where it
+      # is not, the value passes unchecked and the schema stays portable.
+      site: { type: string, format: acme-site-id }
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      default: "2026-01-01"
+      description: Held to format date; "last Tuesday" is refused.
+    when:
+      type: string
+      default: "2026-01-01T06:00:00Z"
+      description: Held to format date-time, which wants a zone as well as a time.
+    email:
+      type: string
+      default: ops@example.org
+      description: Held to format email.
+    endpoint:
+      type: string
+      default: https://postman-echo.com/get
+      description: Held to format uri.
+    digest:
+      type: string
+      default: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+      description: Held to format sha256, which is exactly 64 hex characters.
+
+steps:
+  record:
+    block: value.const
+    config:
+      value:
+        day: "${params.day}"
+        when: "${params.when}"
+        email: "${params.email}"
+        endpoint: "${params.endpoint}"
+        request_id: f47ac10b-58cc-4372-a567-0e02b2c3d479
+        digest: "${params.digest}"
+        site: NO-OSLO-01
+
+  gate:
+    block: validate.schema
+    depends_on: [record]
+    config:
+      input: ${steps.record.output.value}
+      schema: recipe-formatted-record

dirigent_examples/shelves/recipes/schema-referenced.yaml

@@ -0,0 +1,86 @@
+# The same gate, written the other way: the shape lives on the instance and the document
+# names it.
+#
+# In:  one org-unit record.
+# Out: that record, passed through a gate that names ou-record -- a schema the instance
+#      holds, not one this file carries.
+#
+# A schema is a resource in its own right, applied like a connection rather than as part of
+# a pipeline:
+#
+#   dg schema create examples/schemas/ou-record.json
+#   dg schema list
+#
+# and its identity comes out of its own keywords: $id becomes the code every reference uses,
+# title its name, description its body. So the code in the gate below is the $id in that
+# file and nothing else.
+#
+# Referenced is the right choice as soon as a second pipeline holds the same picture of the
+# same payload. One shape, one place to change it, and every pipeline that names it is
+# refused the moment they stop agreeing. It costs a dependency: requires.schemas is where
+# the document says so, and an instance that does not hold ou-record refuses this document
+# at apply, naming what to create first, rather than failing at three in the morning.
+#
+# A --local run starts on an empty throwaway instance, so the schema is handed to it the way
+# connections are:
+#
+#   dg run --local --schema examples/schemas/ou-record.json \
+#     examples/recipes/schema-referenced.yaml
+#
+# Without --schema that run is refused before anything executes, which is the preflight
+# doing its job.
+#
+# To change it: -p level=zero sends a string where the schema wants an integer, and that run
+# fails at the gate naming the instance path and the keyword.
+#
+#   dg run --local --schema examples/schemas/ou-record.json examples/recipes/schema-referenced.yaml
+#   dg run --local --schema examples/schemas/ou-record.json examples/recipes/schema-referenced.yaml -p level=zero
+
+format: dirigent/v1
+kind: pipeline
+code: schema-referenced
+name: A schema the instance holds
+description: Validate a record against ou-record, a schema applied to the instance on its own, and declare the dependency in requires.
+
+tags: [recipes, transform, validate]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+    - validate.schema
+  # The preflight that refuses this document on an instance holding no such schema.
+  schemas:
+    - ou-record
+
+params:
+  type: object
+  properties:
+    level:
+      default: 2
+      description: The record's level; anything but an integer is refused at the gate.
+
+steps:
+  record:
+    block: value.const
+    config:
+      value:
+        id: O6uvpzGd5pu
+        name: Bo
+        level: "${params.level}"
+
+  gate:
+    block: validate.schema
+    depends_on: [record]
+    config:
+      input: ${steps.record.output.value}
+      # A code, never a shape: this one names the schema stored under $id ou-record.
+      schema: ou-record
+
+  describe:
+    block: transform.jq
+    depends_on: [gate]
+    config:
+      input: ${steps.gate.output.value}
+      program: |
+        {checked_id: .id, at_level: .level}

dirigent_examples/shelves/recipes/schema-refuses-then-rule.yaml

@@ -0,0 +1,127 @@
+# A gate that refuses, and the two branches below it: the one that runs and the one that
+# does not.
+#
+# THIS RUN FAILS ON PURPOSE. It ends `failed`, with the gate failed, the load skipped, and
+# the alert succeeded. That is the outcome the recipe is here to show; nothing is broken.
+#
+# In:  a payload whose second record carries a level of 0, which the carried schema refuses.
+# Out: a failed run whose alert step has a summary of what went wrong, and a load step that
+#      never started.
+#
+# The edge into a step carries a rule, and the rule is what decides whether that step runs
+# once its dependencies have settled:
+#
+#   all_success   the default. Every dependency succeeded. The load below never starts.
+#   one_failed    at least one dependency failed. This is an error handler, and it is drawn
+#                 in the DAG rather than hidden inside a block's config.
+#   all_done      every dependency finished, whatever it finished as. This is the cleanup
+#                 edge: a step that must run whether the work passed or failed.
+#
+# A handler cannot read the output of the step that failed, because a failed step has no
+# stored output; there is nothing to reference. So the alert reads what it can -- the run's
+# own identity, and the payload as it was before the gate -- and says which pipeline and
+# which run to look at. Reaching for ${steps.gate.output.value} in a one_failed handler is
+# refused when the document is applied, which is the format catching the mistake early.
+#
+# The run's status is still `failed`, and that is deliberate: handling a failure is not the
+# same as not having had one. A step that is allowed to fail without failing the run says so
+# with `continue_on_failure`, which is a different recipe and a different intent.
+#
+# To change it: -p level=2 makes the payload fit the schema, and the same document runs
+# green with the load taken and the alert skipped -- the pair worth running back to back.
+#
+# The schema is carried in the document, so no server stores this one: it runs with
+# `dg run --local`, and on an instance the same shape is created once with `dg schema create`.
+#
+#   dg run --local examples/recipes/schema-refuses-then-rule.yaml            # fails, by design
+#   dg run --local examples/recipes/schema-refuses-then-rule.yaml -p level=2 # succeeds
+
+format: dirigent/v1
+kind: pipeline
+code: schema-refuses-then-rule
+name: A refused gate and what runs after it
+description: A validation that fails on purpose, an error branch on rule one_failed, a load that is skipped, and a cleanup on all_done.
+
+tags: [recipes, transform, validate, failure]
+
+requires:
+  blocks:
+    - value.const
+    - transform.jq
+    - validate.schema
+
+schemas:
+  recipe-levelled-units:
+    type: object
+    required: [organisationUnits]
+    properties:
+      organisationUnits:
+        type: array
+        minItems: 1
+        items:
+          type: object
+          required: [id, level]
+          properties:
+            id: { type: string, minLength: 1 }
+            level: { type: integer, minimum: 1 }
+
+params:
+  type: object
+  properties:
+    level:
+      type: integer
+      default: 0
+      description: The second unit's level. The default of 0 is what the schema refuses.
+
+steps:
+  payload:
+    block: value.const
+    config:
+      value:
+        organisationUnits:
+          - { id: ImspTQPwCqd, level: 1 }
+          - { id: O6uvpzGd5pu, level: "${params.level}" }
+
+  gate:
+    block: validate.schema
+    depends_on: [payload]
+    config:
+      input: ${steps.payload.output.value}
+      schema: recipe-levelled-units
+
+  load:
+    block: transform.jq
+    depends_on: [gate]
+    # The default rule, written out because this document is about rules. The gate failed,
+    # so this step is skipped and nothing it would have done happened.
+    rule: all_success
+    config:
+      input: ${steps.gate.output.value}
+      program: |
+        {loaded: (.organisationUnits | length)}
+
+  alert:
+    block: transform.jq
+    depends_on: [gate]
+    rule: one_failed
+    config:
+      # A failed step has no stored output, so the handler reads the run's own identity and
+      # the payload as it stood before the gate.
+      input:
+        run: ${run.id}
+        payload: ${steps.payload.output.value}
+      program: |
+        {alert: "org unit levels failed validation",
+         run: .run,
+         units: (.payload.organisationUnits | length),
+         suspect: [.payload.organisationUnits[] | select(.level < 1) | .id]}
+
+  close_out:
+    block: transform.jq
+    depends_on: [load, alert]
+    # Satisfied by a skip as much as by a success, so the run always finishes here.
+    rule: all_done
+    config:
+      input: ${run.id}
+      program: |
+        {closed_run: .}

dirigent_examples/shelves/recipes/storage-copy-dated-archive.yaml

@@ -0,0 +1,114 @@
+# Keep a dated copy of what a run produced, with the layout decided in the document.
+#
+# In:  a payload built inline and handed to storage.write.
+# Out: that file, plus a copy at a dated archive key, and a manifest naming both with the
+#      byte count the copy reported.
+#
+# The first two hops are the pair every value makes on its way out: transform.jq produces a
+# value, storage.write puts it at a URI, and the copy below works on the URI the write
+# reported rather than on anything the run is still holding.
+#
+# storage.copy moves bytes from one URI to another and knows nothing else. It is not an S3
+# block or a file block: the scheme in the URI decides the backend, so file:// to file://,
+# file:// to s3:// and s3:// to s3:// are all the same step, and adding an object store to
+# an instance changes no document.
+#
+# The archive key is where the thinking goes, because a key layout is a decision nobody
+# revisits once a year of files is written under it. This one is prefix/YYYY/MM/DD/name:
+#
+#   Date parts are separate segments, so a listing can be taken per year or per month
+#   without parsing names, and a lifecycle rule can expire a prefix.
+#   The date is the run's parameter and never the wall clock, so a rerun for last Tuesday
+#   writes last Tuesday's key rather than today's.
+#   The name is stable inside the dated prefix, so the newest file is always the one under
+#   the newest key rather than the one with the highest suffix.
+#
+# The copy reports bytes_copied, which is the cheapest possible integrity check: a manifest
+# carrying it lets a later run notice a zero-byte archive without opening it.
+#
+# To change it: -p archive_prefix=... points the same copy at another root, and -p
+# day=2026-02-03 writes into 2026/02/03 instead.
+#
+#   dg run --local examples/recipes/storage-copy-dated-archive.yaml
+#   dg run --local examples/recipes/storage-copy-dated-archive.yaml -p day=2026-02-03
+
+format: dirigent/v1
+kind: pipeline
+code: storage-copy-dated-archive
+name: A dated archive copy
+description: Write a file, copy it to a date-partitioned archive key, and record both URIs and the bytes copied in a manifest.
+
+tags: [recipes, storage, transform]
+
+requires:
+  blocks:
+    - transform.jq
+    - storage.write
+    - storage.copy
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day being archived; the archive key is built from it, not from now.
+    archive_prefix:
+      type: string
+      default: archive/readings
+      description: The root the dated key hangs under.
+
+steps:
+  build:
+    block: transform.jq
+    config:
+      input:
+        day: ${params.day}
+      program: |
+        {day: .day,
+         readings: [range(5) | {station: "st-\(.)", celsius: (. * 2) - 3}]}
+
+  store:
+    block: storage.write
+    depends_on: [build]
+    config:
+      target: ${run.scratch}/current/readings.json
+      value: ${steps.build.output.value}
+
+  key:
+    block: transform.jq
+    depends_on: [build]
+    config:
+      input:
+        day: ${params.day}
+        prefix: ${params.archive_prefix}
+      program: |
+        . as {$day, $prefix}
+        | ($day | split("-")) as [$year, $month, $date]
+        | {key: "\($prefix)/\($year)/\($month)/\($date)/readings.json",
+           year: $year, month: $month, date: $date}
+
+  archive:
+    block: storage.copy
+    depends_on: [store, key]
+    config:
+      source: ${steps.store.output.uri}
+      # The key comes from the step that built it, so the layout is written once.
+      target: ${run.scratch}/${steps.key.output.value.key}
+
+  manifest:
+    block: transform.jq
+    depends_on: [archive]
+    config:
+      input:
+        source: ${steps.archive.output.source}
+        target: ${steps.archive.output.target}
+        bytes: ${steps.archive.output.bytes_copied}
+        day: ${params.day}
+      program: |
+        {day, source, target,
+         archived_bytes: .bytes,
+         # A zero-byte archive is the failure a manifest is for: it is visible here without
+         # anybody opening the object.
+         empty: (.bytes == 0)}

dirigent_examples/shelves/recipes/storage-exists-gate.yaml

@@ -0,0 +1,127 @@
+# Wait for an object before reading it, and set the floor that stops a half-written file
+# being read.
+#
+# In:  a file this pipeline writes itself, so the sensor has something to find.
+# Out: the file's URI, size and modification time from the sensor, and a summary read from
+#      it afterwards.
+#
+# storage.exists is a sensor, not an operator. It observes and changes nothing, each poke is
+# one cheap head request, and "not yet" is the expected answer rather than a failure. While
+# it waits, the run holds no worker: it parks and is poked again on the schedule the step
+# sets.
+#
+# The three knobs are step-level engine semantics, uniform across every sensor and never
+# inside a block's config:
+#
+#   poll        how often to look. Seconds here because the file is already there; minutes
+#               is the honest setting for a drop somebody else produces.
+#   deadline    how long to keep looking before giving up. A sensor without one waits 24h.
+#   on_timeout  what a deadline means. `skip` says nobody dropped a file inside the window,
+#               which is a quiet day rather than a broken pipeline, and the steps below skip
+#               with it. `fail` says the drop was promised, and its absence is an incident.
+#
+# min_size is the one that saves a real pipeline. A producer uploading a large object makes
+# the key visible before the bytes are all there, so a sensor without a floor hands a
+# truncated file to the reader below it. The floor is set to what the smallest legitimate
+# file could be, never to what a typical one is.
+#
+# The uri may glob in its final segments, so `${run.scratch}/drops/*.json` waits for
+# whichever file arrives; the sensor's output names the one it actually found, which is why
+# the reader below reads ${steps.dropped.output.uri} and not the pattern.
+#
+# The sensor observes, it does not read: it reports a URI, a size and a time, and nothing
+# about what is inside. Turning that into a value is storage.read's job, one hop later, and
+# that hop is where max_size decides how much of the drop this run will hold.
+#
+# To change it: point uri at a drop somebody else writes and raise the deadline; with
+# on_timeout: skip that run ends green having done nothing, which is what a quiet day
+# should look like.
+#
+#   dg run --local examples/recipes/storage-exists-gate.yaml
+#   dg run --local examples/recipes/storage-exists-gate.yaml -p min_size=1mb   # skips: no file is that big
+
+format: dirigent/v1
+kind: pipeline
+code: storage-exists-gate
+name: Wait for an object, then read it
+description: Park on a storage.exists sensor with a size floor and a deadline, then read whatever file the sensor reports it found.
+
+tags: [recipes, sensor, storage, transform]
+
+requires:
+  blocks:
+    - transform.jq
+    - storage.write
+    - storage.exists
+    - storage.read
+
+params:
+  type: object
+  properties:
+    day:
+      type: string
+      format: date
+      default: "2026-01-01"
+      description: The day the drop is named for.
+    min_size:
+      type: string
+      default: 32b
+      description: The floor that stops a half-written object being read. 1mb makes this run skip.
+
+steps:
+  produce:
+    block: transform.jq
+    config:
+      input:
+        day: ${params.day}
+      program: |
+        {day: .day, readings: [range(8) | {station: "st-\(.)", celsius: (. - 3)}]}
+
+  drop:
+    block: storage.write
+    depends_on: [produce]
+    config:
+      target: ${run.scratch}/drops/${params.day}.json
+      value: ${steps.produce.output.value}
+
+  dropped:
+    block: storage.exists
+    depends_on: [drop]
+    poll: 2s
+    deadline: 30s
+    # A missing drop is a quiet day: the run ends green with this and everything below it
+    # skipped, rather than raising an incident nobody can act on.
+    on_timeout: skip
+    config:
+      # A glob, so whichever file lands that day is found; the sensor reports which.
+      uri: ${run.scratch}/drops/*.json
+      min_size: ${params.min_size}
+
+  read:
+    block: storage.read
+    depends_on: [dropped]
+    config:
+      # The file the sensor found, not the pattern it was given.
+      source: ${steps.dropped.output.uri}
+      max_size: 1mb
+
+  summary:
+    block: transform.jq
+    depends_on: [read]
+    config:
+      input: ${steps.read.output.value}
+      program: |
+        {day, rows: (.readings | length),
+         warmest: (.readings | max_by(.celsius) | .station)}
+
+  receipt:
+    block: transform.jq
+    depends_on: [dropped, summary]
+    config:
+      input:
+        uri: ${steps.dropped.output.uri}
+        size: ${steps.dropped.output.size}
+        modified_at: ${steps.dropped.output.modified_at}
+        summary: ${steps.summary.output.value}
+      program: |
+        {found: .uri, size_bytes: .size, modified_at, summary}