dirigent-examples 0.17.3__tar.gz → 0.18.1__tar.gz

{dirigent_examples-0.17.3 → dirigent_examples-0.18.1}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: dirigent-examples
-Version: 0.17.3
+Version: 0.18.1
 Summary: The dirigent example corpus: the shelves of runnable documents, installed.
 License-Expression: LicenseRef-Proprietary
 License-File: LICENSE
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: dirigent-client==0.17.3
-Requires-Dist: dirigent-plugin==0.17.3
+Requires-Dist: dirigent-client==0.18.1
+Requires-Dist: dirigent-plugin==0.18.1
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 

{dirigent_examples-0.17.3 → dirigent_examples-0.18.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-examples"
-version = "0.17.3"
+version = "0.18.1"
 description = "The dirigent example corpus: the shelves of runnable documents, installed."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -11,8 +11,8 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-    "dirigent-client==0.17.3",
-    "dirigent-plugin==0.17.3",
+    "dirigent-client==0.18.1",
+    "dirigent-plugin==0.18.1",
 ]
 
 [project.entry-points."dirigent.plugins.v1"]

{dirigent_examples-0.17.3 → dirigent_examples-0.18.1}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-examples"
-version = "0.17.3"
+version = "0.18.1"
 description = "The dirigent example corpus: the shelves of runnable documents, installed."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -12,8 +12,8 @@ classifiers = [
 ]
 dependencies = [
     # The python/ shelf is SDK scripts a reader runs, so the corpus carries the client.
-    "dirigent-client==0.17.3",
-    "dirigent-plugin==0.17.3",
+    "dirigent-client==0.18.1",
+    "dirigent-plugin==0.18.1",
 ]
 
 [project.entry-points."dirigent.plugins.v1"]

{dirigent_examples-0.17.3 → dirigent_examples-0.18.1}/src/dirigent_examples/shelves/docker/docker-remote-daemon.yaml

@@ -26,8 +26,9 @@
 # only as a 0600 file under the run's work directory that goes when the step leaves. A client
 # key for a docker daemon is a credential for root on whatever that daemon runs on.
 #
-# TO MAKE IT YOURS: point host at your own daemon. A daemon that needs no TLS -- an ssh://
-# host, or a unix socket on another path -- takes host alone and none of the three PEMs.
+# TO MAKE IT YOURS: point host at your own daemon. The worker speaks tcp:// and unix://, and
+# a daemon that needs no TLS -- a unix socket on another path -- takes host alone and none of
+# the three PEMs.
 
 format: dirigent/v1
 kind: pipeline

{dirigent_examples-0.17.3 → dirigent_examples-0.18.1}/src/dirigent_examples/shelves/patterns/README.md

@@ -117,6 +117,7 @@ local run is handed with `--also-apply examples/patterns/pipeline-run-child.yaml
 | [sensor-storage-exists.yaml](sensor-storage-exists.yaml) | A glob, a size floor, and a drop that actually lands | `succeeded` |
 | [log-levels.yaml](log-levels.yaml) | `--log-level PATTERN=LEVEL`, a run setting rather than a document one | `succeeded` |
 | [priority-layered.yaml](priority-layered.yaml) | `priority` on the document, on a schedule, and on one ad hoc run | `succeeded` |
+| [importance-critical.yaml](importance-critical.yaml) | `importance` beside `priority`, and the alert rule that reads it | `succeeded` |
 | [connections-referenced-vs-carried.yaml](connections-referenced-vs-carried.yaml) | Named, carried, and absolute; needs `--connections` | `succeeded` |
 
 ## Running them

dirigent_examples-0.18.1/src/dirigent_examples/shelves/patterns/importance-critical.yaml

@@ -0,0 +1,108 @@
+# How much a pipeline matters, which is a different question from how urgently it runs.
+#
+# A document carries TWO words that sound alike and are not:
+#
+#   priority: low | normal | high            when the claim takes this pipeline's attempts
+#   importance: routine | normal | critical  how much it matters that this pipeline works
+#
+# They are deliberately spelled from different vocabularies so no one reads them as one
+# scale. THIS PIPELINE IS BOTH: low priority and critical importance. It runs behind
+# everything a person is waiting on, because it is a slow nightly reconciliation and nobody
+# is watching it go. And when it fails, somebody is woken, because the ledger it writes is
+# what the morning's payments are drawn from.
+#
+# WHO READS EACH ONE. `priority` is read by the claim, once per attempt, and it is pinned on
+# the run at creation. `importance` is read by the alerting rules, when a run settles: a rule
+# may name the LEAST importance a pipeline must carry before it fires.
+#
+#   dg alerts rules create page-ops --event run_failed --importance critical
+#   dg alerts rules create log-all  --event run_failed
+#
+# The first fires for this pipeline and for no lesser one. The second names no importance, so
+# it fires for every pipeline -- this one included. PAGING ADDS TO THE LOG; it does not
+# replace it, so a critical failure raises both and an ordinary one raises only the second.
+#
+# `dg alerts rules list` says the floor inside the scope cell -- "global, critical and above"
+# -- rather than in a column that would be empty on almost every row.
+#
+# WHERE IT IS SAID. `dg pipeline show` prints it with the other document settings, the
+# pipelines listing marks a critical pipeline beside its title, and a listing row carries it
+# on the wire. A routine or normal pipeline wears no mark at all: almost every pipeline is
+# one of those, and a word on every row would say nothing.
+#
+# Hop by hop:
+#
+#   read       the balances the day left behind.
+#   reconcile  the arithmetic that either agrees or does not.
+#   ledger     what the morning is drawn from, which is the reason for the word.
+#
+# EXPECT THIS RUN TO SUCCEED in about a second, with no network.
+#
+#   dg run --local examples/patterns/importance-critical.yaml
+#   dg apply examples/patterns/importance-critical.yaml
+#   dg pipeline show importance-critical        # importance: critical, with the rest
+#   dg alerts rules list                        # the floor, inside the scope cell
+
+format: dirigent/v1
+kind: pipeline
+code: importance-critical
+name: Slow to be claimed, loud when it breaks
+description: |
+  `importance` is one word -- `routine`, `normal` or `critical` -- and it says how much this
+  pipeline matters, not when its work is claimed. `priority` says that, and the two are
+  independent: this document is `low` priority and `critical` importance at once.
+
+  An alert rule may name the least importance a pipeline must carry before it fires, so one
+  rule pages for the critical failures and another logs every failure there is.
+
+tags: [patterns, transform, importance, alerting]
+
+# Behind whatever somebody is waiting on: it is slow, and nobody is watching it run.
+priority: low
+
+# But the morning's payments are drawn from what it writes, so a failure is worth waking for.
+importance: critical
+
+requires:
+  blocks:
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    accounts:
+      type: array
+      description: The accounts the day's balances are reconciled over.
+      items:
+        type: string
+      default: [settlement, float, reserve]
+
+steps:
+  read:
+    block: transform.jq
+    config:
+      input:
+        accounts: "${params.accounts}"
+      program: |
+        {balances: [.accounts[] | {account: ., opened: 1000, closed: 1000}]}
+
+  reconcile:
+    block: transform.jq
+    depends_on: [read]
+    # `transform.jq` answers with its result under `value`, so a later step reaches for that
+    # key rather than for the output as a whole.
+    config:
+      input: "${steps.read.output.value}"
+      program: |
+        {
+          checked: (.balances | length),
+          drift: [.balances[] | select(.opened != .closed) | .account],
+        }
+
+  ledger:
+    block: transform.jq
+    depends_on: [reconcile]
+    config:
+      input: "${steps.reconcile.output.value}"
+      program: |
+        {settled: (.drift | length == 0), accounts: .checked}

{dirigent_examples-0.17.3 → dirigent_examples-0.18.1}/src/dirigent_examples/shelves/patterns/references-cheat-sheet.yaml

@@ -1,6 +1,6 @@
 # Every ${...} form the reference language allows, each used once, in one document.
 #
-# THERE ARE FOUR NAMESPACES AND NOTHING ELSE. No expressions, no arithmetic, no conditionals,
+# THERE ARE FIVE NAMESPACES AND NOTHING ELSE. No expressions, no arithmetic, no conditionals,
 # no functions, no defaults, no coalescing. A reference names a value or it does not resolve.
 #
 #   params.<name>                a run parameter
@@ -17,6 +17,7 @@
 #   run.scratch                  this run's own directory under the artifact root
 #   run.window.start             the start of the interval this run covers
 #   run.window.end               the end of it, exclusive
+#   artifacts                    the instance's storage root, which no run owns
 #
 # TWO RULES ABOUT WHAT A REFERENCE BECOMES:
 #
@@ -53,8 +54,8 @@ kind: pipeline
 code: references-cheat-sheet
 name: The whole reference language
 description: |
-  Four namespaces -- `params`, `steps`, `item`, `run` -- and no expressions, conditionals or
-  functions anywhere.
+  Five namespaces -- `params`, `steps`, `item`, `run`, `artifacts` -- and no expressions,
+  conditionals or functions anywhere.
 
   A whole reference resolves to the typed value; one inside a larger string interpolates as
   text; an unknown one is rejected rather than resolved to empty. Needs `--window`.
@@ -108,6 +109,9 @@ steps:
         # run: the two values every run has.
         run_id: "${run.id}"
         scratch: "${run.scratch}"
+        # artifacts: the storage root itself, which outlives every run written under it.
+        # ${run.scratch} is swept when the run is pruned; ${artifacts}/... is not.
+        artifacts: "${artifacts}"
         # run.window: only on a run that carries one. Half-open, so start is included and
         # end is not.
         window_start: "${run.window.start}"

{dirigent_examples-0.17.3 → dirigent_examples-0.18.1}/src/dirigent_examples/shelves/recipes/README.md

@@ -82,6 +82,7 @@ and a parameter away in the others:
 | File | The question it answers |
 | --- | --- |
 | [storage-write-then-read.yaml](storage-write-then-read.yaml) | How do I park a payload in storage and pick it up again a step later? |
+| [storage-keep-past-the-run.yaml](storage-keep-past-the-run.yaml) | How do I keep something past the run that wrote it, without naming a directory or a bucket? |
 | [storage-copy-dated-archive.yaml](storage-copy-dated-archive.yaml) | How do I keep a dated copy, and what should the key layout be? |
 | [storage-exists-gate.yaml](storage-exists-gate.yaml) | How do I wait for an object, and what stops me reading a half-written one? |
 | [large-output-to-storage.yaml](large-output-to-storage.yaml) | What happens to a payload past the inline threshold, and what should I do about it? |

dirigent_examples-0.18.1/src/dirigent_examples/shelves/recipes/storage-keep-past-the-run.yaml

@@ -0,0 +1,116 @@
+# Fetch something and keep it, at a location the run does not own.
+#
+# In:  a query the echo service reflects back.
+# Out: the same body written twice -- once to the run's scratch, once to a kept location --
+#      and a summary naming both URIs side by side.
+#
+# TWO ANCHORS, TWO LIFETIMES. A document never names a directory or a bucket, only one of the
+# two prefixes the engine resolves for it:
+#
+#   ${run.scratch}   this run's own prefix, <root>/runs/<run id>/... Retention sweeps it when
+#                    the run is pruned, so it is where a working copy goes: a payload handed
+#                    to the next step, a captured stream, anything nobody needs next month.
+#   ${artifacts}     the instance's storage root itself. Retention deletes run prefixes and
+#                    nothing else under it, so an object written here stays until somebody
+#                    deletes it. This is where "fetch it and keep it" lands.
+#
+# Both resolve against whichever backend the instance is configured with -- a directory on a
+# single node, a bucket where an s3 connection serves the scheme -- so the two steps below
+# are the same document on either, and neither carries a path a second instance would have to
+# rewrite. A URI naming the root directly cannot do that: file://kept/x.json reads `kept` as
+# a host, file:///kept/x.json is an absolute path, and the backend refuses both for being
+# outside its root.
+#
+# THE KEY IS STABLE, NOT DATED. ${params.name} is one name and a rerun overwrites it, which
+# is what makes this the newest copy rather than a pile nobody prunes. Where a history is
+# wanted instead, the key goes under a date -- storage-copy-dated-archive.yaml is that
+# layout, and the same choice of prefix applies to it.
+#
+# WHAT --local DOES WITH IT. A plain --local run builds a throwaway instance and deletes it at
+# the end, so the kept object outlives the run but not the instance. Give the run a root of
+# its own and it is still there afterwards, and a second run finds the first run's copy:
+#
+#   dg run --local --root ./.dirigent-local examples/recipes/storage-keep-past-the-run.yaml
+#   ls ./.dirigent-local/artifacts/kept
+#
+# To change it: point DIRIGENT_ARTIFACT_ROOT at s3://<bucket>/<prefix> and the kept object is
+# an object in that bucket, with nothing in this document changed.
+#
+#   dg run --local examples/recipes/storage-keep-past-the-run.yaml
+#   dg run --local examples/recipes/storage-keep-past-the-run.yaml -p name=stations
+
+format: dirigent/v1
+kind: pipeline
+code: storage-keep-past-the-run
+name: Something kept past its run
+description: Fetch a body, write a working copy to the run's scratch and a kept copy under the artifact root, and read the kept one back.
+
+tags: [recipes, http, storage, transform, starter]
+
+requires:
+  blocks:
+    - http.request
+    - storage.write
+    - storage.read
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    name:
+      type: string
+      default: latest
+      description: The name the kept object carries; stable, so a rerun replaces it.
+    station:
+      type: string
+      default: st-1
+      description: A query value, echoed back inside the fetched body.
+
+steps:
+  fetch:
+    block: http.request
+    config:
+      url: https://postman-echo.com/get
+      method: GET
+      query:
+        station: ${params.station}
+      max_response: 1mb
+      timeout: 30s
+
+  working:
+    block: storage.write
+    depends_on: [fetch]
+    config:
+      # The run's own prefix: gone when retention prunes the run.
+      target: ${run.scratch}/fetched.json
+      value: ${steps.fetch.output.body}
+
+  keep:
+    block: storage.write
+    depends_on: [fetch]
+    config:
+      # The root itself: nothing sweeps this, so the object is here until somebody removes it.
+      target: ${artifacts}/kept/${params.name}.json
+      value: ${steps.fetch.output.body}
+
+  back:
+    block: storage.read
+    depends_on: [keep]
+    config:
+      # Where the write said the object landed, not the same path typed twice.
+      source: ${steps.keep.output.uri}
+      max_size: 1mb
+
+  where:
+    block: transform.jq
+    depends_on: [working, back]
+    config:
+      input:
+        swept: ${steps.working.output.uri}
+        kept: ${steps.keep.output.uri}
+        station: ${steps.back.output.value.args.station}
+      program: |
+        {kept, swept, station,
+         # The run id is in one of the two URIs and not in the other, which is the whole
+         # difference between them.
+         kept_is_run_scoped: (.kept | contains("/runs/"))}