dirigent-examples 0.17.3__tar.gz → 0.18.0__tar.gz

{dirigent_examples-0.17.3 → dirigent_examples-0.18.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: dirigent-examples
-Version: 0.17.3
+Version: 0.18.0
 Summary: The dirigent example corpus: the shelves of runnable documents, installed.
 License-Expression: LicenseRef-Proprietary
 License-File: LICENSE
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: dirigent-client==0.17.3
-Requires-Dist: dirigent-plugin==0.17.3
+Requires-Dist: dirigent-client==0.18.0
+Requires-Dist: dirigent-plugin==0.18.0
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 

{dirigent_examples-0.17.3 → dirigent_examples-0.18.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-examples"
-version = "0.17.3"
+version = "0.18.0"
 description = "The dirigent example corpus: the shelves of runnable documents, installed."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -11,8 +11,8 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-    "dirigent-client==0.17.3",
-    "dirigent-plugin==0.17.3",
+    "dirigent-client==0.18.0",
+    "dirigent-plugin==0.18.0",
 ]
 
 [project.entry-points."dirigent.plugins.v1"]

{dirigent_examples-0.17.3 → dirigent_examples-0.18.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-examples"
-version = "0.17.3"
+version = "0.18.0"
 description = "The dirigent example corpus: the shelves of runnable documents, installed."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -12,8 +12,8 @@ classifiers = [
 ]
 dependencies = [
     # The python/ shelf is SDK scripts a reader runs, so the corpus carries the client.
-    "dirigent-client==0.17.3",
-    "dirigent-plugin==0.17.3",
+    "dirigent-client==0.18.0",
+    "dirigent-plugin==0.18.0",
 ]
 
 [project.entry-points."dirigent.plugins.v1"]

{dirigent_examples-0.17.3 → dirigent_examples-0.18.0}/src/dirigent_examples/shelves/docker/docker-remote-daemon.yaml

@@ -26,8 +26,9 @@
 # only as a 0600 file under the run's work directory that goes when the step leaves. A client
 # key for a docker daemon is a credential for root on whatever that daemon runs on.
 #
-# TO MAKE IT YOURS: point host at your own daemon. A daemon that needs no TLS -- an ssh://
-# host, or a unix socket on another path -- takes host alone and none of the three PEMs.
+# TO MAKE IT YOURS: point host at your own daemon. The worker speaks tcp:// and unix://, and
+# a daemon that needs no TLS -- a unix socket on another path -- takes host alone and none of
+# the three PEMs.
 
 format: dirigent/v1
 kind: pipeline

{dirigent_examples-0.17.3 → dirigent_examples-0.18.0}/src/dirigent_examples/shelves/patterns/references-cheat-sheet.yaml

@@ -1,6 +1,6 @@
 # Every ${...} form the reference language allows, each used once, in one document.
 #
-# THERE ARE FOUR NAMESPACES AND NOTHING ELSE. No expressions, no arithmetic, no conditionals,
+# THERE ARE FIVE NAMESPACES AND NOTHING ELSE. No expressions, no arithmetic, no conditionals,
 # no functions, no defaults, no coalescing. A reference names a value or it does not resolve.
 #
 #   params.<name>                a run parameter
@@ -17,6 +17,7 @@
 #   run.scratch                  this run's own directory under the artifact root
 #   run.window.start             the start of the interval this run covers
 #   run.window.end               the end of it, exclusive
+#   artifacts                    the instance's storage root, which no run owns
 #
 # TWO RULES ABOUT WHAT A REFERENCE BECOMES:
 #
@@ -53,8 +54,8 @@ kind: pipeline
 code: references-cheat-sheet
 name: The whole reference language
 description: |
-  Four namespaces -- `params`, `steps`, `item`, `run` -- and no expressions, conditionals or
-  functions anywhere.
+  Five namespaces -- `params`, `steps`, `item`, `run`, `artifacts` -- and no expressions,
+  conditionals or functions anywhere.
 
   A whole reference resolves to the typed value; one inside a larger string interpolates as
   text; an unknown one is rejected rather than resolved to empty. Needs `--window`.
@@ -108,6 +109,9 @@ steps:
         # run: the two values every run has.
         run_id: "${run.id}"
         scratch: "${run.scratch}"
+        # artifacts: the storage root itself, which outlives every run written under it.
+        # ${run.scratch} is swept when the run is pruned; ${artifacts}/... is not.
+        artifacts: "${artifacts}"
         # run.window: only on a run that carries one. Half-open, so start is included and
         # end is not.
         window_start: "${run.window.start}"

{dirigent_examples-0.17.3 → dirigent_examples-0.18.0}/src/dirigent_examples/shelves/recipes/README.md

@@ -82,6 +82,7 @@ and a parameter away in the others:
 | File | The question it answers |
 | --- | --- |
 | [storage-write-then-read.yaml](storage-write-then-read.yaml) | How do I park a payload in storage and pick it up again a step later? |
+| [storage-keep-past-the-run.yaml](storage-keep-past-the-run.yaml) | How do I keep something past the run that wrote it, without naming a directory or a bucket? |
 | [storage-copy-dated-archive.yaml](storage-copy-dated-archive.yaml) | How do I keep a dated copy, and what should the key layout be? |
 | [storage-exists-gate.yaml](storage-exists-gate.yaml) | How do I wait for an object, and what stops me reading a half-written one? |
 | [large-output-to-storage.yaml](large-output-to-storage.yaml) | What happens to a payload past the inline threshold, and what should I do about it? |

dirigent_examples-0.18.0/src/dirigent_examples/shelves/recipes/storage-keep-past-the-run.yaml

@@ -0,0 +1,116 @@
+# Fetch something and keep it, at a location the run does not own.
+#
+# In:  a query the echo service reflects back.
+# Out: the same body written twice -- once to the run's scratch, once to a kept location --
+#      and a summary naming both URIs side by side.
+#
+# TWO ANCHORS, TWO LIFETIMES. A document never names a directory or a bucket, only one of the
+# two prefixes the engine resolves for it:
+#
+#   ${run.scratch}   this run's own prefix, <root>/runs/<run id>/... Retention sweeps it when
+#                    the run is pruned, so it is where a working copy goes: a payload handed
+#                    to the next step, a captured stream, anything nobody needs next month.
+#   ${artifacts}     the instance's storage root itself. Retention deletes run prefixes and
+#                    nothing else under it, so an object written here stays until somebody
+#                    deletes it. This is where "fetch it and keep it" lands.
+#
+# Both resolve against whichever backend the instance is configured with -- a directory on a
+# single node, a bucket where an s3 connection serves the scheme -- so the two steps below
+# are the same document on either, and neither carries a path a second instance would have to
+# rewrite. A URI naming the root directly cannot do that: file://kept/x.json reads `kept` as
+# a host, file:///kept/x.json is an absolute path, and the backend refuses both for being
+# outside its root.
+#
+# THE KEY IS STABLE, NOT DATED. ${params.name} is one name and a rerun overwrites it, which
+# is what makes this the newest copy rather than a pile nobody prunes. Where a history is
+# wanted instead, the key goes under a date -- storage-copy-dated-archive.yaml is that
+# layout, and the same choice of prefix applies to it.
+#
+# WHAT --local DOES WITH IT. A plain --local run builds a throwaway instance and deletes it at
+# the end, so the kept object outlives the run but not the instance. Give the run a root of
+# its own and it is still there afterwards, and a second run finds the first run's copy:
+#
+#   dg run --local --root ./.dirigent-local examples/recipes/storage-keep-past-the-run.yaml
+#   ls ./.dirigent-local/artifacts/kept
+#
+# To change it: point DIRIGENT_ARTIFACT_ROOT at s3://<bucket>/<prefix> and the kept object is
+# an object in that bucket, with nothing in this document changed.
+#
+#   dg run --local examples/recipes/storage-keep-past-the-run.yaml
+#   dg run --local examples/recipes/storage-keep-past-the-run.yaml -p name=stations
+
+format: dirigent/v1
+kind: pipeline
+code: storage-keep-past-the-run
+name: Something kept past its run
+description: Fetch a body, write a working copy to the run's scratch and a kept copy under the artifact root, and read the kept one back.
+
+tags: [recipes, http, storage, transform, starter]
+
+requires:
+  blocks:
+    - http.request
+    - storage.write
+    - storage.read
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    name:
+      type: string
+      default: latest
+      description: The name the kept object carries; stable, so a rerun replaces it.
+    station:
+      type: string
+      default: st-1
+      description: A query value, echoed back inside the fetched body.
+
+steps:
+  fetch:
+    block: http.request
+    config:
+      url: https://postman-echo.com/get
+      method: GET
+      query:
+        station: ${params.station}
+      max_response: 1mb
+      timeout: 30s
+
+  working:
+    block: storage.write
+    depends_on: [fetch]
+    config:
+      # The run's own prefix: gone when retention prunes the run.
+      target: ${run.scratch}/fetched.json
+      value: ${steps.fetch.output.body}
+
+  keep:
+    block: storage.write
+    depends_on: [fetch]
+    config:
+      # The root itself: nothing sweeps this, so the object is here until somebody removes it.
+      target: ${artifacts}/kept/${params.name}.json
+      value: ${steps.fetch.output.body}
+
+  back:
+    block: storage.read
+    depends_on: [keep]
+    config:
+      # Where the write said the object landed, not the same path typed twice.
+      source: ${steps.keep.output.uri}
+      max_size: 1mb
+
+  where:
+    block: transform.jq
+    depends_on: [working, back]
+    config:
+      input:
+        swept: ${steps.working.output.uri}
+        kept: ${steps.keep.output.uri}
+        station: ${steps.back.output.value.args.station}
+      program: |
+        {kept, swept, station,
+         # The run id is in one of the two URIs and not in the other, which is the whole
+         # difference between them.
+         kept_is_run_scoped: (.kept | contains("/runs/"))}