dirigent-examples 0.15.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. dirigent_examples/__init__.py +22 -0
  2. dirigent_examples/py.typed +0 -0
  3. dirigent_examples/shelves/README.md +299 -0
  4. dirigent_examples/shelves/composition/README.md +18 -0
  5. dirigent_examples/shelves/composition/chained-instances.yaml +97 -0
  6. dirigent_examples/shelves/composition/composition-child.yaml +64 -0
  7. dirigent_examples/shelves/composition/composition-parent.yaml +89 -0
  8. dirigent_examples/shelves/connections.yaml +52 -0
  9. dirigent_examples/shelves/demo/README.md +19 -0
  10. dirigent_examples/shelves/demo/markdown-showcase.yaml +117 -0
  11. dirigent_examples/shelves/demo/params-showcase.yaml +92 -0
  12. dirigent_examples/shelves/demo/requires.yaml +65 -0
  13. dirigent_examples/shelves/demo/weekly-import-malawi.yaml +54 -0
  14. dirigent_examples/shelves/demo/weekly-import-nepal.yaml +75 -0
  15. dirigent_examples/shelves/docker/README.md +29 -0
  16. dirigent_examples/shelves/docker/docker-build-push.yaml +110 -0
  17. dirigent_examples/shelves/docker/docker-build-run.yaml +106 -0
  18. dirigent_examples/shelves/docker/docker-compose-database.yaml +124 -0
  19. dirigent_examples/shelves/docker/docker-compose-failing-up.yaml +83 -0
  20. dirigent_examples/shelves/docker/docker-compose-file.yaml +117 -0
  21. dirigent_examples/shelves/docker/docker-compose-profiles-env.yaml +133 -0
  22. dirigent_examples/shelves/docker/docker-compose-stack.yaml +70 -0
  23. dirigent_examples/shelves/docker/docker-hello.yaml +53 -0
  24. dirigent_examples/shelves/docker/docker-remote-daemon.yaml +92 -0
  25. dirigent_examples/shelves/docker/docker-run-failing-teardown.yaml +94 -0
  26. dirigent_examples/shelves/docker/docker-ticker.yaml +49 -0
  27. dirigent_examples/shelves/execute/README.md +16 -0
  28. dirigent_examples/shelves/execute/long-log.yaml +89 -0
  29. dirigent_examples/shelves/failure/README.md +20 -0
  30. dirigent_examples/shelves/failure/error-handler.yaml +89 -0
  31. dirigent_examples/shelves/failure/optional-step.yaml +82 -0
  32. dirigent_examples/shelves/failure/retries.yaml +75 -0
  33. dirigent_examples/shelves/failure/retry-budget.yaml +82 -0
  34. dirigent_examples/shelves/failure/step-timeout.yaml +96 -0
  35. dirigent_examples/shelves/git/README.md +32 -0
  36. dirigent_examples/shelves/git/git-checkout-build.yaml +125 -0
  37. dirigent_examples/shelves/git/git-checkout-compose.yaml +138 -0
  38. dirigent_examples/shelves/git/git-checkout-public.yaml +84 -0
  39. dirigent_examples/shelves/graph/README.md +22 -0
  40. dirigent_examples/shelves/graph/deep-chain.yaml +119 -0
  41. dirigent_examples/shelves/graph/fan-in.yaml +80 -0
  42. dirigent_examples/shelves/graph/fan-out.yaml +66 -0
  43. dirigent_examples/shelves/graph/linear.yaml +66 -0
  44. dirigent_examples/shelves/graph/parallel-branches.yaml +57 -0
  45. dirigent_examples/shelves/graph/parallel-sleep.yaml +55 -0
  46. dirigent_examples/shelves/graph/skip-diamond.yaml +105 -0
  47. dirigent_examples/shelves/graph/wide-fan.yaml +147 -0
  48. dirigent_examples/shelves/hello-world.yaml +30 -0
  49. dirigent_examples/shelves/open-data/README.md +67 -0
  50. dirigent_examples/shelves/open-data/feeds-composition.yaml +120 -0
  51. dirigent_examples/shelves/open-data/gdacs-disaster-updates.yaml +230 -0
  52. dirigent_examples/shelves/open-data/github-releases-relay.yaml +225 -0
  53. dirigent_examples/shelves/open-data/hdx-dataset-watch.yaml +211 -0
  54. dirigent_examples/shelves/open-data/kobo-submissions-to-csv.yaml +149 -0
  55. dirigent_examples/shelves/open-data/nominatim-geocode-facilities.yaml +169 -0
  56. dirigent_examples/shelves/open-data/odk-central-submissions.yaml +146 -0
  57. dirigent_examples/shelves/open-data/open-meteo-weekly-report.yaml +142 -0
  58. dirigent_examples/shelves/open-data/overpass-health-facilities.yaml +154 -0
  59. dirigent_examples/shelves/open-data/usgs-earthquakes-alert.yaml +208 -0
  60. dirigent_examples/shelves/open-data/who-gho-indicators-to-parquet.yaml +163 -0
  61. dirigent_examples/shelves/open-data/wikidata-country-reference.yaml +146 -0
  62. dirigent_examples/shelves/open-data/world-bank-population-trend.yaml +166 -0
  63. dirigent_examples/shelves/patterns/README.md +144 -0
  64. dirigent_examples/shelves/patterns/concurrency-queue.yaml +89 -0
  65. dirigent_examples/shelves/patterns/concurrency-replace.yaml +92 -0
  66. dirigent_examples/shelves/patterns/concurrency-skip.yaml +97 -0
  67. dirigent_examples/shelves/patterns/connections-referenced-vs-carried.yaml +140 -0
  68. dirigent_examples/shelves/patterns/deadline-on-a-sensor.yaml +106 -0
  69. dirigent_examples/shelves/patterns/fan-out-continue.yaml +88 -0
  70. dirigent_examples/shelves/patterns/fan-out-fail-fast.yaml +80 -0
  71. dirigent_examples/shelves/patterns/fan-out-from-params.yaml +84 -0
  72. dirigent_examples/shelves/patterns/fan-out-item-wise.yaml +111 -0
  73. dirigent_examples/shelves/patterns/fan-out-literal-list.yaml +81 -0
  74. dirigent_examples/shelves/patterns/fan-out-nested-objects.yaml +107 -0
  75. dirigent_examples/shelves/patterns/fan-out-then-join.yaml +86 -0
  76. dirigent_examples/shelves/patterns/log-levels.yaml +119 -0
  77. dirigent_examples/shelves/patterns/outputs-inline-vs-storage.yaml +140 -0
  78. dirigent_examples/shelves/patterns/params-every-type.yaml +259 -0
  79. dirigent_examples/shelves/patterns/params-validation-refuses.yaml +131 -0
  80. dirigent_examples/shelves/patterns/pipeline-run-child.yaml +96 -0
  81. dirigent_examples/shelves/patterns/pipeline-run-fire-and-forget.yaml +99 -0
  82. dirigent_examples/shelves/patterns/pipeline-run-strict.yaml +103 -0
  83. dirigent_examples/shelves/patterns/pipeline-run-wait.yaml +107 -0
  84. dirigent_examples/shelves/patterns/pipeline-run-with-params.yaml +124 -0
  85. dirigent_examples/shelves/patterns/poll-cadence.yaml +102 -0
  86. dirigent_examples/shelves/patterns/priority-layered.yaml +120 -0
  87. dirigent_examples/shelves/patterns/references-cheat-sheet.yaml +186 -0
  88. dirigent_examples/shelves/patterns/retry-budget-exhausted.yaml +92 -0
  89. dirigent_examples/shelves/patterns/retry-exponential-backoff.yaml +88 -0
  90. dirigent_examples/shelves/patterns/retry-only-transient.yaml +108 -0
  91. dirigent_examples/shelves/patterns/retry-with-jitter.yaml +102 -0
  92. dirigent_examples/shelves/patterns/rule-all-done.yaml +83 -0
  93. dirigent_examples/shelves/patterns/rule-all-success.yaml +79 -0
  94. dirigent_examples/shelves/patterns/rule-always.yaml +92 -0
  95. dirigent_examples/shelves/patterns/rule-one-failed.yaml +86 -0
  96. dirigent_examples/shelves/patterns/schedule-at-once.yaml +110 -0
  97. dirigent_examples/shelves/patterns/schedule-cron-timezone.yaml +121 -0
  98. dirigent_examples/shelves/patterns/schedule-interval.yaml +109 -0
  99. dirigent_examples/shelves/patterns/schedule-window-half-open.yaml +105 -0
  100. dirigent_examples/shelves/patterns/sensor-http-ready.yaml +121 -0
  101. dirigent_examples/shelves/patterns/sensor-storage-exists.yaml +134 -0
  102. dirigent_examples/shelves/patterns/step-names-and-keys.yaml +99 -0
  103. dirigent_examples/shelves/patterns/timeout-fails-the-step.yaml +94 -0
  104. dirigent_examples/shelves/patterns/timeout-skips-the-step.yaml +102 -0
  105. dirigent_examples/shelves/patterns/webhook-mapping-nested-payload.yaml +125 -0
  106. dirigent_examples/shelves/patterns/webhook-signed.yaml +144 -0
  107. dirigent_examples/shelves/preview/s3-parquet-to-ingestion.yaml +92 -0
  108. dirigent_examples/shelves/python/README.md +31 -0
  109. dirigent_examples/shelves/python/apply_and_run.py +52 -0
  110. dirigent_examples/shelves/python/ci_gate.py +76 -0
  111. dirigent_examples/shelves/python/connections.py +61 -0
  112. dirigent_examples/shelves/python/error_handling.py +84 -0
  113. dirigent_examples/shelves/python/follow_logs.py +39 -0
  114. dirigent_examples/shelves/python/list_and_filter.py +52 -0
  115. dirigent_examples/shelves/queues/README.md +59 -0
  116. dirigent_examples/shelves/queues/kafka-consume-then-transform.yaml +105 -0
  117. dirigent_examples/shelves/queues/kafka-produce-then-consume.yaml +124 -0
  118. dirigent_examples/shelves/queues/rabbitmq-consume-ack-on-success.yaml +102 -0
  119. dirigent_examples/shelves/queues/report-to-kafka.yaml +105 -0
  120. dirigent_examples/shelves/queues/report-to-rabbitmq.yaml +105 -0
  121. dirigent_examples/shelves/recipes/README.md +130 -0
  122. dirigent_examples/shelves/recipes/csv-header-rules.yaml +147 -0
  123. dirigent_examples/shelves/recipes/csv-to-ndjson.yaml +107 -0
  124. dirigent_examples/shelves/recipes/etl-csv-clean-validate-parquet.yaml +207 -0
  125. dirigent_examples/shelves/recipes/filter-by-predicate.yaml +116 -0
  126. dirigent_examples/shelves/recipes/filter-then-map-then-reduce.yaml +109 -0
  127. dirigent_examples/shelves/recipes/http-fetch-validate-post.yaml +142 -0
  128. dirigent_examples/shelves/recipes/http-follow-redirects.yaml +100 -0
  129. dirigent_examples/shelves/recipes/http-get-with-query.yaml +96 -0
  130. dirigent_examples/shelves/recipes/http-headers-and-auth-connection.yaml +110 -0
  131. dirigent_examples/shelves/recipes/http-post-file-from-storage.yaml +117 -0
  132. dirigent_examples/shelves/recipes/http-post-json-echo.yaml +105 -0
  133. dirigent_examples/shelves/recipes/http-post-report.yaml +183 -0
  134. dirigent_examples/shelves/recipes/http-save-body-to-storage.yaml +106 -0
  135. dirigent_examples/shelves/recipes/http-success-status-list.yaml +80 -0
  136. dirigent_examples/shelves/recipes/http-timeout-override.yaml +104 -0
  137. dirigent_examples/shelves/recipes/jq-dedupe-by-key.yaml +78 -0
  138. dirigent_examples/shelves/recipes/jq-defaults-and-nulls.yaml +91 -0
  139. dirigent_examples/shelves/recipes/jq-group-by-and-sum.yaml +74 -0
  140. dirigent_examples/shelves/recipes/jq-join-two-lists.yaml +77 -0
  141. dirigent_examples/shelves/recipes/jq-long-to-wide.yaml +76 -0
  142. dirigent_examples/shelves/recipes/jq-nested-to-flat.yaml +89 -0
  143. dirigent_examples/shelves/recipes/jq-pivot-wide-to-long.yaml +65 -0
  144. dirigent_examples/shelves/recipes/jq-running-totals.yaml +82 -0
  145. dirigent_examples/shelves/recipes/jq-string-cleaning.yaml +88 -0
  146. dirigent_examples/shelves/recipes/jq-top-n.yaml +85 -0
  147. dirigent_examples/shelves/recipes/jq-validate-in-jq-vs-schema.yaml +124 -0
  148. dirigent_examples/shelves/recipes/jq-window-dates.yaml +82 -0
  149. dirigent_examples/shelves/recipes/json-to-csv-flattening.yaml +141 -0
  150. dirigent_examples/shelves/recipes/large-output-to-storage.yaml +134 -0
  151. dirigent_examples/shelves/recipes/map-enrich-with-lookup.yaml +96 -0
  152. dirigent_examples/shelves/recipes/ndjson-to-parquet.yaml +130 -0
  153. dirigent_examples/shelves/recipes/pagination-by-fan-out.yaml +115 -0
  154. dirigent_examples/shelves/recipes/parquet-round-trip-types.yaml +163 -0
  155. dirigent_examples/shelves/recipes/reconcile-two-sources.yaml +159 -0
  156. dirigent_examples/shelves/recipes/report-built-in.yaml +72 -0
  157. dirigent_examples/shelves/recipes/report-daily-digest.yaml +186 -0
  158. dirigent_examples/shelves/recipes/report-to-file.yaml +131 -0
  159. dirigent_examples/shelves/recipes/report-to-webhook.yaml +136 -0
  160. dirigent_examples/shelves/recipes/schema-carried.yaml +112 -0
  161. dirigent_examples/shelves/recipes/schema-formats.yaml +107 -0
  162. dirigent_examples/shelves/recipes/schema-referenced.yaml +86 -0
  163. dirigent_examples/shelves/recipes/schema-refuses-then-rule.yaml +127 -0
  164. dirigent_examples/shelves/recipes/storage-copy-dated-archive.yaml +114 -0
  165. dirigent_examples/shelves/recipes/storage-exists-gate.yaml +127 -0
  166. dirigent_examples/shelves/recipes/storage-manifest-of-a-fan-out.yaml +104 -0
  167. dirigent_examples/shelves/recipes/storage-write-then-read.yaml +119 -0
  168. dirigent_examples/shelves/recipes/webhook-post-hmac.yaml +132 -0
  169. dirigent_examples/shelves/recipes/webhook-post-summary.yaml +142 -0
  170. dirigent_examples/shelves/s3/README.md +34 -0
  171. dirigent_examples/shelves/s3/report-to-s3.yaml +93 -0
  172. dirigent_examples/shelves/s3/s3-copy-and-verify.yaml +105 -0
  173. dirigent_examples/shelves/s3/s3-csv-report.yaml +87 -0
  174. dirigent_examples/shelves/s3/s3-parquet-report.yaml +77 -0
  175. dirigent_examples/shelves/s3/s3-round-trip.yaml +125 -0
  176. dirigent_examples/shelves/schemas/README.md +36 -0
  177. dirigent_examples/shelves/schemas/echo-reading.json +18 -0
  178. dirigent_examples/shelves/schemas/ou-record.json +13 -0
  179. dirigent_examples/shelves/schemas/station-reading.json +13 -0
  180. dirigent_examples/shelves/sensors/README.md +16 -0
  181. dirigent_examples/shelves/sensors/sensor-gate.yaml +65 -0
  182. dirigent_examples/shelves/sensors/time-window.yaml +61 -0
  183. dirigent_examples/shelves/sql/README.md +52 -0
  184. dirigent_examples/shelves/sql/duckdb-parquet-to-report.yaml +146 -0
  185. dirigent_examples/shelves/sql/sql-postgres-readonly.yaml +111 -0
  186. dirigent_examples/shelves/sql/sql-query-to-storage.yaml +85 -0
  187. dirigent_examples/shelves/sql/sql-sqlite-roundtrip.yaml +114 -0
  188. dirigent_examples/shelves/sql/warehouse.sql +42 -0
  189. dirigent_examples/shelves/transform/README.md +36 -0
  190. dirigent_examples/shelves/transform/csv-report.yaml +55 -0
  191. dirigent_examples/shelves/transform/jq-filter-and-map.yaml +70 -0
  192. dirigent_examples/shelves/transform/jq-group-and-aggregate.yaml +70 -0
  193. dirigent_examples/shelves/transform/jq-join-two-sources.yaml +98 -0
  194. dirigent_examples/shelves/transform/jq-reshape.yaml +91 -0
  195. dirigent_examples/shelves/transform/jq-stream-through-storage.yaml +112 -0
  196. dirigent_examples/shelves/transform/ndjson-round-trip.yaml +56 -0
  197. dirigent_examples/shelves/transform/parquet-round-trip.yaml +68 -0
  198. dirigent_examples/shelves/transform/std-convert-fan-out.yaml +142 -0
  199. dirigent_examples/shelves/transform/xml-feed-to-ndjson.yaml +116 -0
  200. dirigent_examples/shelves/transform/yaml-config-to-json.yaml +104 -0
  201. dirigent_examples/shelves/triggers/README.md +45 -0
  202. dirigent_examples/shelves/triggers/at-one-time.yaml +78 -0
  203. dirigent_examples/shelves/triggers/cron-nightly.yaml +79 -0
  204. dirigent_examples/shelves/triggers/cron-windowed.yaml +86 -0
  205. dirigent_examples/shelves/triggers/document-nightly.yaml +80 -0
  206. dirigent_examples/shelves/triggers/interval-rolling.yaml +88 -0
  207. dirigent_examples/shelves/triggers/managed-and-manual.yaml +109 -0
  208. dirigent_examples/shelves/triggers/webhook-trigger.yaml +75 -0
  209. dirigent_examples/shelves/validate/README.md +31 -0
  210. dirigent_examples/shelves/validate/expects-a-shape.yaml +56 -0
  211. dirigent_examples/shelves/validate/the-shape-is-wrong.yaml +46 -0
  212. dirigent_examples-0.15.0.dist-info/METADATA +21 -0
  213. dirigent_examples-0.15.0.dist-info/RECORD +216 -0
  214. dirigent_examples-0.15.0.dist-info/WHEEL +4 -0
  215. dirigent_examples-0.15.0.dist-info/entry_points.txt +3 -0
  216. dirigent_examples-0.15.0.dist-info/licenses/LICENSE +18 -0
@@ -0,0 +1,117 @@
1
+ # A description that exercises every markdown construct the UI claims to render.
2
+ #
3
+ # A pipeline's description is long-form and markdown-capable, and it is the body of the
4
+ # pipeline's screen. Most examples use one paragraph of it. This one uses all of it, so that
5
+ # a renderer change has something to fail against: headings, both kinds of list, nested
6
+ # items, a task list, inline code, a fenced block with a language, a fenced block without
7
+ # one, a table with alignment, a blockquote, a horizontal rule, emphasis, a link, and a long
8
+ # paragraph that has to wrap.
9
+ #
10
+ # It is a real pipeline rather than a stub, because a document that cannot run has no
11
+ # business in examples/ -- the two steps below reshape a small payload and need no network
12
+ # and no allowlist entry. The description is the exhibit; the steps are what make it legal.
13
+ #
14
+ # The rendering rule the corpus follows: a screen's title is the name if there is one and
15
+ # the code otherwise, the code is always on screen in mono and never drawn twice, and the
16
+ # description is the body. So this document carries all three, and the body is what to look
17
+ # at.
18
+ #
19
+ # dg run --local examples/demo/markdown-showcase.yaml
20
+ # dg pipelines show markdown-showcase # the description, rendered
21
+
22
+ format: dirigent/v1
23
+ kind: pipeline
24
+ code: markdown-showcase
25
+ name: Markdown showcase
26
+ description: |
27
+ A pipeline whose **description** is the point. Everything below is one markdown document,
28
+ rendered as the body of this pipeline's screen.
29
+
30
+ ## Headings, and what they are for
31
+
32
+ A description long enough to need sections should have them. This is an `h2`; the format
33
+ puts no ceiling on depth, and a renderer is expected to keep the hierarchy visible.
34
+
35
+ ### A third level
36
+
37
+ Which exists mainly so the scale between levels can be judged.
38
+
39
+ ## Lists
40
+
41
+ Unordered, with a nested level:
42
+
43
+ - Extract, from a source that may or may not be awake
44
+ - Reshape, which is where most of the work actually is
45
+ - `transform.jq` for one whole value
46
+ - `map.jq` for one element at a time
47
+ - `filter.jq` when the answer is a shorter list
48
+ - Load, and then say so
49
+
50
+ Ordered, because order sometimes matters:
51
+
52
+ 1. Apply the document
53
+ 2. Run it once by hand
54
+ 3. Only then give it a clock
55
+
56
+ ## Code
57
+
58
+ Inline code such as `dg run markdown-showcase --watch` sits inside a sentence. A fenced
59
+ block with a language gets highlighted:
60
+
61
+ ```bash
62
+ dg apply examples/demo/markdown-showcase.yaml
63
+ dg run markdown-showcase --watch
64
+ ```
65
+
66
+ And one without a language does not, which is the correct treatment for output:
67
+
68
+ ```
69
+ {"kind": "run", "status": "succeeded", "duration_ms": 412}
70
+ ```
71
+
72
+ ## A table
73
+
74
+ | Block | Reads | Writes | Needs the allowlist |
75
+ | --- | :---: | ---: | --- |
76
+ | `transform.jq` | one value | one value | no |
77
+ | `http.request` | a URL | an inline body or storage | no |
78
+ | `shell.run` | a command | stdout and stderr | **yes** |
79
+ | `storage.copy` | a URI | a URI | no |
80
+
81
+ ## A quote, and a rule
82
+
83
+ > A run is a row, not a process. It survives every restart, and it is queryable long
84
+ > after the machine that executed it is gone.
85
+
86
+ ---
87
+
88
+ ## Emphasis, links, and wrapping
89
+
90
+ Text can be *emphasised*, **strong**, or ***both at once***, and it can carry a link to
91
+ [the transforms page](../docs/transforms.md). This last paragraph is deliberately long so
92
+ that soft wrapping has something to do with it: a description is written as prose in a
93
+ file where the lines are hard-wrapped at a sensible width for reading in an editor, and
94
+ it must arrive on screen as one flowing paragraph rather than as a column of short lines,
95
+ because the editor's line breaks are an accident of the file and not a fact about the
96
+ text.
97
+
98
+ tags: [demo, transform]
99
+
100
+ requires:
101
+ blocks:
102
+ - transform.jq
103
+
104
+ steps:
105
+ collect:
106
+ block: transform.jq
107
+ config:
108
+ input:
109
+ constructs: [headings, lists, code, table, quote, rule, emphasis]
110
+ program: "{counted: (.constructs | length), constructs: .constructs}"
111
+
112
+ summarise:
113
+ block: transform.jq
114
+ depends_on: [collect]
115
+ config:
116
+ input: "${steps.collect.output.value}"
117
+ program: "{note: \"\\(.counted) markdown constructs, all of them in the description\"}"
@@ -0,0 +1,92 @@
1
+ # The parameter schema is plain JSON Schema, and it is load-bearing in three places.
2
+ #
3
+ # It validates every run's parameters, whoever started it: the CLI, the API, a schedule's
4
+ # overrides, or a webhook's mapped payload. It is what the UI renders the run dialog from,
5
+ # so an enum becomes a select and a date format becomes a date picker without any UI code
6
+ # knowing this pipeline exists. And apply checks every ${params.x} in the document against
7
+ # it, so a typo is caught when the document is applied rather than when it fires at 05:00.
8
+ #
9
+ # Defaults are filled in before validation, so a parameter with a default is optional.
10
+ #
11
+ # The CLI reads the schema before it sends anything, so a value is coerced to the type the
12
+ # schema declares. Three ways to pass parameters, all ending in the same validated object:
13
+ #
14
+ # dg run --local examples/demo/params-showcase.yaml -p day=2026-01-01 -p dataset=climate
15
+ #
16
+ # # dotted keys address a nested leaf, and the declared type decides the coercion:
17
+ # # batch_size is an integer, so -p batch_size=1000 sends 1000, not "1000"
18
+ # dg run --local examples/demo/params-showcase.yaml \
19
+ # -p day=2026-01-01 -p dataset=climate -p batch_size=1000 -p dry_run=true
20
+ #
21
+ # # an array or an object is given whole, inline, in JSON or YAML:
22
+ # dg run --local examples/demo/params-showcase.yaml \
23
+ # -p day=2026-01-01 -p dataset=climate -p regions='[east, west, north]'
24
+ #
25
+ # # or a whole payload from a file, with flags overriding it:
26
+ # dg run --local examples/demo/params-showcase.yaml -P params.yaml -p dataset=incidence
27
+ #
28
+ # Array indices are not addressable from a dotted key: -p regions.0=east and
29
+ # -p regions[0]=east are both refused, because a dotted path cannot tell an index from an
30
+ # object key named "0", and brackets are reserved in case element addressing is ever added.
31
+
32
+ format: dirigent/v1
33
+ kind: pipeline
34
+ code: params-showcase
35
+ name: Every parameter shape
36
+ description: Everything a run form is built from, in one parameter schema.
37
+
38
+ tags: [demo, http]
39
+
40
+ params:
41
+ type: object
42
+ required: [day, dataset]
43
+ properties:
44
+ day:
45
+ type: string
46
+ description: The logical date this run covers.
47
+ format: date
48
+
49
+ dataset:
50
+ type: string
51
+ description: Which dataset to process.
52
+ enum: [climate, population, incidence]
53
+
54
+ regions:
55
+ type: array
56
+ description: Regions to process; every one becomes its own run item.
57
+ default: [east, west]
58
+ items:
59
+ type: string
60
+
61
+ batch_size:
62
+ type: integer
63
+ description: How many records to send per request.
64
+ default: 500
65
+ minimum: 1
66
+ maximum: 10000
67
+
68
+ dry_run:
69
+ type: boolean
70
+ description: Validate the payload without writing anything downstream.
71
+ default: false
72
+
73
+ steps:
74
+ process:
75
+ block: http.request
76
+ # The fan-out is over a parameter, so how wide the run is is decided when it starts
77
+ # rather than when the document is written.
78
+ for_each: "${params.regions}"
79
+ items: continue
80
+ config:
81
+ url: https://postman-echo.com/post
82
+ method: POST
83
+ query:
84
+ day: "${params.day}"
85
+ dataset: "${params.dataset}"
86
+ region: "${item}"
87
+ batch_size: "${params.batch_size}"
88
+ dry_run: "${params.dry_run}"
89
+ body:
90
+ day: "${params.day}"
91
+ dataset: "${params.dataset}"
92
+ region: "${item}"
@@ -0,0 +1,65 @@
1
+ # The requires preflight, and referencing a connection by name.
2
+ #
3
+ # Documents are portable because everything is referenced by name -- block ids, connection
4
+ # names, other pipelines -- and never by id. That portability only works if applying a
5
+ # document to an instance that is missing something says so once, with the whole list,
6
+ # rather than one unknown-block error at a time.
7
+ #
8
+ # apply checks requires first: everything named here must exist on the target instance.
9
+ #
10
+ # The connection carries the credentials, and the document carries only its name. Create it
11
+ # once on a server:
12
+ # dg connection create http postman-echo --set base_url=https://postman-echo.com
13
+ #
14
+ # Or hand it to a local run, which has no server to hold it:
15
+ # dg run --local examples/demo/requires.yaml --connections examples/connections.yaml
16
+
17
+ format: dirigent/v1
18
+ kind: pipeline
19
+ code: requires
20
+ name: Declared requirements
21
+ description: Declare the blocks and connections this document cannot run without.
22
+
23
+ tags: [demo, http, storage]
24
+
25
+ requires:
26
+ blocks:
27
+ - http.request
28
+ - storage.write
29
+ - storage.copy
30
+ connections:
31
+ - postman-echo
32
+
33
+ steps:
34
+ authenticate:
35
+ block: http.request
36
+ config:
37
+ # The connection holds the base URL and the credential, so the document names a path
38
+ # and never a secret. What is stored is which connection, not what is in it.
39
+ connection: postman-echo
40
+ path: /basic-auth
41
+ method: GET
42
+
43
+ publish:
44
+ block: http.request
45
+ depends_on: [authenticate]
46
+ config:
47
+ connection: postman-echo
48
+ path: /post
49
+ method: POST
50
+ body:
51
+ authenticated: "${steps.authenticate.output.status}"
52
+
53
+ receipt:
54
+ block: storage.write
55
+ depends_on: [publish]
56
+ config:
57
+ target: "${run.scratch}/reports/receipt.json"
58
+ value: "${steps.publish.output.body}"
59
+
60
+ archive:
61
+ block: storage.copy
62
+ depends_on: [receipt]
63
+ config:
64
+ source: "${steps.receipt.output.uri}"
65
+ target: "${run.scratch}/archive/receipt.json"
@@ -0,0 +1,54 @@
1
+ # The other half of the shared-name pair; weekly-import-nepal.yaml carries the explanation.
2
+ #
3
+ # Both documents declare `name: Weekly import`. Applying both leaves an instance holding two
4
+ # pipelines with the same title and different codes, which is exactly the situation the
5
+ # naming rules allow and the screens are drawn to survive.
6
+ #
7
+ # Kept as a separate file because a dirigent document holds exactly one pipeline. That is
8
+ # also why composition is two files, and why a schedule example carries its own pipeline.
9
+ #
10
+ # dg apply examples/demo/weekly-import-malawi.yaml
11
+ # dg run weekly-import-malawi -p week=2026-W02
12
+
13
+ format: dirigent/v1
14
+ kind: pipeline
15
+ code: weekly-import-malawi
16
+ # The same title as weekly-import-nepal.yaml, on purpose.
17
+ name: Weekly import
18
+ description: |
19
+ The Malawi half of a deliberately ambiguous pair.
20
+
21
+ Same display name as `weekly-import-nepal`, different `code`. The listing shows one title
22
+ twice and two codes once each, which is how a person tells them apart and how every
23
+ reference already did.
24
+
25
+ tags: [demo, transform]
26
+
27
+ requires:
28
+ blocks:
29
+ - transform.jq
30
+
31
+ params:
32
+ type: object
33
+ properties:
34
+ week:
35
+ type: string
36
+ description: The ISO week being imported.
37
+ default: 2026-W01
38
+
39
+ steps:
40
+ read_manifest:
41
+ block: transform.jq
42
+ config:
43
+ input:
44
+ country: malawi
45
+ week: "${params.week}"
46
+ districts: [balaka, chikwawa, dedza, karonga, mzimba, salima, zomba]
47
+ program: "{country, week, districts: (.districts | length)}"
48
+
49
+ summarise:
50
+ block: transform.jq
51
+ depends_on: [read_manifest]
52
+ config:
53
+ input: "${steps.read_manifest.output.value}"
54
+ program: "{note: \"\\(.country) \\(.week): \\(.districts) districts imported\"}"
@@ -0,0 +1,75 @@
1
+ # Two pipelines, one display name, and only the code telling them apart.
2
+ #
3
+ # This document and weekly-import-malawi.yaml both carry `name: Weekly import`. That is legal,
4
+ # it is not a mistake, and an instance holding both is behaving correctly.
5
+ #
6
+ # The four fields every addressable thing carries each have exactly one job:
7
+ #
8
+ # id the uuid a machine holds
9
+ # code the addressable key: constrained, unique, and what a URL, a reference and
10
+ # `dg run` all carry
11
+ # name an optional human title, free-form, with NO identity semantics at all --
12
+ # nothing may ever be referenced by it, and two rows may share one
13
+ # description the long-form body
14
+ #
15
+ # The temptation this pair exists to head off is treating a name as a key: looking a pipeline
16
+ # up by it, keying a dictionary on it, or assuming a listing's titles are distinct. Names are
17
+ # for reading. `dg run weekly-import-nepal` names a pipeline; "Weekly import" names two, and
18
+ # an instance asked to run that has no way to choose.
19
+ #
20
+ # So the UI's rule pays for itself here. A screen shows the name as the title AND the code in
21
+ # mono beside it, always, never drawn twice -- which is what makes two identically titled
22
+ # rows in a listing tellable apart at a glance instead of a support question. Apply both and
23
+ # look at the pipelines list.
24
+ #
25
+ # Nothing here reaches the network or runs code on the worker.
26
+ #
27
+ # dg apply examples/demo/weekly-import-nepal.yaml
28
+ # dg apply examples/demo/weekly-import-malawi.yaml
29
+ # dg pipelines list # two rows, one title, two codes
30
+
31
+ format: dirigent/v1
32
+ kind: pipeline
33
+ code: weekly-import-nepal
34
+ # Shared, deliberately, with weekly-import-malawi.yaml. The code above is what differs, and
35
+ # the code is the only thing that was ever identity.
36
+ name: Weekly import
37
+ description: |
38
+ The Nepal half of a deliberately ambiguous pair.
39
+
40
+ This pipeline and `weekly-import-malawi` share the display name **Weekly import**. Only
41
+ the `code` is identity: `name` is free-form, carries no uniqueness, and nothing may be
42
+ referenced by it.
43
+
44
+ Two rows may share a title. Neither may share a code.
45
+
46
+ tags: [demo, transform]
47
+
48
+ requires:
49
+ blocks:
50
+ - transform.jq
51
+
52
+ params:
53
+ type: object
54
+ properties:
55
+ week:
56
+ type: string
57
+ description: The ISO week being imported.
58
+ default: 2026-W01
59
+
60
+ steps:
61
+ read_manifest:
62
+ block: transform.jq
63
+ config:
64
+ input:
65
+ country: nepal
66
+ week: "${params.week}"
67
+ districts: [bagmati, gandaki, karnali, koshi, lumbini, madhesh, sudurpashchim]
68
+ program: "{country, week, districts: (.districts | length)}"
69
+
70
+ summarise:
71
+ block: transform.jq
72
+ depends_on: [read_manifest]
73
+ config:
74
+ input: "${steps.read_manifest.output.value}"
75
+ program: "{note: \"\\(.country) \\(.week): \\(.districts) districts imported\"}"
@@ -0,0 +1,29 @@
1
+ # Docker examples
2
+
3
+ The `docker.*` family runs containers on the worker, which is why each block needs its id in
4
+ `DIRIGENT_ENABLED_UNSAFE_BLOCKS` before an instance will run it, and a daemon the worker can
5
+ reach. [docs/docker.md](../../docs/docker.md) is the family's home, and
6
+ [docs/security.md](../../docs/security.md) says what mounting the socket grants.
7
+
8
+ ```bash
9
+ dg run --local examples/docker/docker-hello.yaml --enable-unsafe docker.run
10
+ ```
11
+
12
+ The last two documents on the shelf **fail on purpose**: they are here for what a broken
13
+ stack and a failed drive step leave behind, which in both cases is nothing.
14
+
15
+ ## Pipelines
16
+
17
+ | File | What it teaches |
18
+ | --- | --- |
19
+ | [docker-hello.yaml](docker-hello.yaml) | One command in one container: the image pulled first, no network, the streams collected. |
20
+ | [docker-ticker.yaml](docker-ticker.yaml) | The live log: a container talking while it works, each probe appending what arrived since the last one. |
21
+ | [docker-compose-stack.yaml](docker-compose-stack.yaml) | A whole stack: `docker.compose.up` brings it up, a `docker.run` step joins its network, and `docker.compose.down` with `rule: all_done` tears it down on any outcome. |
22
+ | [docker-compose-database.yaml](docker-compose-database.yaml) | A database driven for real: `wait: true` turns a healthcheck into a gate, a `psql` container writes and reads rows, and the volumes go with the teardown. |
23
+ | [docker-compose-file.yaml](docker-compose-file.yaml) | The other compose form: a file an earlier step wrote into the run's work directory, named by its path relative to it. |
24
+ | [docker-compose-profiles-env.yaml](docker-compose-profiles-env.yaml) | Parameters reaching the stack: a variable compose interpolates, and a profile that decides whether the optional service exists. |
25
+ | [docker-build-run.yaml](docker-build-run.yaml) | An image built on the worker and run by tag, because a built image lives only in the store of the daemon that built it. |
26
+ | [docker-build-push.yaml](docker-build-push.yaml) | Pushing what was built: a `docker` connection holding a registry credential, and the tags and digests that come back. |
27
+ | [docker-remote-daemon.yaml](docker-remote-daemon.yaml) | Naming the daemon: a `docker` connection with a host and its client TLS, on every step of one stack's lifecycle. |
28
+ | [docker-compose-failing-up.yaml](docker-compose-failing-up.yaml) | A bring-up that fails, and the containers it does not leave behind: the block tears its own half-built project down. |
29
+ | [docker-run-failing-teardown.yaml](docker-run-failing-teardown.yaml) | A drive step that fails: the run is failed, the teardown ran anyway, and the daemon holds nothing. |
@@ -0,0 +1,110 @@
1
+ # Building an image and pushing it, which is the one thing a tag alone cannot do.
2
+ #
3
+ # docker-build-run.yaml builds an image and runs it on the same worker, where the tag is the
4
+ # whole contract and the image never leaves that daemon's store. This document is the other
5
+ # half: the image is pushed, so a later run on another worker, another cluster or another
6
+ # machine can pull it by name.
7
+ #
8
+ # IT CANNOT RUN WITHOUT A REGISTRY. There is no public registry that will take an anonymous
9
+ # push, so this document does nothing at all until it has a real registry and a credential
10
+ # for it. Create the connection once, with a token that may write packages:
11
+ # dg connection create docker ghcr \
12
+ # --set registry=ghcr.io \
13
+ # --set username=your-account \
14
+ # --set "password=$GITHUB_TOKEN"
15
+ #
16
+ # docker.build declares local_execution, so the engine refuses it unless the instance
17
+ # allowlists its id:
18
+ # export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.run","docker.build"]'
19
+ #
20
+ # PUSHING NEEDS A CONNECTION AND SAYS SO. push: true with no connection is refused when the
21
+ # document is applied, not when it first runs, and a connection carrying no username and
22
+ # password is refused when the step starts. Neither is a build that silently produced a local
23
+ # image nobody can reach.
24
+ #
25
+ # ONE DAEMON, THEREFORE ONE WORKER. The build and the step that wrote its context are on the
26
+ # worker's own filesystem, so requires.workers keeps them on one docker-capable worker. The push
27
+ # is what frees the image from that worker: once it is in the registry, anything can pull it.
28
+ #
29
+ # THE CREDENTIAL NEVER BECOMES AN ARGUMENT. The password is sealed on the connection, reaches
30
+ # `docker login` on stdin, and lives in a DOCKER_CONFIG directory of its own under the run's
31
+ # work directory -- never the worker's own docker config. The step logs out and the directory
32
+ # goes when it leaves, so nothing about the login survives the step, and the password is
33
+ # scrubbed out of every log line and every failure message.
34
+ #
35
+ # THE TAG IS A REGISTRY ADDRESS NOW. A pushed tag is fully qualified: registry, owner, name.
36
+ # The build's output says which tags reached the registry (pushed) and the digest the registry
37
+ # gave each of them (digests), and a digest is the only exact name for what was pushed -- a
38
+ # tag can be moved, a digest cannot.
39
+ #
40
+ # TO MAKE IT YOURS: point image_tag at your own registry and repository, replace the Dockerfile
41
+ # the first step writes with your own, and add a second tag if you publish both a version and
42
+ # a moving "latest".
43
+
44
+ format: dirigent/v1
45
+ kind: pipeline
46
+ code: docker-build-push
47
+ name: Build an image and push it
48
+ description: Build an image from a context in the work directory and push every tag to the registry a docker connection names.
49
+
50
+ tags: [docker, execute]
51
+
52
+ requires:
53
+ blocks:
54
+ - docker.run
55
+ - docker.build
56
+ workers:
57
+ # An image built on worker A is invisible to worker B, so both steps have to meet one
58
+ # docker-capable worker: give those the tag (dg worker --tag docker) and name it here.
59
+ - docker
60
+ connections:
61
+ # Named, not carried: the instance holds the registry credential, and this document
62
+ # refuses to apply where it is absent rather than failing the first time it runs.
63
+ - ghcr
64
+
65
+ params:
66
+ type: object
67
+ additionalProperties: false
68
+ properties:
69
+ image_tag:
70
+ type: string
71
+ default: ghcr.io/your-account/dirigent-greeter:1.0.0
72
+ description: The fully qualified tag the build gives the image and the push sends.
73
+
74
+ steps:
75
+ dockerfile:
76
+ block: docker.run
77
+ deadline: 5m
78
+ config:
79
+ image: alpine:3
80
+ pull: true
81
+ network: none
82
+ # A target with no URI scheme lands in the run's work directory on this worker, which
83
+ # is where the build below reads its context from; a URI would send the file to storage.
84
+ outputs:
85
+ Dockerfile: "context/Dockerfile"
86
+ command: |
87
+ cat > /dirigent/outputs/Dockerfile <<'DOCKERFILE'
88
+ FROM alpine:3
89
+ RUN echo "baked in at build time" > /greeting.txt
90
+ CMD ["cat", "/greeting.txt"]
91
+ DOCKERFILE
92
+ memory: 64mb
93
+ pids_limit: 64
94
+
95
+ build:
96
+ block: docker.build
97
+ depends_on: [dockerfile]
98
+ deadline: 15m
99
+ config:
100
+ context: context
101
+ dockerfile: Dockerfile
102
+ # A pushed tag carries its registry, so this name is where the image ends up as much as
103
+ # what it is called.
104
+ tags: ["${params.image_tag}"]
105
+ pull: true
106
+ # The connection holds the registry, the username and the sealed password. It could hold
107
+ # a daemon too: one connection can name both, and the steps that build and the steps that
108
+ # run would then all name it.
109
+ connection: ghcr
110
+ push: true
@@ -0,0 +1,106 @@
1
+ # Building an image on the worker, then running it, and why both steps must meet one daemon.
2
+ #
3
+ # REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
4
+ # tcp+TLS, or the local socket in dev). docs/docker.md is the family's home.
5
+ #
6
+ # docker.run and docker.build both declare local_execution, so the engine refuses them unless
7
+ # the instance allowlists their ids:
8
+ # dg run --local examples/docker/docker-build-run.yaml --enable-unsafe docker.run,docker.build
9
+ # export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.run","docker.build"]'
10
+ #
11
+ # Three hops, and what each one hands on:
12
+ # dockerfile a container writes a Dockerfile into context/ in the run's work directory.
13
+ # docker.build takes a context DIRECTORY on the worker's filesystem and has no
14
+ # inline form, so getting the context there is the pipeline's job.
15
+ # build buildx builds that directory and loads the result into the worker's own daemon
16
+ # store. The output carries the image id, its size in bytes, and its tags.
17
+ # greet docker.run runs the image by tag with no CMD of its own, so the image's own
18
+ # CMD is what runs. The output carries its stdout, exit code and container id.
19
+ #
20
+ # THE TAG IS THE CONTRACT. Nothing is pushed here, so the image exists only in the store of the
21
+ # daemon that built it, under the tags the build gave it. That name is all the run step has,
22
+ # which is why both steps read it from one parameter. docker-build-push.yaml is the other half:
23
+ # a docker connection with a registry credential, and an image that leaves the worker.
24
+ #
25
+ # ONE DAEMON, THEREFORE ONE WORKER. An image built on worker A is invisible to worker B, so
26
+ # give the docker-capable workers a `docker` tag (dg worker --tag docker) and declare
27
+ # requires.workers: [docker] below, which is what keeps both steps on one of them.
28
+ #
29
+ # TO MAKE IT YOURS: replace the Dockerfile the first step writes with your own, or drop that
30
+ # step and point the build at a context an upstream storage.copy put in the work directory
31
+ # instead, and give the image a tag of your own.
32
+
33
+ format: dirigent/v1
34
+ kind: pipeline
35
+ code: docker-build-run
36
+ name: Build an image, then run it
37
+ description: Write a Dockerfile into the run's work directory, build it with buildx, and run the tag.
38
+
39
+ tags: [docker, execute, starter]
40
+
41
+ requires:
42
+ blocks:
43
+ - docker.run
44
+ - docker.build
45
+ workers:
46
+ - docker
47
+
48
+ params:
49
+ type: object
50
+ additionalProperties: false
51
+ properties:
52
+ image_tag:
53
+ type: string
54
+ default: dirigent-greeter:local
55
+ description: The tag the build gives the image, and the only name the run step has for it.
56
+
57
+ steps:
58
+ dockerfile:
59
+ block: docker.run
60
+ deadline: 5m
61
+ config:
62
+ image: alpine:3
63
+ pull: true
64
+ network: none
65
+ # Whatever the container leaves in its output mount is copied where the target names
66
+ # once it exits. A target with no URI scheme is a path in the run's work directory on
67
+ # this worker, which is where a build context is read from; a URI would send the file to
68
+ # storage instead, for a result that outlives the run.
69
+ outputs:
70
+ Dockerfile: "context/Dockerfile"
71
+ command: |
72
+ cat > /dirigent/outputs/Dockerfile <<'DOCKERFILE'
73
+ FROM alpine:3
74
+ RUN echo "baked in at build time" > /greeting.txt
75
+ CMD ["cat", "/greeting.txt"]
76
+ DOCKERFILE
77
+ memory: 64mb
78
+ pids_limit: 64
79
+
80
+ build:
81
+ block: docker.build
82
+ depends_on: [dockerfile]
83
+ deadline: 10m
84
+ config:
85
+ # A directory inside the run's work directory, never an absolute path: the same
86
+ # "context/" the step above wrote into, with the Dockerfile named relative to it.
87
+ context: context
88
+ dockerfile: Dockerfile
89
+ # The image is loaded into the worker's daemon store under this tag, and the id comes
90
+ # back from buildx's --iidfile rather than scraped out of the build log.
91
+ tags: ["${params.image_tag}"]
92
+ # Fetch a newer alpine:3 rather than building on whatever the daemon happens to hold.
93
+ pull: true
94
+
95
+ greet:
96
+ block: docker.run
97
+ depends_on: [build]
98
+ deadline: 5m
99
+ config:
100
+ image: "${params.image_tag}"
101
+ # pull stays off, and must: the image was never pushed, so a pull would go looking for
102
+ # this tag in a registry and fail with a name the daemon already has locally.
103
+ pull: false
104
+ network: none
105
+ memory: 64mb
106
+ pids_limit: 64