dirigent-examples 0.15.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. dirigent_examples/__init__.py +22 -0
  2. dirigent_examples/py.typed +0 -0
  3. dirigent_examples/shelves/README.md +299 -0
  4. dirigent_examples/shelves/composition/README.md +18 -0
  5. dirigent_examples/shelves/composition/chained-instances.yaml +97 -0
  6. dirigent_examples/shelves/composition/composition-child.yaml +64 -0
  7. dirigent_examples/shelves/composition/composition-parent.yaml +89 -0
  8. dirigent_examples/shelves/connections.yaml +52 -0
  9. dirigent_examples/shelves/demo/README.md +19 -0
  10. dirigent_examples/shelves/demo/markdown-showcase.yaml +117 -0
  11. dirigent_examples/shelves/demo/params-showcase.yaml +92 -0
  12. dirigent_examples/shelves/demo/requires.yaml +65 -0
  13. dirigent_examples/shelves/demo/weekly-import-malawi.yaml +54 -0
  14. dirigent_examples/shelves/demo/weekly-import-nepal.yaml +75 -0
  15. dirigent_examples/shelves/docker/README.md +29 -0
  16. dirigent_examples/shelves/docker/docker-build-push.yaml +110 -0
  17. dirigent_examples/shelves/docker/docker-build-run.yaml +106 -0
  18. dirigent_examples/shelves/docker/docker-compose-database.yaml +124 -0
  19. dirigent_examples/shelves/docker/docker-compose-failing-up.yaml +83 -0
  20. dirigent_examples/shelves/docker/docker-compose-file.yaml +117 -0
  21. dirigent_examples/shelves/docker/docker-compose-profiles-env.yaml +133 -0
  22. dirigent_examples/shelves/docker/docker-compose-stack.yaml +70 -0
  23. dirigent_examples/shelves/docker/docker-hello.yaml +53 -0
  24. dirigent_examples/shelves/docker/docker-remote-daemon.yaml +92 -0
  25. dirigent_examples/shelves/docker/docker-run-failing-teardown.yaml +94 -0
  26. dirigent_examples/shelves/docker/docker-ticker.yaml +49 -0
  27. dirigent_examples/shelves/execute/README.md +16 -0
  28. dirigent_examples/shelves/execute/long-log.yaml +89 -0
  29. dirigent_examples/shelves/failure/README.md +20 -0
  30. dirigent_examples/shelves/failure/error-handler.yaml +89 -0
  31. dirigent_examples/shelves/failure/optional-step.yaml +82 -0
  32. dirigent_examples/shelves/failure/retries.yaml +75 -0
  33. dirigent_examples/shelves/failure/retry-budget.yaml +82 -0
  34. dirigent_examples/shelves/failure/step-timeout.yaml +96 -0
  35. dirigent_examples/shelves/git/README.md +32 -0
  36. dirigent_examples/shelves/git/git-checkout-build.yaml +125 -0
  37. dirigent_examples/shelves/git/git-checkout-compose.yaml +138 -0
  38. dirigent_examples/shelves/git/git-checkout-public.yaml +84 -0
  39. dirigent_examples/shelves/graph/README.md +22 -0
  40. dirigent_examples/shelves/graph/deep-chain.yaml +119 -0
  41. dirigent_examples/shelves/graph/fan-in.yaml +80 -0
  42. dirigent_examples/shelves/graph/fan-out.yaml +66 -0
  43. dirigent_examples/shelves/graph/linear.yaml +66 -0
  44. dirigent_examples/shelves/graph/parallel-branches.yaml +57 -0
  45. dirigent_examples/shelves/graph/parallel-sleep.yaml +55 -0
  46. dirigent_examples/shelves/graph/skip-diamond.yaml +105 -0
  47. dirigent_examples/shelves/graph/wide-fan.yaml +147 -0
  48. dirigent_examples/shelves/hello-world.yaml +30 -0
  49. dirigent_examples/shelves/open-data/README.md +67 -0
  50. dirigent_examples/shelves/open-data/feeds-composition.yaml +120 -0
  51. dirigent_examples/shelves/open-data/gdacs-disaster-updates.yaml +230 -0
  52. dirigent_examples/shelves/open-data/github-releases-relay.yaml +225 -0
  53. dirigent_examples/shelves/open-data/hdx-dataset-watch.yaml +211 -0
  54. dirigent_examples/shelves/open-data/kobo-submissions-to-csv.yaml +149 -0
  55. dirigent_examples/shelves/open-data/nominatim-geocode-facilities.yaml +169 -0
  56. dirigent_examples/shelves/open-data/odk-central-submissions.yaml +146 -0
  57. dirigent_examples/shelves/open-data/open-meteo-weekly-report.yaml +142 -0
  58. dirigent_examples/shelves/open-data/overpass-health-facilities.yaml +154 -0
  59. dirigent_examples/shelves/open-data/usgs-earthquakes-alert.yaml +208 -0
  60. dirigent_examples/shelves/open-data/who-gho-indicators-to-parquet.yaml +163 -0
  61. dirigent_examples/shelves/open-data/wikidata-country-reference.yaml +146 -0
  62. dirigent_examples/shelves/open-data/world-bank-population-trend.yaml +166 -0
  63. dirigent_examples/shelves/patterns/README.md +144 -0
  64. dirigent_examples/shelves/patterns/concurrency-queue.yaml +89 -0
  65. dirigent_examples/shelves/patterns/concurrency-replace.yaml +92 -0
  66. dirigent_examples/shelves/patterns/concurrency-skip.yaml +97 -0
  67. dirigent_examples/shelves/patterns/connections-referenced-vs-carried.yaml +140 -0
  68. dirigent_examples/shelves/patterns/deadline-on-a-sensor.yaml +106 -0
  69. dirigent_examples/shelves/patterns/fan-out-continue.yaml +88 -0
  70. dirigent_examples/shelves/patterns/fan-out-fail-fast.yaml +80 -0
  71. dirigent_examples/shelves/patterns/fan-out-from-params.yaml +84 -0
  72. dirigent_examples/shelves/patterns/fan-out-item-wise.yaml +111 -0
  73. dirigent_examples/shelves/patterns/fan-out-literal-list.yaml +81 -0
  74. dirigent_examples/shelves/patterns/fan-out-nested-objects.yaml +107 -0
  75. dirigent_examples/shelves/patterns/fan-out-then-join.yaml +86 -0
  76. dirigent_examples/shelves/patterns/log-levels.yaml +119 -0
  77. dirigent_examples/shelves/patterns/outputs-inline-vs-storage.yaml +140 -0
  78. dirigent_examples/shelves/patterns/params-every-type.yaml +259 -0
  79. dirigent_examples/shelves/patterns/params-validation-refuses.yaml +131 -0
  80. dirigent_examples/shelves/patterns/pipeline-run-child.yaml +96 -0
  81. dirigent_examples/shelves/patterns/pipeline-run-fire-and-forget.yaml +99 -0
  82. dirigent_examples/shelves/patterns/pipeline-run-strict.yaml +103 -0
  83. dirigent_examples/shelves/patterns/pipeline-run-wait.yaml +107 -0
  84. dirigent_examples/shelves/patterns/pipeline-run-with-params.yaml +124 -0
  85. dirigent_examples/shelves/patterns/poll-cadence.yaml +102 -0
  86. dirigent_examples/shelves/patterns/priority-layered.yaml +120 -0
  87. dirigent_examples/shelves/patterns/references-cheat-sheet.yaml +186 -0
  88. dirigent_examples/shelves/patterns/retry-budget-exhausted.yaml +92 -0
  89. dirigent_examples/shelves/patterns/retry-exponential-backoff.yaml +88 -0
  90. dirigent_examples/shelves/patterns/retry-only-transient.yaml +108 -0
  91. dirigent_examples/shelves/patterns/retry-with-jitter.yaml +102 -0
  92. dirigent_examples/shelves/patterns/rule-all-done.yaml +83 -0
  93. dirigent_examples/shelves/patterns/rule-all-success.yaml +79 -0
  94. dirigent_examples/shelves/patterns/rule-always.yaml +92 -0
  95. dirigent_examples/shelves/patterns/rule-one-failed.yaml +86 -0
  96. dirigent_examples/shelves/patterns/schedule-at-once.yaml +110 -0
  97. dirigent_examples/shelves/patterns/schedule-cron-timezone.yaml +121 -0
  98. dirigent_examples/shelves/patterns/schedule-interval.yaml +109 -0
  99. dirigent_examples/shelves/patterns/schedule-window-half-open.yaml +105 -0
  100. dirigent_examples/shelves/patterns/sensor-http-ready.yaml +121 -0
  101. dirigent_examples/shelves/patterns/sensor-storage-exists.yaml +134 -0
  102. dirigent_examples/shelves/patterns/step-names-and-keys.yaml +99 -0
  103. dirigent_examples/shelves/patterns/timeout-fails-the-step.yaml +94 -0
  104. dirigent_examples/shelves/patterns/timeout-skips-the-step.yaml +102 -0
  105. dirigent_examples/shelves/patterns/webhook-mapping-nested-payload.yaml +125 -0
  106. dirigent_examples/shelves/patterns/webhook-signed.yaml +144 -0
  107. dirigent_examples/shelves/preview/s3-parquet-to-ingestion.yaml +92 -0
  108. dirigent_examples/shelves/python/README.md +31 -0
  109. dirigent_examples/shelves/python/apply_and_run.py +52 -0
  110. dirigent_examples/shelves/python/ci_gate.py +76 -0
  111. dirigent_examples/shelves/python/connections.py +61 -0
  112. dirigent_examples/shelves/python/error_handling.py +84 -0
  113. dirigent_examples/shelves/python/follow_logs.py +39 -0
  114. dirigent_examples/shelves/python/list_and_filter.py +52 -0
  115. dirigent_examples/shelves/queues/README.md +59 -0
  116. dirigent_examples/shelves/queues/kafka-consume-then-transform.yaml +105 -0
  117. dirigent_examples/shelves/queues/kafka-produce-then-consume.yaml +124 -0
  118. dirigent_examples/shelves/queues/rabbitmq-consume-ack-on-success.yaml +102 -0
  119. dirigent_examples/shelves/queues/report-to-kafka.yaml +105 -0
  120. dirigent_examples/shelves/queues/report-to-rabbitmq.yaml +105 -0
  121. dirigent_examples/shelves/recipes/README.md +130 -0
  122. dirigent_examples/shelves/recipes/csv-header-rules.yaml +147 -0
  123. dirigent_examples/shelves/recipes/csv-to-ndjson.yaml +107 -0
  124. dirigent_examples/shelves/recipes/etl-csv-clean-validate-parquet.yaml +207 -0
  125. dirigent_examples/shelves/recipes/filter-by-predicate.yaml +116 -0
  126. dirigent_examples/shelves/recipes/filter-then-map-then-reduce.yaml +109 -0
  127. dirigent_examples/shelves/recipes/http-fetch-validate-post.yaml +142 -0
  128. dirigent_examples/shelves/recipes/http-follow-redirects.yaml +100 -0
  129. dirigent_examples/shelves/recipes/http-get-with-query.yaml +96 -0
  130. dirigent_examples/shelves/recipes/http-headers-and-auth-connection.yaml +110 -0
  131. dirigent_examples/shelves/recipes/http-post-file-from-storage.yaml +117 -0
  132. dirigent_examples/shelves/recipes/http-post-json-echo.yaml +105 -0
  133. dirigent_examples/shelves/recipes/http-post-report.yaml +183 -0
  134. dirigent_examples/shelves/recipes/http-save-body-to-storage.yaml +106 -0
  135. dirigent_examples/shelves/recipes/http-success-status-list.yaml +80 -0
  136. dirigent_examples/shelves/recipes/http-timeout-override.yaml +104 -0
  137. dirigent_examples/shelves/recipes/jq-dedupe-by-key.yaml +78 -0
  138. dirigent_examples/shelves/recipes/jq-defaults-and-nulls.yaml +91 -0
  139. dirigent_examples/shelves/recipes/jq-group-by-and-sum.yaml +74 -0
  140. dirigent_examples/shelves/recipes/jq-join-two-lists.yaml +77 -0
  141. dirigent_examples/shelves/recipes/jq-long-to-wide.yaml +76 -0
  142. dirigent_examples/shelves/recipes/jq-nested-to-flat.yaml +89 -0
  143. dirigent_examples/shelves/recipes/jq-pivot-wide-to-long.yaml +65 -0
  144. dirigent_examples/shelves/recipes/jq-running-totals.yaml +82 -0
  145. dirigent_examples/shelves/recipes/jq-string-cleaning.yaml +88 -0
  146. dirigent_examples/shelves/recipes/jq-top-n.yaml +85 -0
  147. dirigent_examples/shelves/recipes/jq-validate-in-jq-vs-schema.yaml +124 -0
  148. dirigent_examples/shelves/recipes/jq-window-dates.yaml +82 -0
  149. dirigent_examples/shelves/recipes/json-to-csv-flattening.yaml +141 -0
  150. dirigent_examples/shelves/recipes/large-output-to-storage.yaml +134 -0
  151. dirigent_examples/shelves/recipes/map-enrich-with-lookup.yaml +96 -0
  152. dirigent_examples/shelves/recipes/ndjson-to-parquet.yaml +130 -0
  153. dirigent_examples/shelves/recipes/pagination-by-fan-out.yaml +115 -0
  154. dirigent_examples/shelves/recipes/parquet-round-trip-types.yaml +163 -0
  155. dirigent_examples/shelves/recipes/reconcile-two-sources.yaml +159 -0
  156. dirigent_examples/shelves/recipes/report-built-in.yaml +72 -0
  157. dirigent_examples/shelves/recipes/report-daily-digest.yaml +186 -0
  158. dirigent_examples/shelves/recipes/report-to-file.yaml +131 -0
  159. dirigent_examples/shelves/recipes/report-to-webhook.yaml +136 -0
  160. dirigent_examples/shelves/recipes/schema-carried.yaml +112 -0
  161. dirigent_examples/shelves/recipes/schema-formats.yaml +107 -0
  162. dirigent_examples/shelves/recipes/schema-referenced.yaml +86 -0
  163. dirigent_examples/shelves/recipes/schema-refuses-then-rule.yaml +127 -0
  164. dirigent_examples/shelves/recipes/storage-copy-dated-archive.yaml +114 -0
  165. dirigent_examples/shelves/recipes/storage-exists-gate.yaml +127 -0
  166. dirigent_examples/shelves/recipes/storage-manifest-of-a-fan-out.yaml +104 -0
  167. dirigent_examples/shelves/recipes/storage-write-then-read.yaml +119 -0
  168. dirigent_examples/shelves/recipes/webhook-post-hmac.yaml +132 -0
  169. dirigent_examples/shelves/recipes/webhook-post-summary.yaml +142 -0
  170. dirigent_examples/shelves/s3/README.md +34 -0
  171. dirigent_examples/shelves/s3/report-to-s3.yaml +93 -0
  172. dirigent_examples/shelves/s3/s3-copy-and-verify.yaml +105 -0
  173. dirigent_examples/shelves/s3/s3-csv-report.yaml +87 -0
  174. dirigent_examples/shelves/s3/s3-parquet-report.yaml +77 -0
  175. dirigent_examples/shelves/s3/s3-round-trip.yaml +125 -0
  176. dirigent_examples/shelves/schemas/README.md +36 -0
  177. dirigent_examples/shelves/schemas/echo-reading.json +18 -0
  178. dirigent_examples/shelves/schemas/ou-record.json +13 -0
  179. dirigent_examples/shelves/schemas/station-reading.json +13 -0
  180. dirigent_examples/shelves/sensors/README.md +16 -0
  181. dirigent_examples/shelves/sensors/sensor-gate.yaml +65 -0
  182. dirigent_examples/shelves/sensors/time-window.yaml +61 -0
  183. dirigent_examples/shelves/sql/README.md +52 -0
  184. dirigent_examples/shelves/sql/duckdb-parquet-to-report.yaml +146 -0
  185. dirigent_examples/shelves/sql/sql-postgres-readonly.yaml +111 -0
  186. dirigent_examples/shelves/sql/sql-query-to-storage.yaml +85 -0
  187. dirigent_examples/shelves/sql/sql-sqlite-roundtrip.yaml +114 -0
  188. dirigent_examples/shelves/sql/warehouse.sql +42 -0
  189. dirigent_examples/shelves/transform/README.md +36 -0
  190. dirigent_examples/shelves/transform/csv-report.yaml +55 -0
  191. dirigent_examples/shelves/transform/jq-filter-and-map.yaml +70 -0
  192. dirigent_examples/shelves/transform/jq-group-and-aggregate.yaml +70 -0
  193. dirigent_examples/shelves/transform/jq-join-two-sources.yaml +98 -0
  194. dirigent_examples/shelves/transform/jq-reshape.yaml +91 -0
  195. dirigent_examples/shelves/transform/jq-stream-through-storage.yaml +112 -0
  196. dirigent_examples/shelves/transform/ndjson-round-trip.yaml +56 -0
  197. dirigent_examples/shelves/transform/parquet-round-trip.yaml +68 -0
  198. dirigent_examples/shelves/transform/std-convert-fan-out.yaml +142 -0
  199. dirigent_examples/shelves/transform/xml-feed-to-ndjson.yaml +116 -0
  200. dirigent_examples/shelves/transform/yaml-config-to-json.yaml +104 -0
  201. dirigent_examples/shelves/triggers/README.md +45 -0
  202. dirigent_examples/shelves/triggers/at-one-time.yaml +78 -0
  203. dirigent_examples/shelves/triggers/cron-nightly.yaml +79 -0
  204. dirigent_examples/shelves/triggers/cron-windowed.yaml +86 -0
  205. dirigent_examples/shelves/triggers/document-nightly.yaml +80 -0
  206. dirigent_examples/shelves/triggers/interval-rolling.yaml +88 -0
  207. dirigent_examples/shelves/triggers/managed-and-manual.yaml +109 -0
  208. dirigent_examples/shelves/triggers/webhook-trigger.yaml +75 -0
  209. dirigent_examples/shelves/validate/README.md +31 -0
  210. dirigent_examples/shelves/validate/expects-a-shape.yaml +56 -0
  211. dirigent_examples/shelves/validate/the-shape-is-wrong.yaml +46 -0
  212. dirigent_examples-0.15.0.dist-info/METADATA +21 -0
  213. dirigent_examples-0.15.0.dist-info/RECORD +216 -0
  214. dirigent_examples-0.15.0.dist-info/WHEEL +4 -0
  215. dirigent_examples-0.15.0.dist-info/entry_points.txt +3 -0
  216. dirigent_examples-0.15.0.dist-info/licenses/LICENSE +18 -0
@@ -0,0 +1,259 @@
1
+ # Every JSON Schema type a parameter can be, and the formats that actually assert.
2
+ #
3
+ # demo/params-showcase.yaml is about how parameters are PASSED -- flags, files, dotted keys,
4
+ # coercion. This one is about what they are DECLARED as: the six types, the keywords that
5
+ # constrain each of them, and the formats a dirigent instance genuinely checks rather than
6
+ # merely annotates.
7
+ #
8
+ # The schema is plain JSON Schema, draft 2020-12, and it is load-bearing in three places: it
9
+ # validates every run's parameters whoever started them, the UI renders the run dialog from
10
+ # it, and apply checks every ${params.x} in the document against it so a typo is caught at
11
+ # apply time rather than at 05:00.
12
+ #
13
+ # ABOUT format, AND THIS IS THE PART THAT SURPRISES PEOPLE. A format keyword asserts only
14
+ # where the validator is given a checker for it. Parameter validation is not given one, so
15
+ # every format below is an ANNOTATION: it documents intent and it is what the UI builds a date
16
+ # picker from, and -p day=2026-13-40 is accepted. Types and bounds and enums and patterns all
17
+ # assert; formats do not.
18
+ #
19
+ # Where they do assert is validate.schema, which is handed the instance's format checker --
20
+ # the standard formats from the library (date, date-time, time, duration, email, uri, uuid,
21
+ # ipv4, hostname, regex, json-pointer), the ones dirigent adds (ulid, uuid4, uuid7, md5, sha1,
22
+ # sha256, sha512, base64), and whatever a plugin pack contributes, which is how a pack's
23
+ # own format comes to be a format a pipeline can gate on. So a parameter that genuinely has to be a date
24
+ # gets a gate, and this file has one: a pattern is the other answer, and it asserts in the
25
+ # parameter schema itself.
26
+ #
27
+ # Every parameter below has a default, so the whole thing runs with no flags at all. That is
28
+ # a property worth keeping in a corpus: an example nobody can run without reading the header
29
+ # first is an example nobody runs.
30
+ #
31
+ # Hop by hop:
32
+ #
33
+ # echo one transform.jq step that pulls every parameter into one object, so the run's
34
+ # output is the resolved parameter set and the defaults are visible in it.
35
+ # gate validate.schema against the schema this document carries, which is where the
36
+ # formats actually bite. With the defaults it passes and hands the value straight on.
37
+ # shape proves the types survived: an integer is arithmetic, a boolean is a branch, and an
38
+ # array has a length. A parameter that arrived as text would fail here.
39
+ #
40
+ # EXPECT THIS RUN TO SUCCEED in about a second, with no network and no allowlist.
41
+ #
42
+ # To change it: -p day=2026-13-40 is ACCEPTED by the parameter schema and REFUSED by the gate,
43
+ # which is the whole point of having both. params-validation-refuses.yaml is the other half:
44
+ # what a request that fails the parameter schema itself is answered with.
45
+ #
46
+ # dg run --local examples/patterns/params-every-type.yaml
47
+ # dg run --local examples/patterns/params-every-type.yaml -p day=2026-06-01 -p attempts=5
48
+ # dg run --local examples/patterns/params-every-type.yaml -p day=2026-13-40 # fails at the gate
49
+
50
+ format: dirigent/v1
51
+ kind: pipeline
52
+ code: params-every-type
53
+ name: Every parameter type
54
+ description: |
55
+ The six JSON Schema types and the keywords that constrain each of them, with every
56
+ parameter carrying a default so the document runs bare.
57
+
58
+ A `format` in a parameter schema is an **annotation**: it documents intent and the UI
59
+ renders from it, and nothing checks it. Formats assert inside `validate.schema`, which is
60
+ handed the instance's checker -- so this document carries a schema and gates on it.
61
+
62
+ tags: [patterns, transform, validate, params]
63
+
64
+ # Carried in the document rather than named on an instance, so a --local run gates against
65
+ # exactly the same shape a server would. A server refuses a document that carries a schema
66
+ # this way for a pipeline it stores; a published example is where carrying one is right.
67
+ schemas:
68
+ run-inputs:
69
+ type: object
70
+ required: [day, correlation_id, upload_id, expected_digest]
71
+ properties:
72
+ # The same four format keywords as in params below, in the one place they assert.
73
+ day: {type: string, format: date}
74
+ correlation_id: {type: string, format: uuid4}
75
+ upload_id: {type: string, format: ulid}
76
+ expected_digest: {type: string, format: sha256}
77
+
78
+ requires:
79
+ blocks:
80
+ - transform.jq
81
+ - validate.schema
82
+
83
+ params:
84
+ type: object
85
+ # Nothing is required, because everything has a default. required is for the parameter a
86
+ # run has no sensible value for -- the day being backfilled, the tenant being loaded.
87
+ additionalProperties: false
88
+ properties:
89
+ # A string, constrained by length and by pattern. The pattern is anchored, because an
90
+ # unanchored one matches anywhere in the value and refuses almost nothing.
91
+ dataset:
92
+ type: string
93
+ description: Which dataset this run covers.
94
+ default: climate
95
+ minLength: 1
96
+ maxLength: 64
97
+ pattern: "^[a-z][a-z0-9-]*$"
98
+
99
+ # An enum, which is a closed set rather than a type. The UI renders it as a select.
100
+ environment:
101
+ type: string
102
+ description: Where this run writes.
103
+ default: staging
104
+ enum: [staging, production]
105
+
106
+ # An integer with bounds, so a fat-fingered 1000 is refused rather than retried a
107
+ # thousand times.
108
+ attempts:
109
+ type: integer
110
+ description: How many times a downstream loader should try.
111
+ default: 3
112
+ minimum: 1
113
+ maximum: 10
114
+
115
+ # A number, which is any JSON number rather than a whole one. exclusiveMinimum is the
116
+ # keyword for a value that may approach zero and never reach it.
117
+ threshold:
118
+ type: number
119
+ description: The confidence a prediction has to clear.
120
+ default: 0.75
121
+ exclusiveMinimum: 0.0
122
+ maximum: 1.0
123
+
124
+ # A boolean, which the UI renders as a switch. Named for what true means, so nobody has
125
+ # to work out what a false "no_dry_run" would do.
126
+ dry_run:
127
+ type: boolean
128
+ description: When true nothing downstream writes.
129
+ default: true
130
+
131
+ # An array. items constrains every element, and the two bounds are what stop a fan-out
132
+ # over this from being either empty or enormous.
133
+ regions:
134
+ type: array
135
+ description: The regions this run covers.
136
+ default: [east, west]
137
+ minItems: 1
138
+ maxItems: 16
139
+ uniqueItems: true
140
+ items:
141
+ type: string
142
+ minLength: 1
143
+
144
+ # An object, with its own properties and its own closed door. A nested leaf is addressed
145
+ # from the CLI with a dotted key: -p window.days=30.
146
+ window:
147
+ type: object
148
+ description: The rolling window a load covers.
149
+ default: {days: 7, align: midnight}
150
+ additionalProperties: false
151
+ required: [days]
152
+ properties:
153
+ days:
154
+ type: integer
155
+ minimum: 1
156
+ maximum: 365
157
+ align:
158
+ type: string
159
+ enum: [midnight, hour]
160
+
161
+ # Formats, one line each. None of these asserts here; the gate below is what asserts.
162
+ day:
163
+ type: string
164
+ description: A calendar date, which the UI renders as a date picker.
165
+ default: "2026-01-01"
166
+ format: date
167
+
168
+ since:
169
+ type: string
170
+ description: An instant, offset included.
171
+ default: "2026-01-01T00:00:00+00:00"
172
+ format: date-time
173
+
174
+ cutoff:
175
+ type: string
176
+ description: A local time of day, with no date and no zone.
177
+ default: "23:30:00"
178
+ format: time
179
+
180
+ catalog_url:
181
+ type: string
182
+ description: Where the catalog is read from.
183
+ default: https://postman-echo.com/get
184
+ format: uri
185
+
186
+ owner_email:
187
+ type: string
188
+ description: Who to write to about this run.
189
+ default: data-team@example.org
190
+ format: email
191
+
192
+ correlation_id:
193
+ type: string
194
+ description: A UUID version 4 specifically; the plain uuid format would take any version.
195
+ default: 6f0b6e34-6c0e-4c2f-9a1e-2f43a1a0d0b7
196
+ format: uuid4
197
+
198
+ upload_id:
199
+ type: string
200
+ description: A Crockford base32 ULID, one of the formats dirigent contributes.
201
+ default: 01J8Z3M4N5P6Q7R8S9TVWXYZ01
202
+ format: ulid
203
+
204
+ expected_digest:
205
+ type: string
206
+ description: A sha256 hex digest, checked for length and alphabet rather than parsed.
207
+ default: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
208
+ format: sha256
209
+
210
+ steps:
211
+ echo:
212
+ block: transform.jq
213
+ config:
214
+ input:
215
+ dataset: "${params.dataset}"
216
+ environment: "${params.environment}"
217
+ attempts: "${params.attempts}"
218
+ threshold: "${params.threshold}"
219
+ dry_run: "${params.dry_run}"
220
+ regions: "${params.regions}"
221
+ # A whole reference to an object resolves to the object, so the nested shape survives
222
+ # intact rather than being flattened into text.
223
+ window: "${params.window}"
224
+ day: "${params.day}"
225
+ since: "${params.since}"
226
+ cutoff: "${params.cutoff}"
227
+ catalog_url: "${params.catalog_url}"
228
+ owner_email: "${params.owner_email}"
229
+ correlation_id: "${params.correlation_id}"
230
+ upload_id: "${params.upload_id}"
231
+ expected_digest: "${params.expected_digest}"
232
+ program: |
233
+ .
234
+
235
+ gate:
236
+ block: validate.schema
237
+ depends_on: [echo]
238
+ # Where the formats bite. The gate is also a waypoint: everything below reads the gate's
239
+ # output rather than the echo's, so the document itself shows that nothing past this point
240
+ # saw an unchecked value.
241
+ config:
242
+ input: "${steps.echo.output.value}"
243
+ schema: run-inputs
244
+
245
+ shape:
246
+ block: transform.jq
247
+ depends_on: [gate]
248
+ # The proof that types survived the trip. Arithmetic on a string, or a branch on the text
249
+ # "false", would fail here rather than quietly doing the wrong thing.
250
+ config:
251
+ input: "${steps.gate.output.value}"
252
+ program: |
253
+ {
254
+ budget: (.attempts * .window.days),
255
+ confident: (.threshold > 0.5),
256
+ writes: (if .dry_run then "nothing" else .environment end),
257
+ regions: (.regions | length),
258
+ nested_leaf: .window.align
259
+ }
@@ -0,0 +1,131 @@
1
+ # What a bad run request is answered with, and how early the answer comes.
2
+ #
3
+ # Parameter validation is not a step. It happens when the RUN IS CREATED: defaults are filled
4
+ # in, the schema is checked against the resolved object, and a request that does not satisfy
5
+ # it is refused before a run row exists. So a bad request produces no run at all -- nothing to
6
+ # find in dg runs list, nothing half-executed, no side effect anywhere. The error names the
7
+ # location and says what was wrong with it:
8
+ #
9
+ # parameter attempts is invalid: 99 is greater than the maximum of 10
10
+ # parameter tenant is invalid: '' should be non-empty
11
+ # parameter tenant is invalid: 'Nordic' does not match '^[a-z][a-z0-9-]*$'
12
+ #
13
+ # Each guard is worth having on purpose:
14
+ #
15
+ # minimum/maximum catches the fat-fingered order of magnitude.
16
+ # minLength/pattern catches the value of the right type and the wrong shape.
17
+ # required catches the parameter nobody can guess a default for.
18
+ # additionalProperties: false catches the TYPO, which is the one people forget. Without it,
19
+ # regoins=[east] is accepted in silence, regions keeps its default, and
20
+ # the run does the wrong work with a clean bill of health. The CLI
21
+ # catches an unknown -p by name before it sends anything, naming what
22
+ # the pipeline does declare; this keyword is what catches the same typo
23
+ # arriving from the API, from a schedule's pinned params, or from a
24
+ # webhook's mapped payload.
25
+ #
26
+ # WHAT IS NOT CHECKED HERE: format. A format keyword asserts only where the validator is given
27
+ # a checker for it, and parameter validation is not given one -- so day below is documentation
28
+ # and the UI's date picker, and -p day=2026-13-40 starts a run. A value that genuinely has to
29
+ # be a date is gated with validate.schema, which IS handed the instance's format checker;
30
+ # params-every-type.yaml carries that gate. A pattern is the other answer, and a pattern does
31
+ # assert here.
32
+ #
33
+ # Defaults are filled in BEFORE validation, so a parameter with a default is optional and one
34
+ # without a default and inside required is mandatory. There is no third state.
35
+ #
36
+ # Hop by hop:
37
+ #
38
+ # plan builds the load plan out of the validated parameters. If it runs at all, every
39
+ # value it reads has already satisfied the schema -- which is why no step in this
40
+ # file checks anything.
41
+ # receipt posts the plan on, standing in for the work.
42
+ #
43
+ # EXPECT THIS RUN TO SUCCEED with the defaults, in about a second. The demonstration is the
44
+ # runs that never start:
45
+ #
46
+ # dg run --local examples/patterns/params-validation-refuses.yaml
47
+ # dg run --local examples/patterns/params-validation-refuses.yaml -p tenant=Nordic # refused, pattern
48
+ # dg run --local examples/patterns/params-validation-refuses.yaml -p attempts=99 # refused
49
+ # dg run --local examples/patterns/params-validation-refuses.yaml -p regoins='["east"]' # refused
50
+ # dg run --local examples/patterns/params-validation-refuses.yaml -p tenant= # refused, minLength
51
+ #
52
+ # The same schema refuses the same values whoever is asking: the CLI, the API, a schedule's
53
+ # pinned params, and a webhook's mapped payload all land in one validate call. That is what
54
+ # makes a webhook safe to expose -- a caller cannot reach a parameter the mapping does not
55
+ # name, and cannot get a value past the schema even for one it does.
56
+
57
+ format: dirigent/v1
58
+ kind: pipeline
59
+ code: params-validation-refuses
60
+ name: A run request that is refused
61
+ description: |
62
+ Parameters are validated when the run is **created**, so a bad request produces no run at
63
+ all: nothing to find in `dg runs list`, and no side effect anywhere.
64
+
65
+ `format`, bounds, `required` and `additionalProperties: false` are four different guards.
66
+ The last one catches the typo, which is the one people forget.
67
+
68
+ tags: [patterns, http, transform, params]
69
+
70
+ requires:
71
+ blocks:
72
+ - transform.jq
73
+ - http.request
74
+
75
+ params:
76
+ type: object
77
+ # The parameter nobody can guess for you. Everything else has a default, so this is the
78
+ # only one a run must supply -- and there is a default here too, so the example runs bare.
79
+ required: [tenant]
80
+ # The typo guard. Turning this off is how a run does the wrong work and reports success.
81
+ additionalProperties: false
82
+ properties:
83
+ tenant:
84
+ type: string
85
+ description: Whose data this run loads.
86
+ default: nordic
87
+ minLength: 1
88
+ maxLength: 32
89
+ pattern: "^[a-z][a-z0-9-]*$"
90
+ day:
91
+ type: string
92
+ description: The day being loaded. The format here documents and renders; it does not check.
93
+ default: "2026-01-01"
94
+ format: date
95
+ attempts:
96
+ type: integer
97
+ description: How many times the loader tries; bounded so a slip cannot mean ninety-nine.
98
+ default: 3
99
+ minimum: 1
100
+ maximum: 10
101
+ regions:
102
+ type: array
103
+ description: The regions to load. Misspell the name and the run is refused, not defaulted.
104
+ default: [east, west]
105
+ minItems: 1
106
+ items:
107
+ type: string
108
+
109
+ steps:
110
+ plan:
111
+ block: transform.jq
112
+ # Nothing here re-checks anything. A step downstream of validation is entitled to assume
113
+ # the schema held, and a document that validates its own parameters again is a document
114
+ # whose schema was not doing its job.
115
+ config:
116
+ input:
117
+ tenant: "${params.tenant}"
118
+ day: "${params.day}"
119
+ attempts: "${params.attempts}"
120
+ regions: "${params.regions}"
121
+ program: |
122
+ {tenant, day, attempts, regions, work: ((.regions | length) * .attempts)}
123
+
124
+ receipt:
125
+ block: http.request
126
+ depends_on: [plan]
127
+ config:
128
+ url: https://postman-echo.com/post
129
+ method: POST
130
+ body:
131
+ plan: "${steps.plan.output.value}"
@@ -0,0 +1,96 @@
1
+ # The pipeline the four composition examples call, and an ordinary pipeline in every way.
2
+ #
3
+ # Nothing here says it is a child. It has its own code, its own parameter schema, its own run
4
+ # history, and it can be run on its own, scheduled on its own, or called by four different
5
+ # parents. Being callable is not something a document opts into.
6
+ #
7
+ # ITS PARAMETER SCHEMA IS THE INTERFACE. A parent writes these parameters out explicitly
8
+ # rather than forwarding whatever it was run with, and they are validated against the schema
9
+ # below when the parent's step executes -- not when the parent is applied. So a parent that
10
+ # calls this with a region of 42 applies cleanly and fails at that step, which is why
11
+ # additionalProperties: false and the bounds below are worth having.
12
+ #
13
+ # The status parameter exists for one of the parents. pipeline-run-strict.yaml needs a child
14
+ # that settles completed_with_errors on purpose, and this is how: a tolerated failure, which
15
+ # is a failed step whose dependents carry on and whose run is not clean.
16
+ #
17
+ # Hop by hop:
18
+ #
19
+ # fetch a tolerated call. Asked for 200 it succeeds and the run is green; asked for 500
20
+ # it settles failed and is tolerated, so the run ends completed_with_errors with
21
+ # the branch below it still running.
22
+ # summarise reads the parameters back into the small record a parent reads as this pipeline's
23
+ # answer. It runs either way, because a tolerated failure reads as a success to
24
+ # whatever depends on it.
25
+ #
26
+ # EXPECT THIS RUN TO SUCCEED on its defaults, in about a second:
27
+ #
28
+ # dg run --local examples/patterns/pipeline-run-child.yaml -p region=east
29
+ # dg run --local examples/patterns/pipeline-run-child.yaml -p region=east -p status=500 # completed_with_errors
30
+ #
31
+ # Apply it before any of the parents; each of them names it in requires.pipelines, so a parent
32
+ # applied to an instance without it is refused up front with the code to apply first.
33
+
34
+ format: dirigent/v1
35
+ kind: pipeline
36
+ code: pipeline-run-child
37
+ name: The called pipeline
38
+ description: |
39
+ An ordinary pipeline that four composition examples call. Nothing about it declares that
40
+ it is a child; its parameter schema is the whole interface.
41
+
42
+ `-p status=500` makes it settle `completed_with_errors` on purpose, which is what
43
+ `pipeline-run-strict.yaml` needs something to do.
44
+
45
+ tags: [patterns, http, transform, composition]
46
+
47
+ requires:
48
+ blocks:
49
+ - http.request
50
+ - transform.jq
51
+
52
+ params:
53
+ type: object
54
+ required: [region]
55
+ # A closed door, so a parent that misspells a parameter fails at the step with the reason
56
+ # rather than running with a default nobody chose.
57
+ additionalProperties: false
58
+ properties:
59
+ region:
60
+ type: string
61
+ description: Which region this run loads.
62
+ minLength: 1
63
+ maxLength: 32
64
+ pattern: "^[a-z][a-z0-9-]*$"
65
+ day:
66
+ type: string
67
+ format: date
68
+ description: The day being loaded.
69
+ default: "2026-01-01"
70
+ status:
71
+ type: integer
72
+ description: What the tolerated call is answered with; 500 settles this run completed_with_errors.
73
+ default: 200
74
+ enum: [200, 500]
75
+
76
+ steps:
77
+ fetch:
78
+ block: http.request
79
+ # Tolerated: it settles failed, its dependents are shown a success, and the run reports
80
+ # completed_with_errors rather than failed. That third status is what strict is about.
81
+ continue_on_failure: true
82
+ config:
83
+ # A conditional written as data: the parameter picks the status, because the reference
84
+ # language has no branching construct and does not want one.
85
+ url: "https://postman-echo.com/status/${params.status}"
86
+ method: GET
87
+
88
+ summarise:
89
+ block: transform.jq
90
+ depends_on: [fetch]
91
+ config:
92
+ input:
93
+ region: "${params.region}"
94
+ day: "${params.day}"
95
+ program: |
96
+ {loaded: .region, day: .day}
@@ -0,0 +1,99 @@
1
+ # pipeline.run with wait: false -- start the child, hand back its run id, and move on.
2
+ #
3
+ # wait: false finishes the parent's step THE MOMENT THE CHILD RUN EXISTS. The step's output
4
+ # carries the child's code and its run id, and a status of "started" -- which is not a claim
5
+ # about how the child ended, because at that instant nothing knows.
6
+ #
7
+ # So this is a fork, not a call, and the consequences are worth being blunt about:
8
+ #
9
+ # * The child's outcome never reaches this run. A child that fails leaves the parent green.
10
+ # * Nothing downstream of the step may assume the child's work happened. An edge below a
11
+ # fire-and-forget step orders itself against the child's CREATION and nothing else.
12
+ # * The run id in the output is the whole handoff. It is what an operator pastes into
13
+ # dg runs show, and what a notification carries, so that the fork is findable later.
14
+ #
15
+ # Use it for the branch nobody is waiting on: a notification, a cache warm, a downstream job
16
+ # with its own alerting. Use wait: true whenever "and then" is part of the sentence.
17
+ #
18
+ # A third status is possible here and worth knowing about: skipped. If the child's own
19
+ # concurrency policy refuses the run -- concurrency: skip on the child, with one already
20
+ # going -- no child run is created, run_id is null, and the step reports skipped rather than
21
+ # failing. A parent that assumes a run id is always there will read a null.
22
+ #
23
+ # Hop by hop:
24
+ #
25
+ # waited the child, called the ordinary way, so this file has one of each to compare.
26
+ # forked the child again, not waited on. It finishes in milliseconds.
27
+ # handoff reads both outputs side by side: one status is the child's real outcome, the
28
+ # other is "started".
29
+ #
30
+ # EXPECT THIS RUN TO SUCCEED in about three seconds, with waited's status succeeded and
31
+ # forked's status started. In a --local run the throwaway instance goes away when the parent
32
+ # settles, so a forked child may not get to finish -- which is the honest shape of the thing:
33
+ # nobody is waiting for it.
34
+ #
35
+ # dg run --local examples/patterns/pipeline-run-fire-and-forget.yaml \
36
+ # --also-apply examples/patterns/pipeline-run-child.yaml
37
+
38
+ format: dirigent/v1
39
+ kind: pipeline
40
+ code: pipeline-run-fire-and-forget
41
+ name: A parent that does not wait
42
+ description: |
43
+ `wait: false` finishes the step as soon as the child run exists, reporting `started` and
44
+ the child's run id.
45
+
46
+ A fork, not a call: the child's outcome never reaches this run, and nothing downstream may
47
+ assume the child's work happened.
48
+
49
+ tags: [patterns, pipeline, transform, composition]
50
+
51
+ requires:
52
+ blocks:
53
+ - pipeline.run
54
+ - transform.jq
55
+ pipelines:
56
+ - pipeline-run-child
57
+
58
+ params:
59
+ type: object
60
+ properties:
61
+ day:
62
+ type: string
63
+ format: date
64
+ default: "2026-01-01"
65
+
66
+ steps:
67
+ waited:
68
+ block: pipeline.run
69
+ poll: 1s
70
+ config:
71
+ pipeline: pipeline-run-child
72
+ params:
73
+ region: east
74
+ day: "${params.day}"
75
+
76
+ forked:
77
+ block: pipeline.run
78
+ # No poll, because there is nothing to probe: the step is over as soon as the run row
79
+ # exists. A cadence here would be configuration that never fires.
80
+ config:
81
+ pipeline: pipeline-run-child
82
+ params:
83
+ region: west
84
+ day: "${params.day}"
85
+ # The one line this file is about.
86
+ wait: false
87
+
88
+ handoff:
89
+ block: transform.jq
90
+ depends_on: [waited, forked]
91
+ config:
92
+ input:
93
+ waited_status: "${steps.waited.output.status}"
94
+ forked_status: "${steps.forked.output.status}"
95
+ # The handoff. Without this in an output somewhere, a forked child is a run nobody
96
+ # can find on purpose.
97
+ forked_run: "${steps.forked.output.run_id}"
98
+ program: |
99
+ {waited_status, forked_status, forked_run, note: "started is not an outcome"}
@@ -0,0 +1,103 @@
1
+ # strict: what a child that ended completed_with_errors does to the step that called it.
2
+ #
3
+ # completed_with_errors is a third run status, not a shade of failed: it means every step that
4
+ # had to run ran, and at least one tolerated failure happened along the way. Whether that is
5
+ # acceptable is a judgement the CALLER makes, not the child, because the same child called
6
+ # from two places can be load-bearing in one and best-effort in the other. strict is where
7
+ # that judgement is written down.
8
+ #
9
+ # strict: false (the default) a child that ended completed_with_errors SUCCEEDS this step.
10
+ # strict: true it FAILS this step, exactly as a failed child would.
11
+ #
12
+ # A child that ended failed fails the step either way. strict only ever decides the middle
13
+ # case.
14
+ #
15
+ # Both calls below ask the child for the same deliberate tolerated failure, so the child ends
16
+ # completed_with_errors twice and the only difference is the flag.
17
+ #
18
+ # Hop by hop:
19
+ #
20
+ # lenient strict: false against a child that ends completed_with_errors. The step SUCCEEDS,
21
+ # and its output carries the child's real status -- so "succeeded" here is the
22
+ # step's outcome, not a claim about the child.
23
+ # strict strict: true against the same thing. The step FAILS.
24
+ # verdict all_done, so the run has an outcome to report whichever way the pair went. It
25
+ # reads only the lenient step's output: the strict one failed, so it has none.
26
+ #
27
+ # EXPECT THIS RUN TO FAIL, in about five seconds, with lenient succeeded, strict failed and
28
+ # verdict succeeded. The failure is the demonstration.
29
+ #
30
+ # To change it: -p status=200 makes the child end cleanly and both calls go green, which is
31
+ # the point -- strict costs nothing on a healthy child and is the difference between noticing
32
+ # and not noticing on an unhealthy one.
33
+ #
34
+ # dg run --local examples/patterns/pipeline-run-strict.yaml \
35
+ # --also-apply examples/patterns/pipeline-run-child.yaml # fails, by design
36
+ # dg run --local examples/patterns/pipeline-run-strict.yaml \
37
+ # --also-apply examples/patterns/pipeline-run-child.yaml -p status=200 # succeeds
38
+
39
+ format: dirigent/v1
40
+ kind: pipeline
41
+ code: pipeline-run-strict
42
+ name: Strict about a child's errors
43
+ description: |
44
+ `completed_with_errors` is a third status, and whether it is acceptable is the caller's
45
+ judgement: `strict: false` (the default) passes it, `strict: true` fails the step on it.
46
+
47
+ A child that ended `failed` fails the step either way. As written the strict call fails
48
+ and the run reports `failed`.
49
+
50
+ tags: [patterns, pipeline, transform, composition, failure]
51
+
52
+ requires:
53
+ blocks:
54
+ - pipeline.run
55
+ - transform.jq
56
+ pipelines:
57
+ - pipeline-run-child
58
+
59
+ params:
60
+ type: object
61
+ properties:
62
+ status:
63
+ type: integer
64
+ description: What the child's tolerated call is answered with; 500 makes it completed_with_errors.
65
+ default: 500
66
+ enum: [200, 500]
67
+
68
+ steps:
69
+ lenient:
70
+ block: pipeline.run
71
+ poll: 1s
72
+ config:
73
+ pipeline: pipeline-run-child
74
+ params:
75
+ region: east
76
+ status: "${params.status}"
77
+ # The default, written out so the pair reads as two positions rather than one setting.
78
+ strict: false
79
+
80
+ strict:
81
+ block: pipeline.run
82
+ poll: 1s
83
+ config:
84
+ pipeline: pipeline-run-child
85
+ params:
86
+ region: west
87
+ status: "${params.status}"
88
+ # The one line this file is about.
89
+ strict: true
90
+
91
+ verdict:
92
+ block: transform.jq
93
+ depends_on: [lenient, strict]
94
+ # all_done, because the interesting run is the one where the strict call failed, and
95
+ # something still has to say what the pair amounted to.
96
+ rule: all_done
97
+ config:
98
+ input:
99
+ # Only the lenient step is read. The strict one failed, so it has no stored output,
100
+ # and a reference to it would settle this attempt as rejected.
101
+ child_status: "${steps.lenient.output.status}"
102
+ program: |
103
+ {child_status, note: "the same child status, two different step outcomes"}