dirigent-examples 0.15.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. dirigent_examples/__init__.py +22 -0
  2. dirigent_examples/py.typed +0 -0
  3. dirigent_examples/shelves/README.md +299 -0
  4. dirigent_examples/shelves/composition/README.md +18 -0
  5. dirigent_examples/shelves/composition/chained-instances.yaml +97 -0
  6. dirigent_examples/shelves/composition/composition-child.yaml +64 -0
  7. dirigent_examples/shelves/composition/composition-parent.yaml +89 -0
  8. dirigent_examples/shelves/connections.yaml +52 -0
  9. dirigent_examples/shelves/demo/README.md +19 -0
  10. dirigent_examples/shelves/demo/markdown-showcase.yaml +117 -0
  11. dirigent_examples/shelves/demo/params-showcase.yaml +92 -0
  12. dirigent_examples/shelves/demo/requires.yaml +65 -0
  13. dirigent_examples/shelves/demo/weekly-import-malawi.yaml +54 -0
  14. dirigent_examples/shelves/demo/weekly-import-nepal.yaml +75 -0
  15. dirigent_examples/shelves/docker/README.md +29 -0
  16. dirigent_examples/shelves/docker/docker-build-push.yaml +110 -0
  17. dirigent_examples/shelves/docker/docker-build-run.yaml +106 -0
  18. dirigent_examples/shelves/docker/docker-compose-database.yaml +124 -0
  19. dirigent_examples/shelves/docker/docker-compose-failing-up.yaml +83 -0
  20. dirigent_examples/shelves/docker/docker-compose-file.yaml +117 -0
  21. dirigent_examples/shelves/docker/docker-compose-profiles-env.yaml +133 -0
  22. dirigent_examples/shelves/docker/docker-compose-stack.yaml +70 -0
  23. dirigent_examples/shelves/docker/docker-hello.yaml +53 -0
  24. dirigent_examples/shelves/docker/docker-remote-daemon.yaml +92 -0
  25. dirigent_examples/shelves/docker/docker-run-failing-teardown.yaml +94 -0
  26. dirigent_examples/shelves/docker/docker-ticker.yaml +49 -0
  27. dirigent_examples/shelves/execute/README.md +16 -0
  28. dirigent_examples/shelves/execute/long-log.yaml +89 -0
  29. dirigent_examples/shelves/failure/README.md +20 -0
  30. dirigent_examples/shelves/failure/error-handler.yaml +89 -0
  31. dirigent_examples/shelves/failure/optional-step.yaml +82 -0
  32. dirigent_examples/shelves/failure/retries.yaml +75 -0
  33. dirigent_examples/shelves/failure/retry-budget.yaml +82 -0
  34. dirigent_examples/shelves/failure/step-timeout.yaml +96 -0
  35. dirigent_examples/shelves/git/README.md +32 -0
  36. dirigent_examples/shelves/git/git-checkout-build.yaml +125 -0
  37. dirigent_examples/shelves/git/git-checkout-compose.yaml +138 -0
  38. dirigent_examples/shelves/git/git-checkout-public.yaml +84 -0
  39. dirigent_examples/shelves/graph/README.md +22 -0
  40. dirigent_examples/shelves/graph/deep-chain.yaml +119 -0
  41. dirigent_examples/shelves/graph/fan-in.yaml +80 -0
  42. dirigent_examples/shelves/graph/fan-out.yaml +66 -0
  43. dirigent_examples/shelves/graph/linear.yaml +66 -0
  44. dirigent_examples/shelves/graph/parallel-branches.yaml +57 -0
  45. dirigent_examples/shelves/graph/parallel-sleep.yaml +55 -0
  46. dirigent_examples/shelves/graph/skip-diamond.yaml +105 -0
  47. dirigent_examples/shelves/graph/wide-fan.yaml +147 -0
  48. dirigent_examples/shelves/hello-world.yaml +30 -0
  49. dirigent_examples/shelves/open-data/README.md +67 -0
  50. dirigent_examples/shelves/open-data/feeds-composition.yaml +120 -0
  51. dirigent_examples/shelves/open-data/gdacs-disaster-updates.yaml +230 -0
  52. dirigent_examples/shelves/open-data/github-releases-relay.yaml +225 -0
  53. dirigent_examples/shelves/open-data/hdx-dataset-watch.yaml +211 -0
  54. dirigent_examples/shelves/open-data/kobo-submissions-to-csv.yaml +149 -0
  55. dirigent_examples/shelves/open-data/nominatim-geocode-facilities.yaml +169 -0
  56. dirigent_examples/shelves/open-data/odk-central-submissions.yaml +146 -0
  57. dirigent_examples/shelves/open-data/open-meteo-weekly-report.yaml +142 -0
  58. dirigent_examples/shelves/open-data/overpass-health-facilities.yaml +154 -0
  59. dirigent_examples/shelves/open-data/usgs-earthquakes-alert.yaml +208 -0
  60. dirigent_examples/shelves/open-data/who-gho-indicators-to-parquet.yaml +163 -0
  61. dirigent_examples/shelves/open-data/wikidata-country-reference.yaml +146 -0
  62. dirigent_examples/shelves/open-data/world-bank-population-trend.yaml +166 -0
  63. dirigent_examples/shelves/patterns/README.md +144 -0
  64. dirigent_examples/shelves/patterns/concurrency-queue.yaml +89 -0
  65. dirigent_examples/shelves/patterns/concurrency-replace.yaml +92 -0
  66. dirigent_examples/shelves/patterns/concurrency-skip.yaml +97 -0
  67. dirigent_examples/shelves/patterns/connections-referenced-vs-carried.yaml +140 -0
  68. dirigent_examples/shelves/patterns/deadline-on-a-sensor.yaml +106 -0
  69. dirigent_examples/shelves/patterns/fan-out-continue.yaml +88 -0
  70. dirigent_examples/shelves/patterns/fan-out-fail-fast.yaml +80 -0
  71. dirigent_examples/shelves/patterns/fan-out-from-params.yaml +84 -0
  72. dirigent_examples/shelves/patterns/fan-out-item-wise.yaml +111 -0
  73. dirigent_examples/shelves/patterns/fan-out-literal-list.yaml +81 -0
  74. dirigent_examples/shelves/patterns/fan-out-nested-objects.yaml +107 -0
  75. dirigent_examples/shelves/patterns/fan-out-then-join.yaml +86 -0
  76. dirigent_examples/shelves/patterns/log-levels.yaml +119 -0
  77. dirigent_examples/shelves/patterns/outputs-inline-vs-storage.yaml +140 -0
  78. dirigent_examples/shelves/patterns/params-every-type.yaml +259 -0
  79. dirigent_examples/shelves/patterns/params-validation-refuses.yaml +131 -0
  80. dirigent_examples/shelves/patterns/pipeline-run-child.yaml +96 -0
  81. dirigent_examples/shelves/patterns/pipeline-run-fire-and-forget.yaml +99 -0
  82. dirigent_examples/shelves/patterns/pipeline-run-strict.yaml +103 -0
  83. dirigent_examples/shelves/patterns/pipeline-run-wait.yaml +107 -0
  84. dirigent_examples/shelves/patterns/pipeline-run-with-params.yaml +124 -0
  85. dirigent_examples/shelves/patterns/poll-cadence.yaml +102 -0
  86. dirigent_examples/shelves/patterns/priority-layered.yaml +120 -0
  87. dirigent_examples/shelves/patterns/references-cheat-sheet.yaml +186 -0
  88. dirigent_examples/shelves/patterns/retry-budget-exhausted.yaml +92 -0
  89. dirigent_examples/shelves/patterns/retry-exponential-backoff.yaml +88 -0
  90. dirigent_examples/shelves/patterns/retry-only-transient.yaml +108 -0
  91. dirigent_examples/shelves/patterns/retry-with-jitter.yaml +102 -0
  92. dirigent_examples/shelves/patterns/rule-all-done.yaml +83 -0
  93. dirigent_examples/shelves/patterns/rule-all-success.yaml +79 -0
  94. dirigent_examples/shelves/patterns/rule-always.yaml +92 -0
  95. dirigent_examples/shelves/patterns/rule-one-failed.yaml +86 -0
  96. dirigent_examples/shelves/patterns/schedule-at-once.yaml +110 -0
  97. dirigent_examples/shelves/patterns/schedule-cron-timezone.yaml +121 -0
  98. dirigent_examples/shelves/patterns/schedule-interval.yaml +109 -0
  99. dirigent_examples/shelves/patterns/schedule-window-half-open.yaml +105 -0
  100. dirigent_examples/shelves/patterns/sensor-http-ready.yaml +121 -0
  101. dirigent_examples/shelves/patterns/sensor-storage-exists.yaml +134 -0
  102. dirigent_examples/shelves/patterns/step-names-and-keys.yaml +99 -0
  103. dirigent_examples/shelves/patterns/timeout-fails-the-step.yaml +94 -0
  104. dirigent_examples/shelves/patterns/timeout-skips-the-step.yaml +102 -0
  105. dirigent_examples/shelves/patterns/webhook-mapping-nested-payload.yaml +125 -0
  106. dirigent_examples/shelves/patterns/webhook-signed.yaml +144 -0
  107. dirigent_examples/shelves/preview/s3-parquet-to-ingestion.yaml +92 -0
  108. dirigent_examples/shelves/python/README.md +31 -0
  109. dirigent_examples/shelves/python/apply_and_run.py +52 -0
  110. dirigent_examples/shelves/python/ci_gate.py +76 -0
  111. dirigent_examples/shelves/python/connections.py +61 -0
  112. dirigent_examples/shelves/python/error_handling.py +84 -0
  113. dirigent_examples/shelves/python/follow_logs.py +39 -0
  114. dirigent_examples/shelves/python/list_and_filter.py +52 -0
  115. dirigent_examples/shelves/queues/README.md +59 -0
  116. dirigent_examples/shelves/queues/kafka-consume-then-transform.yaml +105 -0
  117. dirigent_examples/shelves/queues/kafka-produce-then-consume.yaml +124 -0
  118. dirigent_examples/shelves/queues/rabbitmq-consume-ack-on-success.yaml +102 -0
  119. dirigent_examples/shelves/queues/report-to-kafka.yaml +105 -0
  120. dirigent_examples/shelves/queues/report-to-rabbitmq.yaml +105 -0
  121. dirigent_examples/shelves/recipes/README.md +130 -0
  122. dirigent_examples/shelves/recipes/csv-header-rules.yaml +147 -0
  123. dirigent_examples/shelves/recipes/csv-to-ndjson.yaml +107 -0
  124. dirigent_examples/shelves/recipes/etl-csv-clean-validate-parquet.yaml +207 -0
  125. dirigent_examples/shelves/recipes/filter-by-predicate.yaml +116 -0
  126. dirigent_examples/shelves/recipes/filter-then-map-then-reduce.yaml +109 -0
  127. dirigent_examples/shelves/recipes/http-fetch-validate-post.yaml +142 -0
  128. dirigent_examples/shelves/recipes/http-follow-redirects.yaml +100 -0
  129. dirigent_examples/shelves/recipes/http-get-with-query.yaml +96 -0
  130. dirigent_examples/shelves/recipes/http-headers-and-auth-connection.yaml +110 -0
  131. dirigent_examples/shelves/recipes/http-post-file-from-storage.yaml +117 -0
  132. dirigent_examples/shelves/recipes/http-post-json-echo.yaml +105 -0
  133. dirigent_examples/shelves/recipes/http-post-report.yaml +183 -0
  134. dirigent_examples/shelves/recipes/http-save-body-to-storage.yaml +106 -0
  135. dirigent_examples/shelves/recipes/http-success-status-list.yaml +80 -0
  136. dirigent_examples/shelves/recipes/http-timeout-override.yaml +104 -0
  137. dirigent_examples/shelves/recipes/jq-dedupe-by-key.yaml +78 -0
  138. dirigent_examples/shelves/recipes/jq-defaults-and-nulls.yaml +91 -0
  139. dirigent_examples/shelves/recipes/jq-group-by-and-sum.yaml +74 -0
  140. dirigent_examples/shelves/recipes/jq-join-two-lists.yaml +77 -0
  141. dirigent_examples/shelves/recipes/jq-long-to-wide.yaml +76 -0
  142. dirigent_examples/shelves/recipes/jq-nested-to-flat.yaml +89 -0
  143. dirigent_examples/shelves/recipes/jq-pivot-wide-to-long.yaml +65 -0
  144. dirigent_examples/shelves/recipes/jq-running-totals.yaml +82 -0
  145. dirigent_examples/shelves/recipes/jq-string-cleaning.yaml +88 -0
  146. dirigent_examples/shelves/recipes/jq-top-n.yaml +85 -0
  147. dirigent_examples/shelves/recipes/jq-validate-in-jq-vs-schema.yaml +124 -0
  148. dirigent_examples/shelves/recipes/jq-window-dates.yaml +82 -0
  149. dirigent_examples/shelves/recipes/json-to-csv-flattening.yaml +141 -0
  150. dirigent_examples/shelves/recipes/large-output-to-storage.yaml +134 -0
  151. dirigent_examples/shelves/recipes/map-enrich-with-lookup.yaml +96 -0
  152. dirigent_examples/shelves/recipes/ndjson-to-parquet.yaml +130 -0
  153. dirigent_examples/shelves/recipes/pagination-by-fan-out.yaml +115 -0
  154. dirigent_examples/shelves/recipes/parquet-round-trip-types.yaml +163 -0
  155. dirigent_examples/shelves/recipes/reconcile-two-sources.yaml +159 -0
  156. dirigent_examples/shelves/recipes/report-built-in.yaml +72 -0
  157. dirigent_examples/shelves/recipes/report-daily-digest.yaml +186 -0
  158. dirigent_examples/shelves/recipes/report-to-file.yaml +131 -0
  159. dirigent_examples/shelves/recipes/report-to-webhook.yaml +136 -0
  160. dirigent_examples/shelves/recipes/schema-carried.yaml +112 -0
  161. dirigent_examples/shelves/recipes/schema-formats.yaml +107 -0
  162. dirigent_examples/shelves/recipes/schema-referenced.yaml +86 -0
  163. dirigent_examples/shelves/recipes/schema-refuses-then-rule.yaml +127 -0
  164. dirigent_examples/shelves/recipes/storage-copy-dated-archive.yaml +114 -0
  165. dirigent_examples/shelves/recipes/storage-exists-gate.yaml +127 -0
  166. dirigent_examples/shelves/recipes/storage-manifest-of-a-fan-out.yaml +104 -0
  167. dirigent_examples/shelves/recipes/storage-write-then-read.yaml +119 -0
  168. dirigent_examples/shelves/recipes/webhook-post-hmac.yaml +132 -0
  169. dirigent_examples/shelves/recipes/webhook-post-summary.yaml +142 -0
  170. dirigent_examples/shelves/s3/README.md +34 -0
  171. dirigent_examples/shelves/s3/report-to-s3.yaml +93 -0
  172. dirigent_examples/shelves/s3/s3-copy-and-verify.yaml +105 -0
  173. dirigent_examples/shelves/s3/s3-csv-report.yaml +87 -0
  174. dirigent_examples/shelves/s3/s3-parquet-report.yaml +77 -0
  175. dirigent_examples/shelves/s3/s3-round-trip.yaml +125 -0
  176. dirigent_examples/shelves/schemas/README.md +36 -0
  177. dirigent_examples/shelves/schemas/echo-reading.json +18 -0
  178. dirigent_examples/shelves/schemas/ou-record.json +13 -0
  179. dirigent_examples/shelves/schemas/station-reading.json +13 -0
  180. dirigent_examples/shelves/sensors/README.md +16 -0
  181. dirigent_examples/shelves/sensors/sensor-gate.yaml +65 -0
  182. dirigent_examples/shelves/sensors/time-window.yaml +61 -0
  183. dirigent_examples/shelves/sql/README.md +52 -0
  184. dirigent_examples/shelves/sql/duckdb-parquet-to-report.yaml +146 -0
  185. dirigent_examples/shelves/sql/sql-postgres-readonly.yaml +111 -0
  186. dirigent_examples/shelves/sql/sql-query-to-storage.yaml +85 -0
  187. dirigent_examples/shelves/sql/sql-sqlite-roundtrip.yaml +114 -0
  188. dirigent_examples/shelves/sql/warehouse.sql +42 -0
  189. dirigent_examples/shelves/transform/README.md +36 -0
  190. dirigent_examples/shelves/transform/csv-report.yaml +55 -0
  191. dirigent_examples/shelves/transform/jq-filter-and-map.yaml +70 -0
  192. dirigent_examples/shelves/transform/jq-group-and-aggregate.yaml +70 -0
  193. dirigent_examples/shelves/transform/jq-join-two-sources.yaml +98 -0
  194. dirigent_examples/shelves/transform/jq-reshape.yaml +91 -0
  195. dirigent_examples/shelves/transform/jq-stream-through-storage.yaml +112 -0
  196. dirigent_examples/shelves/transform/ndjson-round-trip.yaml +56 -0
  197. dirigent_examples/shelves/transform/parquet-round-trip.yaml +68 -0
  198. dirigent_examples/shelves/transform/std-convert-fan-out.yaml +142 -0
  199. dirigent_examples/shelves/transform/xml-feed-to-ndjson.yaml +116 -0
  200. dirigent_examples/shelves/transform/yaml-config-to-json.yaml +104 -0
  201. dirigent_examples/shelves/triggers/README.md +45 -0
  202. dirigent_examples/shelves/triggers/at-one-time.yaml +78 -0
  203. dirigent_examples/shelves/triggers/cron-nightly.yaml +79 -0
  204. dirigent_examples/shelves/triggers/cron-windowed.yaml +86 -0
  205. dirigent_examples/shelves/triggers/document-nightly.yaml +80 -0
  206. dirigent_examples/shelves/triggers/interval-rolling.yaml +88 -0
  207. dirigent_examples/shelves/triggers/managed-and-manual.yaml +109 -0
  208. dirigent_examples/shelves/triggers/webhook-trigger.yaml +75 -0
  209. dirigent_examples/shelves/validate/README.md +31 -0
  210. dirigent_examples/shelves/validate/expects-a-shape.yaml +56 -0
  211. dirigent_examples/shelves/validate/the-shape-is-wrong.yaml +46 -0
  212. dirigent_examples-0.15.0.dist-info/METADATA +21 -0
  213. dirigent_examples-0.15.0.dist-info/RECORD +216 -0
  214. dirigent_examples-0.15.0.dist-info/WHEEL +4 -0
  215. dirigent_examples-0.15.0.dist-info/entry_points.txt +3 -0
  216. dirigent_examples-0.15.0.dist-info/licenses/LICENSE +18 -0
@@ -0,0 +1,116 @@
1
+ # An XML feed read as records, one line each, and the mapping that makes it reversible.
2
+ #
3
+ # convert.std reads an element as an object: an attribute is a key prefixed with @, the text
4
+ # of an element with no children is the bare string, a child element is a key of its tag, and
5
+ # a tag that repeats is a list in document order. Nothing else fits in, so mixed content --
6
+ # text sitting beside child elements -- is refused naming the element rather than flattened.
7
+ #
8
+ # xml to ndjson is the direction that scales: the root's children are the records, read with
9
+ # iterparse and dropped as they are written, so a feed far larger than memory converts. It is
10
+ # one-way, because an ndjson line carries no root element name to write a document under;
11
+ # xml to json is the pair that round-trips, and this document runs both over the one feed
12
+ # object it stores first.
13
+ #
14
+ # A conversion names two uris and carries no value, so each direction here lands in the run's
15
+ # scratch space and a storage.read is what brings it back in: the ndjson as text a jq program
16
+ # splits, the json as the value it already is.
17
+ #
18
+ # Everything read out of an element is text, exactly as it is out of a csv: 4.5 is the three
19
+ # characters until a jq program says tonumber.
20
+ #
21
+ # dg run --local examples/transform/xml-feed-to-ndjson.yaml --keep
22
+
23
+ format: dirigent/v1
24
+ kind: pipeline
25
+ code: xml-feed-to-ndjson
26
+ name: An XML feed as records
27
+ description: |
28
+ Store an XML feed, convert it to one record per line, aggregate the records with jq, and
29
+ convert the same object to one whole value to show the mapping that round-trips.
30
+
31
+ The mapping, in four rules:
32
+
33
+ 1. an attribute is a key prefixed with `@`
34
+ 2. an element's text is the value where it has no children
35
+ 3. a child element is a key of its tag
36
+ 4. a tag that repeats is a list, in document order
37
+
38
+ tags: [transform, storage]
39
+
40
+ requires:
41
+ blocks:
42
+ - convert.std
43
+ - transform.jq
44
+ - storage.read
45
+ - storage.write
46
+
47
+ steps:
48
+ feed:
49
+ block: storage.write
50
+ config:
51
+ target: ${run.scratch}/feed.xml
52
+ # Written inline so the example needs no network. In a real pipeline this object is
53
+ # the feed an upstream http.request put in storage, and records starts there.
54
+ text: |
55
+ <?xml version="1.0" encoding="utf-8"?>
56
+ <feed source="stations">
57
+ <reading station="st-1"><region>east</region><celsius>4.5</celsius></reading>
58
+ <reading station="st-2"><region>east</region><celsius>-3.0</celsius></reading>
59
+ <reading station="st-3"><region>west</region><celsius>1.2</celsius></reading>
60
+ </feed>
61
+ content_type: application/xml
62
+
63
+ records:
64
+ block: convert.std
65
+ depends_on: [feed]
66
+ config:
67
+ source: ${steps.feed.output.uri}
68
+ target: ${run.scratch}/readings.ndjson
69
+ from: xml
70
+ # The records are the root's children, so each <reading> becomes one line. The root's
71
+ # own attribute, source="stations", is not on any of them: it belongs to the document,
72
+ # and the json direction below is where it survives.
73
+ to: ndjson
74
+
75
+ lines:
76
+ block: storage.read
77
+ depends_on: [records]
78
+ config:
79
+ source: ${steps.records.output.target}
80
+ # Nothing records what an .ndjson object is, and an object a read cannot name is
81
+ # refused rather than guessed at, so the step says it here.
82
+ content_type: application/x-ndjson
83
+
84
+ mean_by_region:
85
+ block: transform.jq
86
+ depends_on: [lines]
87
+ config:
88
+ # ndjson is one JSON value per line and is text rather than the JSON family, so the
89
+ # read handed over text and the program splits before it parses. tonumber for the one
90
+ # field it wants as a number.
91
+ input: ${steps.lines.output.text}
92
+ program: |
93
+ split("\n")
94
+ | map(select(length > 0) | fromjson)
95
+ | group_by(.region)
96
+ | [.[] | {region: .[0].region,
97
+ stations: [.[] | ."@station"],
98
+ mean_celsius: ([.[] | .celsius | tonumber] | add / length)}]
99
+
100
+ whole_document:
101
+ block: convert.std
102
+ depends_on: [feed]
103
+ config:
104
+ # The same feed the other way: one object, keyed by the root element's tag, carrying
105
+ # the document's own @source and the repeated readings as a list. This is the pair
106
+ # that round-trips -- json to xml writes this object back out as the feed.
107
+ source: ${steps.feed.output.uri}
108
+ target: ${run.scratch}/feed.json
109
+ from: xml
110
+ to: json
111
+
112
+ document:
113
+ block: storage.read
114
+ depends_on: [whole_document]
115
+ config:
116
+ source: ${steps.whole_document.output.target}
@@ -0,0 +1,104 @@
1
+ # A YAML document is one value, so a config file converts to the object it describes.
2
+ #
3
+ # This is the half of convert.std that is not the sequence of records: yaml and json trade
4
+ # one whole value in either direction, so a settings file becomes a JSON object a jq program
5
+ # can read, and a JSON value becomes a YAML document again. The stream spelling -- one `---`
6
+ # document per ndjson line -- is the other half, and examples/transform/ndjson-round-trip.yaml
7
+ # is where the sequence side lives.
8
+ #
9
+ # The config is an object in storage from the first hop, because a conversion reads a uri
10
+ # and writes a uri and never holds a value. storage.read is what brings the parsed config
11
+ # into the run for jq, and storage.write is what puts the answer back where the last
12
+ # conversion can pick it up.
13
+ #
14
+ # The one thing to know before pointing this at a real file: YAML says things JSON cannot.
15
+ # An unquoted 2026-03-01 is a date, not a string, and the codec refuses it naming its path
16
+ # rather than guessing what text it meant. That is why the retention date below is quoted.
17
+ #
18
+ # dg run --local examples/transform/yaml-config-to-json.yaml --keep
19
+
20
+ format: dirigent/v1
21
+ kind: pipeline
22
+ code: yaml-config-to-json
23
+ name: A config as an object
24
+ description: |
25
+ Read a stored YAML config as the JSON object it describes, take what a run needs out of
26
+ it, and write the answer back out as YAML.
27
+
28
+ `convert.std` reads `yaml` two ways. This document uses the first:
29
+
30
+ 1. one **document** is one value, traded with `json`
31
+ 2. one **stream** of documents is one line each, traded with `ndjson`
32
+
33
+ tags: [transform, storage]
34
+
35
+ requires:
36
+ blocks:
37
+ - convert.std
38
+ - transform.jq
39
+ - storage.read
40
+ - storage.write
41
+
42
+ steps:
43
+ config:
44
+ block: storage.write
45
+ config:
46
+ target: ${run.scratch}/settings.yaml
47
+ # Written inline so the example needs no network. In a real pipeline this object is
48
+ # the config a git.checkout or an http.request put in storage, and parse starts there.
49
+ text: |
50
+ name: daily-load
51
+ schedule: 0 3 * * *
52
+ # Quoted, because an unquoted date is a YAML date and JSON has no date: the codec
53
+ # refuses it at $.retention.before rather than deciding what text it meant.
54
+ retention:
55
+ before: "2026-03-01"
56
+ targets:
57
+ - code: warehouse
58
+ enabled: true
59
+ - code: archive
60
+ enabled: false
61
+ content_type: application/yaml
62
+
63
+ parse:
64
+ block: convert.std
65
+ depends_on: [config]
66
+ config:
67
+ source: ${steps.config.output.uri}
68
+ target: ${run.scratch}/settings.json
69
+ from: yaml
70
+ to: json
71
+
72
+ settings:
73
+ block: storage.read
74
+ depends_on: [parse]
75
+ config:
76
+ source: ${steps.parse.output.target}
77
+
78
+ enabled:
79
+ block: transform.jq
80
+ depends_on: [settings]
81
+ config:
82
+ # What the read handed over is the config as a value: keys are keys, and true is a
83
+ # boolean rather than the word.
84
+ input: ${steps.settings.output.value}
85
+ program: |
86
+ {name, schedule, before: .retention.before,
87
+ targets: [.targets[] | select(.enabled) | .code]}
88
+
89
+ store:
90
+ block: storage.write
91
+ depends_on: [enabled]
92
+ config:
93
+ target: ${run.scratch}/enabled.json
94
+ value: ${steps.enabled.output.value}
95
+
96
+ as_yaml:
97
+ block: convert.std
98
+ depends_on: [store]
99
+ config:
100
+ # Back the way it came: one JSON value, one YAML document.
101
+ source: ${steps.store.output.uri}
102
+ target: ${run.scratch}/enabled.yaml
103
+ from: json
104
+ to: yaml
@@ -0,0 +1,45 @@
1
+ # Trigger examples
2
+
3
+ What starts a run on its own: a clock, or an inbound webhook. A trigger is declared in the
4
+ document and materialised by `dg apply`; the scheduler fires the clocks and the server
5
+ answers the hooks, on the instance the document was applied to. Operational state -- paused,
6
+ last fired, the firing and delivery history -- stays in the instance, so none of it appears
7
+ in these files. [docs/design.md](../../docs/design.md#the-misfire-policy) is the page behind
8
+ the clocks.
9
+
10
+ Each file is prefixed with the clock kind it teaches, the way `transform/`'s files are
11
+ prefixed with the engine kind: `cron-` for a calendar moment, `interval-` for a rolling
12
+ cadence, `at-` for a single firing. Every one of them also runs ad hoc, with nothing on the
13
+ allowlist and no network:
14
+
15
+ ```bash
16
+ dg run --local examples/triggers/cron-nightly.yaml -p day=2026-01-01
17
+ ```
18
+
19
+ One of them reads the window its firing covers, which no document declares and only a run
20
+ carries, so running it ad hoc means saying which interval it is for:
21
+
22
+ ```bash
23
+ dg run --local examples/triggers/cron-windowed.yaml --window 2026-06-01..2026-06-02
24
+ ```
25
+
26
+ ## Pipelines
27
+
28
+ | File | What it teaches |
29
+ | --- | --- |
30
+ | [cron-nightly.yaml](cron-nightly.yaml) | A calendar moment: five in the morning in the schedule's own zone, across daylight saving, with parameters pinned over the pipeline's defaults. |
31
+ | [cron-windowed.yaml](cron-windowed.yaml) | The interval a firing covers rather than the moment it happened at: `${run.window.start}` and `${run.window.end}`, derived from the cadence and read like a parameter. |
32
+ | [interval-rolling.yaml](interval-rolling.yaml) | A rolling cadence: when an interval beats a cron, and the misfire grace that turns a weekend outage into one firing rather than sixty. |
33
+ | [at-one-time.yaml](at-one-time.yaml) | The one-time firing: the backfill and the launch, and how a moment written without an offset is read in the schedule's own zone. |
34
+ | [managed-and-manual.yaml](managed-and-manual.yaml) | Against a real instance: what an apply materialises, what it removes, and the hand-made schedule it leaves alone. |
35
+ | [document-nightly.yaml](document-nightly.yaml) | A `kind: triggers` document: clocks for a pipeline defined elsewhere, owning its own rows and refusing a pipeline no instance holds. |
36
+ | [webhook-trigger.yaml](webhook-trigger.yaml) | The inbound trigger: a token minted at apply, a strict payload-to-parameter mapping, and everything else in the POST ignored. |
37
+
38
+ A pipeline carries as many schedules as it needs, each with its own zone and its own
39
+ parameters: nightly against production and hourly against staging is two schedules on one
40
+ pipeline, never two pipelines.
41
+
42
+ Every file here but one is a pipeline document that carries its own clocks, which is the
43
+ primary form: one file is one deployable unit, and its digest versions the triggers with the
44
+ steps. `document-nightly.yaml` is the other form, for a pipeline somebody else's file
45
+ defines -- it names `managed-and-manual`, so read the two together.
@@ -0,0 +1,78 @@
1
+ # The third clock: one firing, at one instant, and then never again.
2
+ #
3
+ # The schedule fires only on an instance this document has been applied to. The same
4
+ # document runs ad hoc anywhere, with nothing on the allowlist and no network:
5
+ #
6
+ # dg run --local examples/triggers/at-one-time.yaml -p from=2025-01-01 -p to=2025-12-31
7
+ # dg apply examples/triggers/at-one-time.yaml # against an instance
8
+ # dg schedule list at-one-time
9
+ #
10
+ # This is the backfill and the launch: a job that has to happen once, at a moment agreed
11
+ # with somebody, and that nobody should have to be awake for. Declaring it in the document
12
+ # makes the moment reviewable in a pull request rather than typed into a terminal at night.
13
+
14
+ format: dirigent/v1
15
+ kind: pipeline
16
+ code: at-one-time
17
+ name: A single agreed instant
18
+ description: A one-time schedule firing a bounded backfill at a single agreed instant.
19
+
20
+ tags: [triggers, transform, schedule]
21
+
22
+ requires:
23
+ blocks:
24
+ - transform.jq
25
+
26
+ params:
27
+ type: object
28
+ properties:
29
+ from:
30
+ type: string
31
+ format: date
32
+ description: First day of the backfill window, inclusive.
33
+ default: "2026-01-01"
34
+ to:
35
+ type: string
36
+ format: date
37
+ description: Last day of the backfill window, inclusive.
38
+ default: "2026-01-07"
39
+ environment:
40
+ type: string
41
+ enum: [staging, production]
42
+ default: staging
43
+
44
+ steps:
45
+ window:
46
+ block: transform.jq
47
+ config:
48
+ input:
49
+ from: "${params.from}"
50
+ to: "${params.to}"
51
+ environment: "${params.environment}"
52
+ program: |
53
+ {environment, from, to, months: [range(1; 13)]}
54
+
55
+ backfill:
56
+ block: transform.jq
57
+ depends_on: [window]
58
+ config:
59
+ input: "${steps.window.output.value}"
60
+ program: |
61
+ {environment, backfilled: "\(.from)..\(.to)", batches: (.months | length)}
62
+
63
+ triggers:
64
+ schedules:
65
+ - code: launch-backfill
66
+ # A moment written without an offset is anchored in the zone this schedule declares,
67
+ # not in UTC and not in the host's local time: this is half past four in the morning
68
+ # in Oslo. Write the offset yourself -- 2027-01-06T03:30:00+00:00 -- and it is taken
69
+ # as given and the zone below is not consulted.
70
+ at: 2027-01-06 04:30:00
71
+ timezone: Europe/Oslo
72
+ # Once the instant has gone by there is no next firing, so the schedule pauses itself
73
+ # rather than being deleted: its parameters and its firing history stay readable with
74
+ # dg schedule firings at-one-time launch-backfill.
75
+ params:
76
+ from: "2025-01-01"
77
+ to: "2025-12-31"
78
+ environment: production
@@ -0,0 +1,79 @@
1
+ # A calendar moment: five in the morning, Oslo time, every day.
2
+ #
3
+ # The schedule fires only on an instance this document has been applied to. The same
4
+ # document runs ad hoc anywhere, with nothing on the allowlist and no network:
5
+ #
6
+ # dg run --local examples/triggers/cron-nightly.yaml -p day=2026-01-01
7
+ # dg apply examples/triggers/cron-nightly.yaml # against an instance
8
+ # dg schedule list cron-nightly
9
+ #
10
+ # The pipeline is deliberately trivial -- one jq program shaping a small report -- because
11
+ # what this file teaches is the clock above it, not the work below it.
12
+
13
+ format: dirigent/v1
14
+ kind: pipeline
15
+ code: cron-nightly
16
+ name: Nightly load
17
+ description: |
18
+ One pipeline on a nightly cron schedule, in its own timezone, with pinned parameters.
19
+
20
+ What this file teaches is the **clock**, not the work below it. The zone belongs
21
+ to the schedule rather than to whichever host runs the scheduler, so `0 5 * * *`
22
+ in `Europe/Oslo` stays five in the morning across a daylight-saving boundary.
23
+
24
+ tags: [triggers, transform, schedule]
25
+
26
+ requires:
27
+ blocks:
28
+ - transform.jq
29
+
30
+ params:
31
+ type: object
32
+ properties:
33
+ day:
34
+ type: string
35
+ format: date
36
+ description: The day being loaded, which the schedule pins for every firing.
37
+ default: "2026-01-01"
38
+ environment:
39
+ type: string
40
+ enum: [staging, production]
41
+ default: staging
42
+
43
+ steps:
44
+ plan:
45
+ block: transform.jq
46
+ config:
47
+ input:
48
+ day: "${params.day}"
49
+ environment: "${params.environment}"
50
+ program: |
51
+ {day, environment, datasets: ["cases", "climate"]}
52
+
53
+ report:
54
+ block: transform.jq
55
+ depends_on: [plan]
56
+ config:
57
+ input: "${steps.plan.output.value}"
58
+ program: |
59
+ {loaded: .day, environment: .environment, datasets: (.datasets | length)}
60
+
61
+ triggers:
62
+ schedules:
63
+ - code: nightly
64
+ # A trigger carries the same pair: a code that addresses it, and an optional
65
+ # name and description that only ever read.
66
+ name: Nightly, Oslo time
67
+ description: Fires the day's load once the upstream extract has settled.
68
+ cron: "0 5 * * *"
69
+ # The zone belongs to the schedule, not to the host the scheduler happens to run on.
70
+ # This is five in the morning in Oslo all year, so it stays five in the morning across
71
+ # a daylight-saving boundary rather than sliding to four or six.
72
+ timezone: Europe/Oslo
73
+ # Parameters pinned here override the pipeline's defaults for this schedule's firings
74
+ # only: environment defaults to staging above, and every nightly firing is production.
75
+ # A second schedule on the same pipeline pins its own, which is why nightly against
76
+ # production and hourly against staging is two schedules and not two pipelines.
77
+ params:
78
+ day: "2026-01-01"
79
+ environment: production
@@ -0,0 +1,86 @@
1
+ # The interval a firing covers, rather than the moment it happened at.
2
+ #
3
+ # A schedule answers "when does this start". A window answers "what does this run cover",
4
+ # which is the question a data pipeline actually has: the nightly load at 05:00 is meant to
5
+ # read yesterday, not this instant. Every schedule-fired run carries the interval that just
6
+ # closed -- ${run.window.start} up to but not including ${run.window.end} -- and a step reads
7
+ # it the way it reads a parameter.
8
+ #
9
+ # THE WINDOW IS A PROPERTY OF THE RUN, NOT OF THIS DOCUMENT. Nothing below declares it: the
10
+ # cadence above computes it. So an ad hoc run has to be given one, and a run that carries no
11
+ # window refuses the reference rather than resolving it to an empty string and quietly asking
12
+ # the upstream system for everything it has:
13
+ #
14
+ # dg run --local examples/triggers/cron-windowed.yaml --window 2026-06-01..2026-06-02
15
+ # dg apply examples/triggers/cron-windowed.yaml # against an instance
16
+ # dg backfill cron-windowed --schedule nightly \
17
+ # --from 2026-06-01T05:00:00Z --to 2026-06-08T05:00:00Z --dry-run
18
+ #
19
+ # The window derived from a cron cadence is wall-clock, not a fixed 24 hours: in Europe/Oslo
20
+ # the spring-forward night is 23 hours long and the fall-back night is 25, and the pair of
21
+ # instants below says so honestly rather than rounding it away.
22
+
23
+ format: dirigent/v1
24
+ kind: pipeline
25
+ code: cron-windowed
26
+ name: Windowed daily load
27
+ description: |
28
+ A nightly schedule whose runs read the **interval that just closed** rather than
29
+ the instant they started at.
30
+
31
+ `${run.window.start}` and `${run.window.end}` are ISO 8601 instants, half-open:
32
+ the start is included and the end is not, so two consecutive firings tile the
33
+ timeline without overlapping or leaving a gap.
34
+
35
+ tags: [triggers, transform, schedule]
36
+
37
+ requires:
38
+ blocks:
39
+ - transform.jq
40
+
41
+ params:
42
+ type: object
43
+ properties:
44
+ dataset:
45
+ type: string
46
+ description: Which dataset the load reads, which the schedule pins for every firing.
47
+ default: cases
48
+
49
+ steps:
50
+ # The query a real extractor would be handed. Half-open is the whole point of writing both
51
+ # ends: `>= from` and `< to` never double-counts a row that lands exactly on a boundary.
52
+ query:
53
+ block: transform.jq
54
+ config:
55
+ input:
56
+ dataset: "${params.dataset}"
57
+ from: "${run.window.start}"
58
+ to: "${run.window.end}"
59
+ program: |
60
+ {
61
+ dataset,
62
+ from,
63
+ to,
64
+ predicate: "observed_at >= \(.from) and observed_at < \(.to)"
65
+ }
66
+
67
+ receipt:
68
+ block: transform.jq
69
+ depends_on: [query]
70
+ config:
71
+ input: "${steps.query.output.value}"
72
+ program: |
73
+ {loaded: .dataset, covering: "\(.from)..\(.to)"}
74
+
75
+ triggers:
76
+ schedules:
77
+ - code: nightly
78
+ name: Nightly, Oslo time
79
+ description: Reads the day that closed at five this morning.
80
+ cron: "0 5 * * *"
81
+ # The zone belongs to the schedule, so the window it derives is a day of Oslo's clock
82
+ # and not a fixed number of hours. This is what makes the daylight-saving nights come
83
+ # out right without anything in the steps below knowing that daylight saving exists.
84
+ timezone: Europe/Oslo
85
+ params:
86
+ dataset: cases
@@ -0,0 +1,80 @@
1
+ # Clocks for a pipeline defined somewhere else: the ops team's clock file.
2
+ #
3
+ # A kind: triggers document declares nothing but schedules and webhooks, and names the
4
+ # pipeline they fire. The pipeline itself lives in managed-and-manual.yaml, owned by
5
+ # whoever writes pipelines; this file is owned by whoever decides when they run. Two files,
6
+ # two review cycles, one instance.
7
+ #
8
+ # Applying it does exactly what a pipeline document's triggers: section does, with one
9
+ # difference: the rows it creates are owned by THIS document rather than by the pipeline's.
10
+ # So an apply here creates, redeclares and retires its own schedules and its own webhook,
11
+ # and never touches the two schedules and the webhook managed-and-manual.yaml declares, nor
12
+ # anything an operator made by hand.
13
+ #
14
+ # It refuses three things, and says which at apply:
15
+ #
16
+ # - a pipeline: that no instance holds, or that is deactivated. Apply the pipeline first;
17
+ # a directory apply does that for you, because pipelines go before triggers documents.
18
+ # - a schedule or webhook code the pipeline already carries under a different owner, and
19
+ # names the owner in the refusal.
20
+ # - pinned params: the pipeline's current parameter schema would reject, checked here at
21
+ # apply and again at every firing.
22
+ #
23
+ # Try it against a running instance, in this order:
24
+ #
25
+ # dg apply examples/triggers/managed-and-manual.yaml # the pipeline, first
26
+ # dg apply examples/triggers/document-nightly.yaml # its clocks, second
27
+ # dg schedule list managed-and-manual # five now: two inline, three here
28
+ # dg trigger-document show document-nightly # what this file owns
29
+ # dg trigger-document delete document-nightly # its three go; the inline two stay
30
+ #
31
+ # There is nothing to run: this document has no steps. `dg run --local` on it refuses and
32
+ # says to run the pipeline it names.
33
+
34
+ format: dirigent/v1
35
+ kind: triggers
36
+ code: document-nightly
37
+ name: Nightly batch clocks
38
+ description: Schedules and a webhook for a pipeline another document defines.
39
+
40
+ pipeline: managed-and-manual
41
+
42
+ triggers:
43
+ schedules:
44
+ # Four in the morning Oslo time, an hour ahead of the pipeline's own nightly schedule:
45
+ # this is the operations copy that loads yesterday against production, and it is a
46
+ # separate code so both can exist without either owner editing the other's file.
47
+ - code: ops-nightly
48
+ cron: "0 4 * * *"
49
+ timezone: Europe/Oslo
50
+ params:
51
+ day: "2026-01-01"
52
+ environment: production
53
+
54
+ # Monday mornings only, for the weekly reconciliation the same pipeline serves. A cron
55
+ # expression says this in one line; an interval could not, because a week of hours
56
+ # drifts across daylight saving and Monday does not.
57
+ - code: ops-weekly
58
+ cron: "0 6 * * 1"
59
+ timezone: Europe/Oslo
60
+ params:
61
+ day: "2026-01-01"
62
+ environment: production
63
+
64
+ # A rolling smoke test against staging, on an interval rather than a calendar: nothing
65
+ # about it needs to land at a particular wall-clock time, and an interval keeps firing
66
+ # at a fixed spacing however the clocks move.
67
+ - code: ops-staging-smoke
68
+ interval: 6h
69
+ params:
70
+ day: "2026-01-01"
71
+ environment: staging
72
+
73
+ webhooks:
74
+ # The operations team's own endpoint, beside the pipeline's. Its token is minted by the
75
+ # instance and shown once, and deleting this document deletes the endpoint with it --
76
+ # which is the point of putting it here rather than in the pipeline's own file.
77
+ - code: ops-manual-kick
78
+ params_from_payload:
79
+ day: "$.run.date"
80
+ environment: "$.run.environment"
@@ -0,0 +1,88 @@
1
+ # A rolling cadence: every hour from whenever it was declared, not at any particular time.
2
+ #
3
+ # The schedule fires only on an instance this document has been applied to. The same
4
+ # document runs ad hoc anywhere, with nothing on the allowlist and no network:
5
+ #
6
+ # dg run --local examples/triggers/interval-rolling.yaml
7
+ # dg apply examples/triggers/interval-rolling.yaml # against an instance
8
+ # dg schedule list interval-rolling
9
+ #
10
+ # An interval beats a cron when what you want is a cadence -- freshness, a poll, a rolling
11
+ # window -- and nobody cares which minute of the hour it lands on. A cron beats an interval
12
+ # when the firing is a calendar moment someone downstream expects: the nightly load, the
13
+ # Monday report, the close of the month. Write "every 1h" as an interval and "at 05:00 Oslo"
14
+ # as a cron, and neither has to fake the other.
15
+
16
+ format: dirigent/v1
17
+ kind: pipeline
18
+ code: interval-rolling
19
+ name: A rolling interval
20
+ description: A pipeline on a rolling hourly interval, refreshing a window rather than hitting a calendar moment.
21
+
22
+ # Two firings of a rolling refresh are not worth having at once: if one is still going when
23
+ # the next hour comes round, skip that firing rather than stack a second run on it.
24
+
25
+ tags: [triggers, transform, schedule]
26
+
27
+ concurrency: skip
28
+
29
+ requires:
30
+ blocks:
31
+ - transform.jq
32
+
33
+ params:
34
+ type: object
35
+ properties:
36
+ window_hours:
37
+ type: integer
38
+ default: 24
39
+ minimum: 1
40
+ maximum: 168
41
+ environment:
42
+ type: string
43
+ enum: [staging, production]
44
+ default: staging
45
+
46
+ steps:
47
+ window:
48
+ block: transform.jq
49
+ config:
50
+ input:
51
+ hours: "${params.window_hours}"
52
+ environment: "${params.environment}"
53
+ program: |
54
+ {environment, hours, buckets: [range(0; .hours; 6)]}
55
+
56
+ refresh:
57
+ block: transform.jq
58
+ depends_on: [window]
59
+ config:
60
+ input: "${steps.window.output.value}"
61
+ program: |
62
+ {environment, refreshed_hours: .hours, batches: (.buckets | length)}
63
+
64
+ triggers:
65
+ schedules:
66
+ - code: hourly-refresh
67
+ interval: 1h
68
+ # A duration is the same length in every zone, so an interval never consults the
69
+ # calendar and the zone here is read by nothing. It is the cron and at clocks that
70
+ # need one.
71
+ timezone: UTC
72
+ params:
73
+ window_hours: 24
74
+ environment: staging
75
+
76
+ # The misfire policy is what keeps an hourly schedule from becoming a catchup storm.
77
+ # A firing late by less than the grace (scheduler_misfire_grace, five minutes by
78
+ # default) is ordinary: the clock advances from the slot it owed, so the cadence stays
79
+ # anchored where it was rather than drifting by the length of the tick or by how long a
80
+ # run took. A firing late by more than the grace is a misfire: it fires exactly once and
81
+ # the next firing is computed from now, abandoning every slot that was missed. A
82
+ # scheduler down over a weekend therefore wakes up and runs this once, not sixty times.
83
+ - code: six-hourly-production
84
+ interval: 6h
85
+ timezone: UTC
86
+ params:
87
+ window_hours: 168
88
+ environment: production