dirigent-examples 0.15.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. dirigent_examples/__init__.py +22 -0
  2. dirigent_examples/py.typed +0 -0
  3. dirigent_examples/shelves/README.md +299 -0
  4. dirigent_examples/shelves/composition/README.md +18 -0
  5. dirigent_examples/shelves/composition/chained-instances.yaml +97 -0
  6. dirigent_examples/shelves/composition/composition-child.yaml +64 -0
  7. dirigent_examples/shelves/composition/composition-parent.yaml +89 -0
  8. dirigent_examples/shelves/connections.yaml +52 -0
  9. dirigent_examples/shelves/demo/README.md +19 -0
  10. dirigent_examples/shelves/demo/markdown-showcase.yaml +117 -0
  11. dirigent_examples/shelves/demo/params-showcase.yaml +92 -0
  12. dirigent_examples/shelves/demo/requires.yaml +65 -0
  13. dirigent_examples/shelves/demo/weekly-import-malawi.yaml +54 -0
  14. dirigent_examples/shelves/demo/weekly-import-nepal.yaml +75 -0
  15. dirigent_examples/shelves/docker/README.md +29 -0
  16. dirigent_examples/shelves/docker/docker-build-push.yaml +110 -0
  17. dirigent_examples/shelves/docker/docker-build-run.yaml +106 -0
  18. dirigent_examples/shelves/docker/docker-compose-database.yaml +124 -0
  19. dirigent_examples/shelves/docker/docker-compose-failing-up.yaml +83 -0
  20. dirigent_examples/shelves/docker/docker-compose-file.yaml +117 -0
  21. dirigent_examples/shelves/docker/docker-compose-profiles-env.yaml +133 -0
  22. dirigent_examples/shelves/docker/docker-compose-stack.yaml +70 -0
  23. dirigent_examples/shelves/docker/docker-hello.yaml +53 -0
  24. dirigent_examples/shelves/docker/docker-remote-daemon.yaml +92 -0
  25. dirigent_examples/shelves/docker/docker-run-failing-teardown.yaml +94 -0
  26. dirigent_examples/shelves/docker/docker-ticker.yaml +49 -0
  27. dirigent_examples/shelves/execute/README.md +16 -0
  28. dirigent_examples/shelves/execute/long-log.yaml +89 -0
  29. dirigent_examples/shelves/failure/README.md +20 -0
  30. dirigent_examples/shelves/failure/error-handler.yaml +89 -0
  31. dirigent_examples/shelves/failure/optional-step.yaml +82 -0
  32. dirigent_examples/shelves/failure/retries.yaml +75 -0
  33. dirigent_examples/shelves/failure/retry-budget.yaml +82 -0
  34. dirigent_examples/shelves/failure/step-timeout.yaml +96 -0
  35. dirigent_examples/shelves/git/README.md +32 -0
  36. dirigent_examples/shelves/git/git-checkout-build.yaml +125 -0
  37. dirigent_examples/shelves/git/git-checkout-compose.yaml +138 -0
  38. dirigent_examples/shelves/git/git-checkout-public.yaml +84 -0
  39. dirigent_examples/shelves/graph/README.md +22 -0
  40. dirigent_examples/shelves/graph/deep-chain.yaml +119 -0
  41. dirigent_examples/shelves/graph/fan-in.yaml +80 -0
  42. dirigent_examples/shelves/graph/fan-out.yaml +66 -0
  43. dirigent_examples/shelves/graph/linear.yaml +66 -0
  44. dirigent_examples/shelves/graph/parallel-branches.yaml +57 -0
  45. dirigent_examples/shelves/graph/parallel-sleep.yaml +55 -0
  46. dirigent_examples/shelves/graph/skip-diamond.yaml +105 -0
  47. dirigent_examples/shelves/graph/wide-fan.yaml +147 -0
  48. dirigent_examples/shelves/hello-world.yaml +30 -0
  49. dirigent_examples/shelves/open-data/README.md +67 -0
  50. dirigent_examples/shelves/open-data/feeds-composition.yaml +120 -0
  51. dirigent_examples/shelves/open-data/gdacs-disaster-updates.yaml +230 -0
  52. dirigent_examples/shelves/open-data/github-releases-relay.yaml +225 -0
  53. dirigent_examples/shelves/open-data/hdx-dataset-watch.yaml +211 -0
  54. dirigent_examples/shelves/open-data/kobo-submissions-to-csv.yaml +149 -0
  55. dirigent_examples/shelves/open-data/nominatim-geocode-facilities.yaml +169 -0
  56. dirigent_examples/shelves/open-data/odk-central-submissions.yaml +146 -0
  57. dirigent_examples/shelves/open-data/open-meteo-weekly-report.yaml +142 -0
  58. dirigent_examples/shelves/open-data/overpass-health-facilities.yaml +154 -0
  59. dirigent_examples/shelves/open-data/usgs-earthquakes-alert.yaml +208 -0
  60. dirigent_examples/shelves/open-data/who-gho-indicators-to-parquet.yaml +163 -0
  61. dirigent_examples/shelves/open-data/wikidata-country-reference.yaml +146 -0
  62. dirigent_examples/shelves/open-data/world-bank-population-trend.yaml +166 -0
  63. dirigent_examples/shelves/patterns/README.md +144 -0
  64. dirigent_examples/shelves/patterns/concurrency-queue.yaml +89 -0
  65. dirigent_examples/shelves/patterns/concurrency-replace.yaml +92 -0
  66. dirigent_examples/shelves/patterns/concurrency-skip.yaml +97 -0
  67. dirigent_examples/shelves/patterns/connections-referenced-vs-carried.yaml +140 -0
  68. dirigent_examples/shelves/patterns/deadline-on-a-sensor.yaml +106 -0
  69. dirigent_examples/shelves/patterns/fan-out-continue.yaml +88 -0
  70. dirigent_examples/shelves/patterns/fan-out-fail-fast.yaml +80 -0
  71. dirigent_examples/shelves/patterns/fan-out-from-params.yaml +84 -0
  72. dirigent_examples/shelves/patterns/fan-out-item-wise.yaml +111 -0
  73. dirigent_examples/shelves/patterns/fan-out-literal-list.yaml +81 -0
  74. dirigent_examples/shelves/patterns/fan-out-nested-objects.yaml +107 -0
  75. dirigent_examples/shelves/patterns/fan-out-then-join.yaml +86 -0
  76. dirigent_examples/shelves/patterns/log-levels.yaml +119 -0
  77. dirigent_examples/shelves/patterns/outputs-inline-vs-storage.yaml +140 -0
  78. dirigent_examples/shelves/patterns/params-every-type.yaml +259 -0
  79. dirigent_examples/shelves/patterns/params-validation-refuses.yaml +131 -0
  80. dirigent_examples/shelves/patterns/pipeline-run-child.yaml +96 -0
  81. dirigent_examples/shelves/patterns/pipeline-run-fire-and-forget.yaml +99 -0
  82. dirigent_examples/shelves/patterns/pipeline-run-strict.yaml +103 -0
  83. dirigent_examples/shelves/patterns/pipeline-run-wait.yaml +107 -0
  84. dirigent_examples/shelves/patterns/pipeline-run-with-params.yaml +124 -0
  85. dirigent_examples/shelves/patterns/poll-cadence.yaml +102 -0
  86. dirigent_examples/shelves/patterns/priority-layered.yaml +120 -0
  87. dirigent_examples/shelves/patterns/references-cheat-sheet.yaml +186 -0
  88. dirigent_examples/shelves/patterns/retry-budget-exhausted.yaml +92 -0
  89. dirigent_examples/shelves/patterns/retry-exponential-backoff.yaml +88 -0
  90. dirigent_examples/shelves/patterns/retry-only-transient.yaml +108 -0
  91. dirigent_examples/shelves/patterns/retry-with-jitter.yaml +102 -0
  92. dirigent_examples/shelves/patterns/rule-all-done.yaml +83 -0
  93. dirigent_examples/shelves/patterns/rule-all-success.yaml +79 -0
  94. dirigent_examples/shelves/patterns/rule-always.yaml +92 -0
  95. dirigent_examples/shelves/patterns/rule-one-failed.yaml +86 -0
  96. dirigent_examples/shelves/patterns/schedule-at-once.yaml +110 -0
  97. dirigent_examples/shelves/patterns/schedule-cron-timezone.yaml +121 -0
  98. dirigent_examples/shelves/patterns/schedule-interval.yaml +109 -0
  99. dirigent_examples/shelves/patterns/schedule-window-half-open.yaml +105 -0
  100. dirigent_examples/shelves/patterns/sensor-http-ready.yaml +121 -0
  101. dirigent_examples/shelves/patterns/sensor-storage-exists.yaml +134 -0
  102. dirigent_examples/shelves/patterns/step-names-and-keys.yaml +99 -0
  103. dirigent_examples/shelves/patterns/timeout-fails-the-step.yaml +94 -0
  104. dirigent_examples/shelves/patterns/timeout-skips-the-step.yaml +102 -0
  105. dirigent_examples/shelves/patterns/webhook-mapping-nested-payload.yaml +125 -0
  106. dirigent_examples/shelves/patterns/webhook-signed.yaml +144 -0
  107. dirigent_examples/shelves/preview/s3-parquet-to-ingestion.yaml +92 -0
  108. dirigent_examples/shelves/python/README.md +31 -0
  109. dirigent_examples/shelves/python/apply_and_run.py +52 -0
  110. dirigent_examples/shelves/python/ci_gate.py +76 -0
  111. dirigent_examples/shelves/python/connections.py +61 -0
  112. dirigent_examples/shelves/python/error_handling.py +84 -0
  113. dirigent_examples/shelves/python/follow_logs.py +39 -0
  114. dirigent_examples/shelves/python/list_and_filter.py +52 -0
  115. dirigent_examples/shelves/queues/README.md +59 -0
  116. dirigent_examples/shelves/queues/kafka-consume-then-transform.yaml +105 -0
  117. dirigent_examples/shelves/queues/kafka-produce-then-consume.yaml +124 -0
  118. dirigent_examples/shelves/queues/rabbitmq-consume-ack-on-success.yaml +102 -0
  119. dirigent_examples/shelves/queues/report-to-kafka.yaml +105 -0
  120. dirigent_examples/shelves/queues/report-to-rabbitmq.yaml +105 -0
  121. dirigent_examples/shelves/recipes/README.md +130 -0
  122. dirigent_examples/shelves/recipes/csv-header-rules.yaml +147 -0
  123. dirigent_examples/shelves/recipes/csv-to-ndjson.yaml +107 -0
  124. dirigent_examples/shelves/recipes/etl-csv-clean-validate-parquet.yaml +207 -0
  125. dirigent_examples/shelves/recipes/filter-by-predicate.yaml +116 -0
  126. dirigent_examples/shelves/recipes/filter-then-map-then-reduce.yaml +109 -0
  127. dirigent_examples/shelves/recipes/http-fetch-validate-post.yaml +142 -0
  128. dirigent_examples/shelves/recipes/http-follow-redirects.yaml +100 -0
  129. dirigent_examples/shelves/recipes/http-get-with-query.yaml +96 -0
  130. dirigent_examples/shelves/recipes/http-headers-and-auth-connection.yaml +110 -0
  131. dirigent_examples/shelves/recipes/http-post-file-from-storage.yaml +117 -0
  132. dirigent_examples/shelves/recipes/http-post-json-echo.yaml +105 -0
  133. dirigent_examples/shelves/recipes/http-post-report.yaml +183 -0
  134. dirigent_examples/shelves/recipes/http-save-body-to-storage.yaml +106 -0
  135. dirigent_examples/shelves/recipes/http-success-status-list.yaml +80 -0
  136. dirigent_examples/shelves/recipes/http-timeout-override.yaml +104 -0
  137. dirigent_examples/shelves/recipes/jq-dedupe-by-key.yaml +78 -0
  138. dirigent_examples/shelves/recipes/jq-defaults-and-nulls.yaml +91 -0
  139. dirigent_examples/shelves/recipes/jq-group-by-and-sum.yaml +74 -0
  140. dirigent_examples/shelves/recipes/jq-join-two-lists.yaml +77 -0
  141. dirigent_examples/shelves/recipes/jq-long-to-wide.yaml +76 -0
  142. dirigent_examples/shelves/recipes/jq-nested-to-flat.yaml +89 -0
  143. dirigent_examples/shelves/recipes/jq-pivot-wide-to-long.yaml +65 -0
  144. dirigent_examples/shelves/recipes/jq-running-totals.yaml +82 -0
  145. dirigent_examples/shelves/recipes/jq-string-cleaning.yaml +88 -0
  146. dirigent_examples/shelves/recipes/jq-top-n.yaml +85 -0
  147. dirigent_examples/shelves/recipes/jq-validate-in-jq-vs-schema.yaml +124 -0
  148. dirigent_examples/shelves/recipes/jq-window-dates.yaml +82 -0
  149. dirigent_examples/shelves/recipes/json-to-csv-flattening.yaml +141 -0
  150. dirigent_examples/shelves/recipes/large-output-to-storage.yaml +134 -0
  151. dirigent_examples/shelves/recipes/map-enrich-with-lookup.yaml +96 -0
  152. dirigent_examples/shelves/recipes/ndjson-to-parquet.yaml +130 -0
  153. dirigent_examples/shelves/recipes/pagination-by-fan-out.yaml +115 -0
  154. dirigent_examples/shelves/recipes/parquet-round-trip-types.yaml +163 -0
  155. dirigent_examples/shelves/recipes/reconcile-two-sources.yaml +159 -0
  156. dirigent_examples/shelves/recipes/report-built-in.yaml +72 -0
  157. dirigent_examples/shelves/recipes/report-daily-digest.yaml +186 -0
  158. dirigent_examples/shelves/recipes/report-to-file.yaml +131 -0
  159. dirigent_examples/shelves/recipes/report-to-webhook.yaml +136 -0
  160. dirigent_examples/shelves/recipes/schema-carried.yaml +112 -0
  161. dirigent_examples/shelves/recipes/schema-formats.yaml +107 -0
  162. dirigent_examples/shelves/recipes/schema-referenced.yaml +86 -0
  163. dirigent_examples/shelves/recipes/schema-refuses-then-rule.yaml +127 -0
  164. dirigent_examples/shelves/recipes/storage-copy-dated-archive.yaml +114 -0
  165. dirigent_examples/shelves/recipes/storage-exists-gate.yaml +127 -0
  166. dirigent_examples/shelves/recipes/storage-manifest-of-a-fan-out.yaml +104 -0
  167. dirigent_examples/shelves/recipes/storage-write-then-read.yaml +119 -0
  168. dirigent_examples/shelves/recipes/webhook-post-hmac.yaml +132 -0
  169. dirigent_examples/shelves/recipes/webhook-post-summary.yaml +142 -0
  170. dirigent_examples/shelves/s3/README.md +34 -0
  171. dirigent_examples/shelves/s3/report-to-s3.yaml +93 -0
  172. dirigent_examples/shelves/s3/s3-copy-and-verify.yaml +105 -0
  173. dirigent_examples/shelves/s3/s3-csv-report.yaml +87 -0
  174. dirigent_examples/shelves/s3/s3-parquet-report.yaml +77 -0
  175. dirigent_examples/shelves/s3/s3-round-trip.yaml +125 -0
  176. dirigent_examples/shelves/schemas/README.md +36 -0
  177. dirigent_examples/shelves/schemas/echo-reading.json +18 -0
  178. dirigent_examples/shelves/schemas/ou-record.json +13 -0
  179. dirigent_examples/shelves/schemas/station-reading.json +13 -0
  180. dirigent_examples/shelves/sensors/README.md +16 -0
  181. dirigent_examples/shelves/sensors/sensor-gate.yaml +65 -0
  182. dirigent_examples/shelves/sensors/time-window.yaml +61 -0
  183. dirigent_examples/shelves/sql/README.md +52 -0
  184. dirigent_examples/shelves/sql/duckdb-parquet-to-report.yaml +146 -0
  185. dirigent_examples/shelves/sql/sql-postgres-readonly.yaml +111 -0
  186. dirigent_examples/shelves/sql/sql-query-to-storage.yaml +85 -0
  187. dirigent_examples/shelves/sql/sql-sqlite-roundtrip.yaml +114 -0
  188. dirigent_examples/shelves/sql/warehouse.sql +42 -0
  189. dirigent_examples/shelves/transform/README.md +36 -0
  190. dirigent_examples/shelves/transform/csv-report.yaml +55 -0
  191. dirigent_examples/shelves/transform/jq-filter-and-map.yaml +70 -0
  192. dirigent_examples/shelves/transform/jq-group-and-aggregate.yaml +70 -0
  193. dirigent_examples/shelves/transform/jq-join-two-sources.yaml +98 -0
  194. dirigent_examples/shelves/transform/jq-reshape.yaml +91 -0
  195. dirigent_examples/shelves/transform/jq-stream-through-storage.yaml +112 -0
  196. dirigent_examples/shelves/transform/ndjson-round-trip.yaml +56 -0
  197. dirigent_examples/shelves/transform/parquet-round-trip.yaml +68 -0
  198. dirigent_examples/shelves/transform/std-convert-fan-out.yaml +142 -0
  199. dirigent_examples/shelves/transform/xml-feed-to-ndjson.yaml +116 -0
  200. dirigent_examples/shelves/transform/yaml-config-to-json.yaml +104 -0
  201. dirigent_examples/shelves/triggers/README.md +45 -0
  202. dirigent_examples/shelves/triggers/at-one-time.yaml +78 -0
  203. dirigent_examples/shelves/triggers/cron-nightly.yaml +79 -0
  204. dirigent_examples/shelves/triggers/cron-windowed.yaml +86 -0
  205. dirigent_examples/shelves/triggers/document-nightly.yaml +80 -0
  206. dirigent_examples/shelves/triggers/interval-rolling.yaml +88 -0
  207. dirigent_examples/shelves/triggers/managed-and-manual.yaml +109 -0
  208. dirigent_examples/shelves/triggers/webhook-trigger.yaml +75 -0
  209. dirigent_examples/shelves/validate/README.md +31 -0
  210. dirigent_examples/shelves/validate/expects-a-shape.yaml +56 -0
  211. dirigent_examples/shelves/validate/the-shape-is-wrong.yaml +46 -0
  212. dirigent_examples-0.15.0.dist-info/METADATA +21 -0
  213. dirigent_examples-0.15.0.dist-info/RECORD +216 -0
  214. dirigent_examples-0.15.0.dist-info/WHEEL +4 -0
  215. dirigent_examples-0.15.0.dist-info/entry_points.txt +3 -0
  216. dirigent_examples-0.15.0.dist-info/licenses/LICENSE +18 -0
@@ -0,0 +1,124 @@
1
+ # A database stack brought up, driven with real SQL, and torn down whatever happened.
2
+ #
3
+ # REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
4
+ # tcp+TLS, or the local socket in dev). docs/docker.md is the family's home.
5
+ #
6
+ # Every block here declares local_execution, so the engine refuses them unless the instance
7
+ # allowlists their ids:
8
+ # dg run --local examples/docker/docker-compose-database.yaml \
9
+ # --enable-unsafe docker.compose.up,docker.compose.down,docker.run
10
+ # export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up","docker.compose.down","docker.run"]'
11
+ #
12
+ # Three hops, and what each one hands on:
13
+ # stack one postgres service, brought up detached and left running for the rest of the
14
+ # run. Output: every service's container with its state and health, the networks
15
+ # they sit on, and default_network.
16
+ # query a psql container joined to that network, which creates a table, inserts two
17
+ # rows and selects them back. Output: the exit code and the stdout those selects
18
+ # printed, which is the step's result a downstream step would read.
19
+ # teardown the same project down again, volumes included.
20
+ #
21
+ # WAIT IS WHAT TURNS A HEALTHCHECK INTO A GATE. Postgres accepts connections some seconds
22
+ # after its container starts, and a compose file's healthcheck is only an opinion the daemon
23
+ # records unless something waits on it. wait: true holds the up step open until every service
24
+ # is running and healthy, so the query step meets a database that is ready rather than
25
+ # racing it and failing on connection refused. Without it the stack step returns the moment
26
+ # compose has started the containers.
27
+ #
28
+ # THE NETWORK NAME COMES FROM THE UP STEP. Compose puts the project on a network of its own
29
+ # and the up step reports it as default_network; the query step names that as its network
30
+ # mode, which is what lets it resolve the service by the name the compose file gave it -- db.
31
+ #
32
+ # THE TEARDOWN RUNS ON ANY OUTCOME. rule: all_done means the down step runs whether the query
33
+ # passed or failed, so a database is never left running past the run that started it. Both
34
+ # compose steps default their project name from the run id, so the down addresses exactly the
35
+ # project the up created with nothing wired between them.
36
+ #
37
+ # TO MAKE IT YOURS: put your own schema and queries in the psql argv, or swap postgres for
38
+ # whatever your pipeline actually talks to; the shape -- up with wait, drive, down with
39
+ # all_done -- does not change.
40
+
41
+ format: dirigent/v1
42
+ kind: pipeline
43
+ code: docker-compose-database
44
+ name: A database stack, queried and torn down
45
+ description: Bring PostgreSQL up behind its healthcheck, run SQL against it, and remove it with its volumes.
46
+
47
+ tags: [docker, execute]
48
+
49
+ requires:
50
+ blocks:
51
+ - docker.compose.up
52
+ - docker.compose.down
53
+ - docker.run
54
+
55
+ steps:
56
+ stack:
57
+ block: docker.compose.up
58
+ deadline: 10m
59
+ config:
60
+ # Hold the step open until pg_isready succeeds, and give up after two minutes rather
61
+ # than sitting on the step's whole deadline if the image never comes up at all.
62
+ wait: true
63
+ wait_timeout: 2m
64
+ pull: always
65
+ content: |
66
+ services:
67
+ db:
68
+ image: postgres:17-alpine
69
+ environment:
70
+ POSTGRES_USER: dirigent
71
+ POSTGRES_PASSWORD: dirigent
72
+ POSTGRES_DB: dirigent
73
+ healthcheck:
74
+ test: ["CMD-SHELL", "pg_isready -U dirigent -d dirigent"]
75
+ interval: 2s
76
+ timeout: 2s
77
+ retries: 15
78
+
79
+ query:
80
+ block: docker.run
81
+ depends_on: [stack]
82
+ deadline: 5m
83
+ config:
84
+ # The postgres image ships psql, so the client needs no image of its own; its
85
+ # entrypoint execs any command that is not the server itself.
86
+ image: postgres:17-alpine
87
+ pull: true
88
+ # Joining the project's network is what makes "db" a hostname here.
89
+ network: "${steps.stack.output.default_network}"
90
+ env:
91
+ PGPASSWORD: dirigent
92
+ # argv, not command: no shell is involved, so each statement is one argument however it
93
+ # is quoted. -A -t strips psql's alignment and headers, leaving rows a later step can
94
+ # read; ON_ERROR_STOP makes the first failing statement the container's exit code.
95
+ argv:
96
+ - psql
97
+ - -h
98
+ - db
99
+ - -U
100
+ - dirigent
101
+ - -d
102
+ - dirigent
103
+ - -v
104
+ - ON_ERROR_STOP=1
105
+ - -A
106
+ - -t
107
+ - -c
108
+ - "create table reading (station text primary key, celsius numeric)"
109
+ - -c
110
+ - "insert into reading values ('bergen', 7.1), ('oslo', 4.5)"
111
+ - -c
112
+ - "select station, celsius from reading order by station"
113
+ memory: 256mb
114
+ pids_limit: 128
115
+
116
+ teardown:
117
+ block: docker.compose.down
118
+ depends_on: [query]
119
+ # all_done runs the teardown on success or failure, so the stack outlives neither.
120
+ rule: all_done
121
+ deadline: 5m
122
+ config:
123
+ # The database's data volume goes with it: a run's stack leaves nothing on the worker.
124
+ down_volumes: true
@@ -0,0 +1,83 @@
1
+ # A bring-up that fails, and the containers it does not leave behind.
2
+ #
3
+ # THIS PIPELINE FAILS ON PURPOSE. It is here to show what a broken stack looks like from the
4
+ # outside, so the expected outcome is a failed run and an empty daemon.
5
+ #
6
+ # REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
7
+ # tcp+TLS, or the local socket in dev). docs/docker.md is the family's home.
8
+ #
9
+ # docker.compose.up declares local_execution, so the engine refuses it unless the instance
10
+ # allowlists its id:
11
+ # dg run --local examples/docker/docker-compose-failing-up.yaml --enable-unsafe docker.compose.up
12
+ # export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up"]'
13
+ #
14
+ # One hop. The stack has two services: "healthy" comes up and passes its check, and "broken"
15
+ # comes up and never will -- its healthcheck is a command that exits non-zero, so after two
16
+ # retries two seconds apart the daemon marks the container unhealthy. wait: true is waiting on
17
+ # exactly that, so compose gives up and exits non-zero, and the step fails with what compose
18
+ # said on stderr. wait_timeout: 30s is the backstop for a check that hangs rather than fails.
19
+ #
20
+ # WHAT A FAILED BRING-UP LEAVES BEHIND: nothing. This is the one case a block cleans up after
21
+ # itself, because compose has already started containers by the time it decides the stack did
22
+ # not come up. On a non-zero exit -- and equally on a timeout or a cancelled step -- the block
23
+ # runs a down of the same project before it raises, so the half-built project goes with the
24
+ # failure. cleanup: true is the default; it is written out below because it is the point of
25
+ # the example. It never touches a bring-up that SUCCEEDED: that one persists for the run, and
26
+ # a docker.compose.down step is what ends it.
27
+ #
28
+ # HOW TO READ THE FAILURE. The step's error is compose's own last words -- typically
29
+ # "container <project>-broken-1 is unhealthy" -- and the run's step list shows the one step
30
+ # failed with nothing after it. The whole of the CLI's two streams are artifacts of the
31
+ # attempt, and the block logs whether the cleanup worked, so a stack that could not be torn
32
+ # down says so rather than disappearing quietly. Afterwards:
33
+ # docker ps -a --filter label=com.docker.compose.project
34
+ # lists nothing from this run.
35
+ #
36
+ # TO MAKE IT YOURS: this is the failure mode to reach for when a service of yours is slow
37
+ # rather than broken. Raise retries and interval, or give the service a start_period, before
38
+ # concluding that a stack cannot come up.
39
+
40
+ format: dirigent/v1
41
+ kind: pipeline
42
+ code: docker-compose-failing-up
43
+ name: A stack that will not come up
44
+ description: A service whose healthcheck never passes, so the bring-up fails and cleans up after itself.
45
+
46
+ tags: [docker, execute]
47
+
48
+ requires:
49
+ blocks:
50
+ - docker.compose.up
51
+
52
+ steps:
53
+ stack:
54
+ block: docker.compose.up
55
+ deadline: 5m
56
+ config:
57
+ wait: true
58
+ # Short on purpose: the example should fail in seconds, not sit out a default that was
59
+ # chosen for stacks meant to work.
60
+ wait_timeout: 30s
61
+ timeout: 2m
62
+ pull: always
63
+ # Tear the half-built project down before failing the step. This is the default, and
64
+ # it is what keeps a failed run from orphaning containers.
65
+ cleanup: true
66
+ content: |
67
+ services:
68
+ healthy:
69
+ image: alpine:3
70
+ command: ["sleep", "600"]
71
+ healthcheck:
72
+ test: ["CMD-SHELL", "exit 0"]
73
+ interval: 2s
74
+ timeout: 2s
75
+ retries: 2
76
+ broken:
77
+ image: alpine:3
78
+ command: ["sleep", "600"]
79
+ healthcheck:
80
+ test: ["CMD-SHELL", "exit 1"]
81
+ interval: 2s
82
+ timeout: 2s
83
+ retries: 2
@@ -0,0 +1,117 @@
1
+ # A compose stack brought up from a FILE on the worker, not from content in the document.
2
+ #
3
+ # REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
4
+ # tcp+TLS, or the local socket in dev). docs/docker.md is the family's home.
5
+ #
6
+ # Every block here declares local_execution, so the engine refuses them unless the instance
7
+ # allowlists their ids:
8
+ # dg run --local examples/docker/docker-compose-file.yaml \
9
+ # --enable-unsafe docker.compose.up,docker.compose.down,docker.run
10
+ # export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up","docker.compose.down","docker.run"]'
11
+ #
12
+ # Four hops, and what each one hands on:
13
+ # write a container writes a compose file into stack/ in the run's work directory.
14
+ # Output: where each declared file landed.
15
+ # stack compose brought up from that path. Output: the services, the networks, and
16
+ # compose_file -- the path the CLI was actually given.
17
+ # probe a container on the stack's network, fetching the service by name.
18
+ # teardown the same project down again, on any outcome.
19
+ #
20
+ # WHERE FILES LIVE DURING A RUN. A run has two places. Scratch -- ${run.scratch}, a prefix
21
+ # under the instance's artifact root -- is storage, and is where a result that outlives the
22
+ # run goes. The work directory is this worker's own filesystem, and is where a tool that opens
23
+ # a file reads it from. Compose runs a CLI, so it names a file in the work directory and
24
+ # nowhere else: the path is always relative to it, never absolute, and never climbs out.
25
+ # "stack/docker-compose.yaml" below is the same file the write step wrote there.
26
+ #
27
+ # THE TWO FORMS. docker.compose.up takes exactly one of content or file. Inline content keeps
28
+ # a small stack in the document, and the block writes it into the work directory for the CLI
29
+ # anyway; file is the form for a document an earlier step produced -- generated here, but just
30
+ # as often a storage.copy out of an object store, or a repository checkout. Either way the up
31
+ # step reports compose_file, the path it used, so a later step needing the same document -- a
32
+ # down that has to resolve the project by file rather than by label -- can name it without
33
+ # knowing which form was used.
34
+ #
35
+ # TO MAKE IT YOURS: replace the write step with whatever actually produces your compose file,
36
+ # and leave the rest alone. Note that a compose file with build: stanzas needs its build
37
+ # contexts in the work directory too; v1 targets stacks of pre-built images.
38
+
39
+ format: dirigent/v1
40
+ kind: pipeline
41
+ code: docker-compose-file
42
+ name: A stack from a compose file on the worker
43
+ description: Write a compose file into the run's work directory, bring the stack up from that path, and tear it down.
44
+
45
+ tags: [docker, execute]
46
+
47
+ requires:
48
+ blocks:
49
+ - docker.run
50
+ - docker.compose.up
51
+ - docker.compose.down
52
+
53
+ steps:
54
+ write:
55
+ block: docker.run
56
+ deadline: 5m
57
+ config:
58
+ image: alpine:3
59
+ pull: true
60
+ network: none
61
+ # Whatever the container leaves in its output mount is copied where the target names
62
+ # once it exits. A target with no URI scheme is a path in the run's work directory on
63
+ # this worker, which is where the compose CLI opens its file from; a URI would send the
64
+ # file to storage instead, for a result that outlives the run.
65
+ outputs:
66
+ docker-compose.yaml: "stack/docker-compose.yaml"
67
+ command: |
68
+ cat > /dirigent/outputs/docker-compose.yaml <<'COMPOSE'
69
+ services:
70
+ web:
71
+ image: nginx:alpine
72
+ healthcheck:
73
+ test: ["CMD-SHELL", "wget -qO- http://localhost/ >/dev/null 2>&1 || exit 1"]
74
+ interval: 2s
75
+ timeout: 2s
76
+ retries: 5
77
+ COMPOSE
78
+ memory: 64mb
79
+ pids_limit: 64
80
+
81
+ stack:
82
+ block: docker.compose.up
83
+ depends_on: [write]
84
+ deadline: 10m
85
+ config:
86
+ # Relative to the run's work directory, matching what the write step produced. The CLI
87
+ # is always run with --project-directory set to that directory, so relative paths inside
88
+ # the compose file resolve against it rather than against this file's directory.
89
+ file: stack/docker-compose.yaml
90
+ wait: true
91
+ wait_timeout: 2m
92
+ pull: always
93
+
94
+ probe:
95
+ block: docker.run
96
+ depends_on: [stack]
97
+ deadline: 5m
98
+ config:
99
+ image: alpine:3
100
+ pull: true
101
+ # The network the up step reported: joining it resolves "web" to the service.
102
+ network: "${steps.stack.output.default_network}"
103
+ argv: [wget, -qO-, "http://web/"]
104
+ memory: 64mb
105
+ pids_limit: 64
106
+
107
+ teardown:
108
+ block: docker.compose.down
109
+ depends_on: [probe]
110
+ rule: all_done
111
+ deadline: 5m
112
+ config:
113
+ # A project tears down by its label alone, so the file is optional here; passing back
114
+ # the path the up step reported is how a teardown that does need -f gets it, whichever
115
+ # form the up step was given.
116
+ file: "${steps.stack.output.compose_file}"
117
+ down_volumes: true
@@ -0,0 +1,133 @@
1
+ # Pipeline parameters reaching into a compose stack: an interpolated variable, and a profile.
2
+ #
3
+ # REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
4
+ # tcp+TLS, or the local socket in dev). docs/docker.md is the family's home.
5
+ #
6
+ # Every block here declares local_execution, so the engine refuses them unless the instance
7
+ # allowlists their ids:
8
+ # dg run --local examples/docker/docker-compose-profiles-env.yaml \
9
+ # --enable-unsafe docker.compose.up,docker.compose.down,docker.run
10
+ # export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up","docker.compose.down","docker.run"]'
11
+ #
12
+ # Three hops, and what each one hands on:
13
+ # stack a two-service stack where one service is optional. Output: the containers the
14
+ # project actually has, so the profile's effect is visible in the record.
15
+ # present a container on the stack's network, fetching the greeting the first service is
16
+ # serving and asking whether the optional one is there at all. Output: its stdout,
17
+ # which is the parameter's value followed by a verdict on the profile.
18
+ # teardown the same project down again, on any outcome.
19
+ #
20
+ # TWO WAYS A PARAMETER REACHES COMPOSE, and they are not the same mechanism:
21
+ # env variables set for the compose CLI process itself, which compose interpolates
22
+ # into the file it reads. ${params.greeting} is resolved by the engine before the
23
+ # block sees the config, so the value is a plain string by the time compose runs.
24
+ # profiles the --profile flags the CLI is given. A service that declares profiles is left
25
+ # out of every command that does not activate one of them, so this is how a
26
+ # pipeline turns an optional service on and off without two compose files.
27
+ #
28
+ # WRITE $${GREETING} FOR COMPOSE, ${...} FOR THE ENGINE. Every ${...} in a document is a
29
+ # reference the engine resolves, and it fails the attempt on a name it does not know, so a
30
+ # variable compose is meant to interpolate is written with the escape: $${GREETING} reaches
31
+ # compose as the literal ${GREETING}, and the engine never looks the name up.
32
+ #
33
+ # project_name is set explicitly here to show the field. It defaults to a name derived from
34
+ # the run id -- shared by the up, the steps that drive it and the down, and unique between
35
+ # runs -- so an explicit one has to keep both those promises itself, which is what ${run.id}
36
+ # does. A fixed string would collide with the next run of the same pipeline.
37
+ #
38
+ # TO MAKE IT YOURS: the parameter names are the only thing specific to this file. Pass
39
+ # -p profile=none to see the same stack come up without its optional service.
40
+
41
+ format: dirigent/v1
42
+ kind: pipeline
43
+ code: docker-compose-profiles-env
44
+ name: Parameters into a compose stack
45
+ description: A compose stack whose greeting comes from a parameter and whose optional service comes from a profile.
46
+
47
+ tags: [docker, execute]
48
+
49
+ requires:
50
+ blocks:
51
+ - docker.compose.up
52
+ - docker.compose.down
53
+ - docker.run
54
+
55
+ params:
56
+ type: object
57
+ additionalProperties: false
58
+ properties:
59
+ greeting:
60
+ type: string
61
+ default: hello from a parameter
62
+ description: Interpolated into the compose file as $${GREETING} and served by the greeter.
63
+ profile:
64
+ type: string
65
+ default: extras
66
+ description: The compose profile to activate; anything but "extras" leaves the reporter out.
67
+
68
+ steps:
69
+ stack:
70
+ block: docker.compose.up
71
+ deadline: 10m
72
+ config:
73
+ wait: true
74
+ wait_timeout: 2m
75
+ pull: always
76
+ project_name: "dirigent-profiles-${run.id}"
77
+ # A whole reference resolves to the value itself, so this is a one-element list of the
78
+ # parameter's string rather than a list containing a template.
79
+ profiles: ["${params.profile}"]
80
+ # Set for the CLI process, which is where compose reads interpolation from.
81
+ env:
82
+ GREETING: "${params.greeting}"
83
+ content: |
84
+ services:
85
+ greeter:
86
+ image: nginx:alpine
87
+ # Compose substitutes GREETING out of the env above before docker sees this, so
88
+ # the file the daemon runs carries the parameter's value, not the variable. The
89
+ # greeting is written where nginx serves from, which is how a later step reads it.
90
+ command:
91
+ - sh
92
+ - -c
93
+ - echo $${GREETING} > /usr/share/nginx/html/index.html && exec nginx -g 'daemon off;'
94
+ healthcheck:
95
+ test: ["CMD-SHELL", "wget -qO- http://127.0.0.1/ >/dev/null 2>&1 || exit 1"]
96
+ interval: 2s
97
+ timeout: 2s
98
+ retries: 5
99
+ reporter:
100
+ # Declared under a profile, so this service exists only when that profile is
101
+ # activated. The greeter, declaring none, is always part of the project.
102
+ profiles: [extras]
103
+ image: alpine:3
104
+ command: ["sleep", "600"]
105
+
106
+ present:
107
+ block: docker.run
108
+ depends_on: [stack]
109
+ deadline: 5m
110
+ config:
111
+ image: alpine:3
112
+ pull: true
113
+ network: "${steps.stack.output.default_network}"
114
+ # The first line proves what compose interpolated; the second proves what the profile
115
+ # selected, because a name that does not resolve on the project's network is a service
116
+ # the profile left out.
117
+ argv:
118
+ - sh
119
+ - -c
120
+ - 'wget -qO- http://greeter/; nslookup reporter >/dev/null 2>&1 && echo "reporter is up" || echo "reporter was left out"'
121
+ memory: 64mb
122
+ pids_limit: 64
123
+
124
+ teardown:
125
+ block: docker.compose.down
126
+ depends_on: [present]
127
+ rule: all_done
128
+ deadline: 5m
129
+ config:
130
+ # The project name is not defaulted here, so the teardown has to name the same one the
131
+ # up step built.
132
+ project_name: "dirigent-profiles-${run.id}"
133
+ down_volumes: true
@@ -0,0 +1,70 @@
1
+ # Bringing a compose stack up, driving it, and tearing it down even on failure.
2
+ #
3
+ # REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
4
+ # tcp+TLS, or the local socket in dev). docs/docker.md is the family's home; docs/operations.md
5
+ # says what mounting the host socket grants.
6
+ #
7
+ # docker.compose.up and docker.compose.down declare local_execution, so the engine refuses
8
+ # them unless the instance allowlists their ids:
9
+ # dg run --local examples/docker/docker-compose-stack.yaml \
10
+ # --enable-unsafe docker.compose.up,docker.compose.down,docker.run
11
+ # export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up","docker.compose.down","docker.run"]'
12
+ #
13
+ # The lifecycle is the point: up brings the stack up detached and it PERSISTS, so the drive
14
+ # step reaches it; down runs with rule: all_done, so the stack is torn down whether the drive
15
+ # step passed or failed. Both default their project name from the run id, so down finds the
16
+ # stack up created with no wiring.
17
+
18
+ format: dirigent/v1
19
+ kind: pipeline
20
+ code: docker-compose-stack
21
+ name: A compose stack, driven and torn down
22
+ description: Bring a two-service stack up, reach it from a container, and tear it down on any outcome.
23
+
24
+ tags: [docker, execute]
25
+
26
+ requires:
27
+ blocks:
28
+ - docker.compose.up
29
+ - docker.compose.down
30
+ - docker.run
31
+
32
+ steps:
33
+ stack:
34
+ block: docker.compose.up
35
+ deadline: 5m
36
+ config:
37
+ # wait: true holds the step open until every service is healthy, so the drive step
38
+ # never races the stack coming up. pull: always fetches the images on a fresh worker.
39
+ wait: true
40
+ pull: always
41
+ content: |
42
+ services:
43
+ api:
44
+ image: nginx:alpine
45
+ healthcheck:
46
+ test: ["CMD-SHELL", "wget -qO- http://localhost/ >/dev/null 2>&1 || exit 1"]
47
+ interval: 2s
48
+ timeout: 2s
49
+ retries: 5
50
+
51
+ drive:
52
+ block: docker.run
53
+ depends_on: [stack]
54
+ deadline: 2m
55
+ config:
56
+ image: alpine:3
57
+ pull: true
58
+ # The network the up step reported: joining it is how this container resolves and
59
+ # reaches the "api" service by name.
60
+ network: "${steps.stack.output.default_network}"
61
+ command: wget -qO- http://api/
62
+
63
+ teardown:
64
+ block: docker.compose.down
65
+ depends_on: [drive]
66
+ # all_done runs the teardown on success or failure, so a stack is never left running.
67
+ rule: all_done
68
+ deadline: 2m
69
+ config:
70
+ down_volumes: true
@@ -0,0 +1,53 @@
1
+ # Running a container as a step, and what that costs in trust.
2
+ #
3
+ # REQUIRES A DOCKER SOCKET: a Docker daemon the worker can reach, at /var/run/docker.sock
4
+ # or wherever DOCKER_HOST points. Without one the document still validates and applies; it
5
+ # fails at the step, saying the daemon could not be reached.
6
+ #
7
+ # docker.run declares local_execution, so the engine refuses it unless the instance
8
+ # allowlists the id. Reaching the Docker socket is reaching root on the host:
9
+ # dg run --local examples/docker/docker-hello.yaml --enable-unsafe docker.run
10
+ # export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.run"]' # for the instance
11
+ #
12
+ # The block is asynchronous: execute creates the container and starts it detached, and the
13
+ # engine probes it. A container that runs for an hour costs one parked row, not an hour of
14
+ # a worker, and the deadline stays on the step.
15
+ #
16
+ # network: none is the default: an image a document named does not reach the worker's
17
+ # network unless the step says so. pull: true fetches the image first, so this runs on a
18
+ # machine that has never seen alpine.
19
+
20
+ format: dirigent/v1
21
+ kind: pipeline
22
+ code: docker-hello
23
+ name: Hello from a container
24
+ description: Run a command in a container, with no network and the image pulled first.
25
+
26
+ tags: [docker, execute]
27
+
28
+ requires:
29
+ blocks:
30
+ - docker.run
31
+
32
+ params:
33
+ type: object
34
+ additionalProperties: false
35
+ properties:
36
+ message:
37
+ type: string
38
+ default: hello from a container
39
+ description: What the container echoes.
40
+
41
+ steps:
42
+ greet:
43
+ block: docker.run
44
+ deadline: 5m
45
+ config:
46
+ image: alpine:3
47
+ pull: true
48
+ # A container that needs nothing from the network is given none, and the memory and
49
+ # process caps bound what a runaway image can take from the worker it landed on.
50
+ network: none
51
+ argv: [echo, "${params.message}"]
52
+ memory: 64mb
53
+ pids_limit: 64
@@ -0,0 +1,92 @@
1
+ # Naming the daemon: the same compose stack, run on a daemon a connection points at.
2
+ #
3
+ # The compose lifecycle example (docker-compose-stack.yaml) uses whatever daemon the worker's
4
+ # own environment names. This one names it in the document instead, through a `docker`
5
+ # connection the instance holds, and that connection carries the client TLS the daemon
6
+ # demands.
7
+ #
8
+ # REQUIRES A DOCKER CONNECTION the instance already holds. Create it once, with the client
9
+ # half of the daemon's certificates -- a dind sidecar writes them to DOCKER_TLS_CERTDIR/client:
10
+ # dg connection create docker build-daemon \
11
+ # --set host=tcp://docker:2376 \
12
+ # --set "tls_ca=$(cat /certs/client/ca.pem)" \
13
+ # --set "tls_cert=$(cat /certs/client/cert.pem)" \
14
+ # --set "tls_key=$(cat /certs/client/key.pem)"
15
+ #
16
+ # The three blocks declare local_execution, so the engine refuses them unless the instance
17
+ # allowlists their ids:
18
+ # export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up","docker.compose.down","docker.run"]'
19
+ #
20
+ # EVERY STEP NAMES THE SAME CONNECTION, and that is the whole discipline here. A stack exists
21
+ # only on the daemon that was told to create it, so an `up` on one daemon and a `down` on
22
+ # another tear down nothing: the second daemon has no such project. The drive step is in it
23
+ # too, because it joins a network the first daemon owns.
24
+ #
25
+ # tls_key is sealed: encrypted at rest, redacted in every API response, and reaching the CLI
26
+ # only as a 0600 file under the run's work directory that goes when the step leaves. A client
27
+ # key for a docker daemon is a credential for root on whatever that daemon runs on.
28
+ #
29
+ # TO MAKE IT YOURS: point host at your own daemon. A daemon that needs no TLS -- an ssh://
30
+ # host, or a unix socket on another path -- takes host alone and none of the three PEMs.
31
+
32
+ format: dirigent/v1
33
+ kind: pipeline
34
+ code: docker-remote-daemon
35
+ name: A compose stack on a named daemon
36
+ description: Bring a stack up, drive it and tear it down on the daemon a docker connection names.
37
+
38
+ tags: [docker, execute]
39
+
40
+ requires:
41
+ blocks:
42
+ - docker.compose.up
43
+ - docker.compose.down
44
+ - docker.run
45
+ workers:
46
+ # The compose file and its build contexts are on the worker's own filesystem, so all three
47
+ # steps have to meet one docker-capable worker even though the daemon is named here.
48
+ - docker
49
+ connections:
50
+ # Named, not carried: the instance holds the connection and its sealed TLS key, and this
51
+ # document refuses to apply where it is absent rather than failing the first time it runs.
52
+ - build-daemon
53
+
54
+ steps:
55
+ stack:
56
+ block: docker.compose.up
57
+ deadline: 5m
58
+ config:
59
+ connection: build-daemon
60
+ wait: true
61
+ pull: always
62
+ content: |
63
+ services:
64
+ api:
65
+ image: nginx:alpine
66
+ healthcheck:
67
+ test: ["CMD-SHELL", "wget -qO- http://localhost/ >/dev/null 2>&1 || exit 1"]
68
+ interval: 2s
69
+ timeout: 2s
70
+ retries: 5
71
+
72
+ drive:
73
+ block: docker.run
74
+ depends_on: [stack]
75
+ deadline: 2m
76
+ config:
77
+ connection: build-daemon
78
+ image: alpine:3
79
+ pull: true
80
+ # The network the up step reported, which exists on that daemon and nowhere else.
81
+ network: "${steps.stack.output.default_network}"
82
+ command: wget -qO- http://api/
83
+
84
+ teardown:
85
+ block: docker.compose.down
86
+ depends_on: [drive]
87
+ # all_done runs the teardown on success or failure, so a stack is never left running.
88
+ rule: all_done
89
+ deadline: 2m
90
+ config:
91
+ connection: build-daemon
92
+ down_volumes: true