dirigent-examples 0.15.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dirigent_examples/__init__.py +22 -0
- dirigent_examples/py.typed +0 -0
- dirigent_examples/shelves/README.md +299 -0
- dirigent_examples/shelves/composition/README.md +18 -0
- dirigent_examples/shelves/composition/chained-instances.yaml +97 -0
- dirigent_examples/shelves/composition/composition-child.yaml +64 -0
- dirigent_examples/shelves/composition/composition-parent.yaml +89 -0
- dirigent_examples/shelves/connections.yaml +52 -0
- dirigent_examples/shelves/demo/README.md +19 -0
- dirigent_examples/shelves/demo/markdown-showcase.yaml +117 -0
- dirigent_examples/shelves/demo/params-showcase.yaml +92 -0
- dirigent_examples/shelves/demo/requires.yaml +65 -0
- dirigent_examples/shelves/demo/weekly-import-malawi.yaml +54 -0
- dirigent_examples/shelves/demo/weekly-import-nepal.yaml +75 -0
- dirigent_examples/shelves/docker/README.md +29 -0
- dirigent_examples/shelves/docker/docker-build-push.yaml +110 -0
- dirigent_examples/shelves/docker/docker-build-run.yaml +106 -0
- dirigent_examples/shelves/docker/docker-compose-database.yaml +124 -0
- dirigent_examples/shelves/docker/docker-compose-failing-up.yaml +83 -0
- dirigent_examples/shelves/docker/docker-compose-file.yaml +117 -0
- dirigent_examples/shelves/docker/docker-compose-profiles-env.yaml +133 -0
- dirigent_examples/shelves/docker/docker-compose-stack.yaml +70 -0
- dirigent_examples/shelves/docker/docker-hello.yaml +53 -0
- dirigent_examples/shelves/docker/docker-remote-daemon.yaml +92 -0
- dirigent_examples/shelves/docker/docker-run-failing-teardown.yaml +94 -0
- dirigent_examples/shelves/docker/docker-ticker.yaml +49 -0
- dirigent_examples/shelves/execute/README.md +16 -0
- dirigent_examples/shelves/execute/long-log.yaml +89 -0
- dirigent_examples/shelves/failure/README.md +20 -0
- dirigent_examples/shelves/failure/error-handler.yaml +89 -0
- dirigent_examples/shelves/failure/optional-step.yaml +82 -0
- dirigent_examples/shelves/failure/retries.yaml +75 -0
- dirigent_examples/shelves/failure/retry-budget.yaml +82 -0
- dirigent_examples/shelves/failure/step-timeout.yaml +96 -0
- dirigent_examples/shelves/git/README.md +32 -0
- dirigent_examples/shelves/git/git-checkout-build.yaml +125 -0
- dirigent_examples/shelves/git/git-checkout-compose.yaml +138 -0
- dirigent_examples/shelves/git/git-checkout-public.yaml +84 -0
- dirigent_examples/shelves/graph/README.md +22 -0
- dirigent_examples/shelves/graph/deep-chain.yaml +119 -0
- dirigent_examples/shelves/graph/fan-in.yaml +80 -0
- dirigent_examples/shelves/graph/fan-out.yaml +66 -0
- dirigent_examples/shelves/graph/linear.yaml +66 -0
- dirigent_examples/shelves/graph/parallel-branches.yaml +57 -0
- dirigent_examples/shelves/graph/parallel-sleep.yaml +55 -0
- dirigent_examples/shelves/graph/skip-diamond.yaml +105 -0
- dirigent_examples/shelves/graph/wide-fan.yaml +147 -0
- dirigent_examples/shelves/hello-world.yaml +30 -0
- dirigent_examples/shelves/open-data/README.md +67 -0
- dirigent_examples/shelves/open-data/feeds-composition.yaml +120 -0
- dirigent_examples/shelves/open-data/gdacs-disaster-updates.yaml +230 -0
- dirigent_examples/shelves/open-data/github-releases-relay.yaml +225 -0
- dirigent_examples/shelves/open-data/hdx-dataset-watch.yaml +211 -0
- dirigent_examples/shelves/open-data/kobo-submissions-to-csv.yaml +149 -0
- dirigent_examples/shelves/open-data/nominatim-geocode-facilities.yaml +169 -0
- dirigent_examples/shelves/open-data/odk-central-submissions.yaml +146 -0
- dirigent_examples/shelves/open-data/open-meteo-weekly-report.yaml +142 -0
- dirigent_examples/shelves/open-data/overpass-health-facilities.yaml +154 -0
- dirigent_examples/shelves/open-data/usgs-earthquakes-alert.yaml +208 -0
- dirigent_examples/shelves/open-data/who-gho-indicators-to-parquet.yaml +163 -0
- dirigent_examples/shelves/open-data/wikidata-country-reference.yaml +146 -0
- dirigent_examples/shelves/open-data/world-bank-population-trend.yaml +166 -0
- dirigent_examples/shelves/patterns/README.md +144 -0
- dirigent_examples/shelves/patterns/concurrency-queue.yaml +89 -0
- dirigent_examples/shelves/patterns/concurrency-replace.yaml +92 -0
- dirigent_examples/shelves/patterns/concurrency-skip.yaml +97 -0
- dirigent_examples/shelves/patterns/connections-referenced-vs-carried.yaml +140 -0
- dirigent_examples/shelves/patterns/deadline-on-a-sensor.yaml +106 -0
- dirigent_examples/shelves/patterns/fan-out-continue.yaml +88 -0
- dirigent_examples/shelves/patterns/fan-out-fail-fast.yaml +80 -0
- dirigent_examples/shelves/patterns/fan-out-from-params.yaml +84 -0
- dirigent_examples/shelves/patterns/fan-out-item-wise.yaml +111 -0
- dirigent_examples/shelves/patterns/fan-out-literal-list.yaml +81 -0
- dirigent_examples/shelves/patterns/fan-out-nested-objects.yaml +107 -0
- dirigent_examples/shelves/patterns/fan-out-then-join.yaml +86 -0
- dirigent_examples/shelves/patterns/log-levels.yaml +119 -0
- dirigent_examples/shelves/patterns/outputs-inline-vs-storage.yaml +140 -0
- dirigent_examples/shelves/patterns/params-every-type.yaml +259 -0
- dirigent_examples/shelves/patterns/params-validation-refuses.yaml +131 -0
- dirigent_examples/shelves/patterns/pipeline-run-child.yaml +96 -0
- dirigent_examples/shelves/patterns/pipeline-run-fire-and-forget.yaml +99 -0
- dirigent_examples/shelves/patterns/pipeline-run-strict.yaml +103 -0
- dirigent_examples/shelves/patterns/pipeline-run-wait.yaml +107 -0
- dirigent_examples/shelves/patterns/pipeline-run-with-params.yaml +124 -0
- dirigent_examples/shelves/patterns/poll-cadence.yaml +102 -0
- dirigent_examples/shelves/patterns/priority-layered.yaml +120 -0
- dirigent_examples/shelves/patterns/references-cheat-sheet.yaml +186 -0
- dirigent_examples/shelves/patterns/retry-budget-exhausted.yaml +92 -0
- dirigent_examples/shelves/patterns/retry-exponential-backoff.yaml +88 -0
- dirigent_examples/shelves/patterns/retry-only-transient.yaml +108 -0
- dirigent_examples/shelves/patterns/retry-with-jitter.yaml +102 -0
- dirigent_examples/shelves/patterns/rule-all-done.yaml +83 -0
- dirigent_examples/shelves/patterns/rule-all-success.yaml +79 -0
- dirigent_examples/shelves/patterns/rule-always.yaml +92 -0
- dirigent_examples/shelves/patterns/rule-one-failed.yaml +86 -0
- dirigent_examples/shelves/patterns/schedule-at-once.yaml +110 -0
- dirigent_examples/shelves/patterns/schedule-cron-timezone.yaml +121 -0
- dirigent_examples/shelves/patterns/schedule-interval.yaml +109 -0
- dirigent_examples/shelves/patterns/schedule-window-half-open.yaml +105 -0
- dirigent_examples/shelves/patterns/sensor-http-ready.yaml +121 -0
- dirigent_examples/shelves/patterns/sensor-storage-exists.yaml +134 -0
- dirigent_examples/shelves/patterns/step-names-and-keys.yaml +99 -0
- dirigent_examples/shelves/patterns/timeout-fails-the-step.yaml +94 -0
- dirigent_examples/shelves/patterns/timeout-skips-the-step.yaml +102 -0
- dirigent_examples/shelves/patterns/webhook-mapping-nested-payload.yaml +125 -0
- dirigent_examples/shelves/patterns/webhook-signed.yaml +144 -0
- dirigent_examples/shelves/preview/s3-parquet-to-ingestion.yaml +92 -0
- dirigent_examples/shelves/python/README.md +31 -0
- dirigent_examples/shelves/python/apply_and_run.py +52 -0
- dirigent_examples/shelves/python/ci_gate.py +76 -0
- dirigent_examples/shelves/python/connections.py +61 -0
- dirigent_examples/shelves/python/error_handling.py +84 -0
- dirigent_examples/shelves/python/follow_logs.py +39 -0
- dirigent_examples/shelves/python/list_and_filter.py +52 -0
- dirigent_examples/shelves/queues/README.md +59 -0
- dirigent_examples/shelves/queues/kafka-consume-then-transform.yaml +105 -0
- dirigent_examples/shelves/queues/kafka-produce-then-consume.yaml +124 -0
- dirigent_examples/shelves/queues/rabbitmq-consume-ack-on-success.yaml +102 -0
- dirigent_examples/shelves/queues/report-to-kafka.yaml +105 -0
- dirigent_examples/shelves/queues/report-to-rabbitmq.yaml +105 -0
- dirigent_examples/shelves/recipes/README.md +130 -0
- dirigent_examples/shelves/recipes/csv-header-rules.yaml +147 -0
- dirigent_examples/shelves/recipes/csv-to-ndjson.yaml +107 -0
- dirigent_examples/shelves/recipes/etl-csv-clean-validate-parquet.yaml +207 -0
- dirigent_examples/shelves/recipes/filter-by-predicate.yaml +116 -0
- dirigent_examples/shelves/recipes/filter-then-map-then-reduce.yaml +109 -0
- dirigent_examples/shelves/recipes/http-fetch-validate-post.yaml +142 -0
- dirigent_examples/shelves/recipes/http-follow-redirects.yaml +100 -0
- dirigent_examples/shelves/recipes/http-get-with-query.yaml +96 -0
- dirigent_examples/shelves/recipes/http-headers-and-auth-connection.yaml +110 -0
- dirigent_examples/shelves/recipes/http-post-file-from-storage.yaml +117 -0
- dirigent_examples/shelves/recipes/http-post-json-echo.yaml +105 -0
- dirigent_examples/shelves/recipes/http-post-report.yaml +183 -0
- dirigent_examples/shelves/recipes/http-save-body-to-storage.yaml +106 -0
- dirigent_examples/shelves/recipes/http-success-status-list.yaml +80 -0
- dirigent_examples/shelves/recipes/http-timeout-override.yaml +104 -0
- dirigent_examples/shelves/recipes/jq-dedupe-by-key.yaml +78 -0
- dirigent_examples/shelves/recipes/jq-defaults-and-nulls.yaml +91 -0
- dirigent_examples/shelves/recipes/jq-group-by-and-sum.yaml +74 -0
- dirigent_examples/shelves/recipes/jq-join-two-lists.yaml +77 -0
- dirigent_examples/shelves/recipes/jq-long-to-wide.yaml +76 -0
- dirigent_examples/shelves/recipes/jq-nested-to-flat.yaml +89 -0
- dirigent_examples/shelves/recipes/jq-pivot-wide-to-long.yaml +65 -0
- dirigent_examples/shelves/recipes/jq-running-totals.yaml +82 -0
- dirigent_examples/shelves/recipes/jq-string-cleaning.yaml +88 -0
- dirigent_examples/shelves/recipes/jq-top-n.yaml +85 -0
- dirigent_examples/shelves/recipes/jq-validate-in-jq-vs-schema.yaml +124 -0
- dirigent_examples/shelves/recipes/jq-window-dates.yaml +82 -0
- dirigent_examples/shelves/recipes/json-to-csv-flattening.yaml +141 -0
- dirigent_examples/shelves/recipes/large-output-to-storage.yaml +134 -0
- dirigent_examples/shelves/recipes/map-enrich-with-lookup.yaml +96 -0
- dirigent_examples/shelves/recipes/ndjson-to-parquet.yaml +130 -0
- dirigent_examples/shelves/recipes/pagination-by-fan-out.yaml +115 -0
- dirigent_examples/shelves/recipes/parquet-round-trip-types.yaml +163 -0
- dirigent_examples/shelves/recipes/reconcile-two-sources.yaml +159 -0
- dirigent_examples/shelves/recipes/report-built-in.yaml +72 -0
- dirigent_examples/shelves/recipes/report-daily-digest.yaml +186 -0
- dirigent_examples/shelves/recipes/report-to-file.yaml +131 -0
- dirigent_examples/shelves/recipes/report-to-webhook.yaml +136 -0
- dirigent_examples/shelves/recipes/schema-carried.yaml +112 -0
- dirigent_examples/shelves/recipes/schema-formats.yaml +107 -0
- dirigent_examples/shelves/recipes/schema-referenced.yaml +86 -0
- dirigent_examples/shelves/recipes/schema-refuses-then-rule.yaml +127 -0
- dirigent_examples/shelves/recipes/storage-copy-dated-archive.yaml +114 -0
- dirigent_examples/shelves/recipes/storage-exists-gate.yaml +127 -0
- dirigent_examples/shelves/recipes/storage-manifest-of-a-fan-out.yaml +104 -0
- dirigent_examples/shelves/recipes/storage-write-then-read.yaml +119 -0
- dirigent_examples/shelves/recipes/webhook-post-hmac.yaml +132 -0
- dirigent_examples/shelves/recipes/webhook-post-summary.yaml +142 -0
- dirigent_examples/shelves/s3/README.md +34 -0
- dirigent_examples/shelves/s3/report-to-s3.yaml +93 -0
- dirigent_examples/shelves/s3/s3-copy-and-verify.yaml +105 -0
- dirigent_examples/shelves/s3/s3-csv-report.yaml +87 -0
- dirigent_examples/shelves/s3/s3-parquet-report.yaml +77 -0
- dirigent_examples/shelves/s3/s3-round-trip.yaml +125 -0
- dirigent_examples/shelves/schemas/README.md +36 -0
- dirigent_examples/shelves/schemas/echo-reading.json +18 -0
- dirigent_examples/shelves/schemas/ou-record.json +13 -0
- dirigent_examples/shelves/schemas/station-reading.json +13 -0
- dirigent_examples/shelves/sensors/README.md +16 -0
- dirigent_examples/shelves/sensors/sensor-gate.yaml +65 -0
- dirigent_examples/shelves/sensors/time-window.yaml +61 -0
- dirigent_examples/shelves/sql/README.md +52 -0
- dirigent_examples/shelves/sql/duckdb-parquet-to-report.yaml +146 -0
- dirigent_examples/shelves/sql/sql-postgres-readonly.yaml +111 -0
- dirigent_examples/shelves/sql/sql-query-to-storage.yaml +85 -0
- dirigent_examples/shelves/sql/sql-sqlite-roundtrip.yaml +114 -0
- dirigent_examples/shelves/sql/warehouse.sql +42 -0
- dirigent_examples/shelves/transform/README.md +36 -0
- dirigent_examples/shelves/transform/csv-report.yaml +55 -0
- dirigent_examples/shelves/transform/jq-filter-and-map.yaml +70 -0
- dirigent_examples/shelves/transform/jq-group-and-aggregate.yaml +70 -0
- dirigent_examples/shelves/transform/jq-join-two-sources.yaml +98 -0
- dirigent_examples/shelves/transform/jq-reshape.yaml +91 -0
- dirigent_examples/shelves/transform/jq-stream-through-storage.yaml +112 -0
- dirigent_examples/shelves/transform/ndjson-round-trip.yaml +56 -0
- dirigent_examples/shelves/transform/parquet-round-trip.yaml +68 -0
- dirigent_examples/shelves/transform/std-convert-fan-out.yaml +142 -0
- dirigent_examples/shelves/transform/xml-feed-to-ndjson.yaml +116 -0
- dirigent_examples/shelves/transform/yaml-config-to-json.yaml +104 -0
- dirigent_examples/shelves/triggers/README.md +45 -0
- dirigent_examples/shelves/triggers/at-one-time.yaml +78 -0
- dirigent_examples/shelves/triggers/cron-nightly.yaml +79 -0
- dirigent_examples/shelves/triggers/cron-windowed.yaml +86 -0
- dirigent_examples/shelves/triggers/document-nightly.yaml +80 -0
- dirigent_examples/shelves/triggers/interval-rolling.yaml +88 -0
- dirigent_examples/shelves/triggers/managed-and-manual.yaml +109 -0
- dirigent_examples/shelves/triggers/webhook-trigger.yaml +75 -0
- dirigent_examples/shelves/validate/README.md +31 -0
- dirigent_examples/shelves/validate/expects-a-shape.yaml +56 -0
- dirigent_examples/shelves/validate/the-shape-is-wrong.yaml +46 -0
- dirigent_examples-0.15.0.dist-info/METADATA +21 -0
- dirigent_examples-0.15.0.dist-info/RECORD +216 -0
- dirigent_examples-0.15.0.dist-info/WHEEL +4 -0
- dirigent_examples-0.15.0.dist-info/entry_points.txt +3 -0
- dirigent_examples-0.15.0.dist-info/licenses/LICENSE +18 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# A database stack brought up, driven with real SQL, and torn down whatever happened.
|
|
2
|
+
#
|
|
3
|
+
# REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
|
|
4
|
+
# tcp+TLS, or the local socket in dev). docs/docker.md is the family's home.
|
|
5
|
+
#
|
|
6
|
+
# Every block here declares local_execution, so the engine refuses them unless the instance
|
|
7
|
+
# allowlists their ids:
|
|
8
|
+
# dg run --local examples/docker/docker-compose-database.yaml \
|
|
9
|
+
# --enable-unsafe docker.compose.up,docker.compose.down,docker.run
|
|
10
|
+
# export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up","docker.compose.down","docker.run"]'
|
|
11
|
+
#
|
|
12
|
+
# Three hops, and what each one hands on:
|
|
13
|
+
# stack one postgres service, brought up detached and left running for the rest of the
|
|
14
|
+
# run. Output: every service's container with its state and health, the networks
|
|
15
|
+
# they sit on, and default_network.
|
|
16
|
+
# query a psql container joined to that network, which creates a table, inserts two
|
|
17
|
+
# rows and selects them back. Output: the exit code and the stdout those selects
|
|
18
|
+
# printed, which is the step's result a downstream step would read.
|
|
19
|
+
# teardown the same project down again, volumes included.
|
|
20
|
+
#
|
|
21
|
+
# WAIT IS WHAT TURNS A HEALTHCHECK INTO A GATE. Postgres accepts connections some seconds
|
|
22
|
+
# after its container starts, and a compose file's healthcheck is only an opinion the daemon
|
|
23
|
+
# records unless something waits on it. wait: true holds the up step open until every service
|
|
24
|
+
# is running and healthy, so the query step meets a database that is ready rather than
|
|
25
|
+
# racing it and failing on connection refused. Without it the stack step returns the moment
|
|
26
|
+
# compose has started the containers.
|
|
27
|
+
#
|
|
28
|
+
# THE NETWORK NAME COMES FROM THE UP STEP. Compose puts the project on a network of its own
|
|
29
|
+
# and the up step reports it as default_network; the query step names that as its network
|
|
30
|
+
# mode, which is what lets it resolve the service by the name the compose file gave it -- db.
|
|
31
|
+
#
|
|
32
|
+
# THE TEARDOWN RUNS ON ANY OUTCOME. rule: all_done means the down step runs whether the query
|
|
33
|
+
# passed or failed, so a database is never left running past the run that started it. Both
|
|
34
|
+
# compose steps default their project name from the run id, so the down addresses exactly the
|
|
35
|
+
# project the up created with nothing wired between them.
|
|
36
|
+
#
|
|
37
|
+
# TO MAKE IT YOURS: put your own schema and queries in the psql argv, or swap postgres for
|
|
38
|
+
# whatever your pipeline actually talks to; the shape -- up with wait, drive, down with
|
|
39
|
+
# all_done -- does not change.
|
|
40
|
+
|
|
41
|
+
format: dirigent/v1
|
|
42
|
+
kind: pipeline
|
|
43
|
+
code: docker-compose-database
|
|
44
|
+
name: A database stack, queried and torn down
|
|
45
|
+
description: Bring PostgreSQL up behind its healthcheck, run SQL against it, and remove it with its volumes.
|
|
46
|
+
|
|
47
|
+
tags: [docker, execute]
|
|
48
|
+
|
|
49
|
+
requires:
|
|
50
|
+
blocks:
|
|
51
|
+
- docker.compose.up
|
|
52
|
+
- docker.compose.down
|
|
53
|
+
- docker.run
|
|
54
|
+
|
|
55
|
+
steps:
|
|
56
|
+
stack:
|
|
57
|
+
block: docker.compose.up
|
|
58
|
+
deadline: 10m
|
|
59
|
+
config:
|
|
60
|
+
# Hold the step open until pg_isready succeeds, and give up after two minutes rather
|
|
61
|
+
# than sitting on the step's whole deadline if the image never comes up at all.
|
|
62
|
+
wait: true
|
|
63
|
+
wait_timeout: 2m
|
|
64
|
+
pull: always
|
|
65
|
+
content: |
|
|
66
|
+
services:
|
|
67
|
+
db:
|
|
68
|
+
image: postgres:17-alpine
|
|
69
|
+
environment:
|
|
70
|
+
POSTGRES_USER: dirigent
|
|
71
|
+
POSTGRES_PASSWORD: dirigent
|
|
72
|
+
POSTGRES_DB: dirigent
|
|
73
|
+
healthcheck:
|
|
74
|
+
test: ["CMD-SHELL", "pg_isready -U dirigent -d dirigent"]
|
|
75
|
+
interval: 2s
|
|
76
|
+
timeout: 2s
|
|
77
|
+
retries: 15
|
|
78
|
+
|
|
79
|
+
query:
|
|
80
|
+
block: docker.run
|
|
81
|
+
depends_on: [stack]
|
|
82
|
+
deadline: 5m
|
|
83
|
+
config:
|
|
84
|
+
# The postgres image ships psql, so the client needs no image of its own; its
|
|
85
|
+
# entrypoint execs any command that is not the server itself.
|
|
86
|
+
image: postgres:17-alpine
|
|
87
|
+
pull: true
|
|
88
|
+
# Joining the project's network is what makes "db" a hostname here.
|
|
89
|
+
network: "${steps.stack.output.default_network}"
|
|
90
|
+
env:
|
|
91
|
+
PGPASSWORD: dirigent
|
|
92
|
+
# argv, not command: no shell is involved, so each statement is one argument however it
|
|
93
|
+
# is quoted. -A -t strips psql's alignment and headers, leaving rows a later step can
|
|
94
|
+
# read; ON_ERROR_STOP makes the first failing statement the container's exit code.
|
|
95
|
+
argv:
|
|
96
|
+
- psql
|
|
97
|
+
- -h
|
|
98
|
+
- db
|
|
99
|
+
- -U
|
|
100
|
+
- dirigent
|
|
101
|
+
- -d
|
|
102
|
+
- dirigent
|
|
103
|
+
- -v
|
|
104
|
+
- ON_ERROR_STOP=1
|
|
105
|
+
- -A
|
|
106
|
+
- -t
|
|
107
|
+
- -c
|
|
108
|
+
- "create table reading (station text primary key, celsius numeric)"
|
|
109
|
+
- -c
|
|
110
|
+
- "insert into reading values ('bergen', 7.1), ('oslo', 4.5)"
|
|
111
|
+
- -c
|
|
112
|
+
- "select station, celsius from reading order by station"
|
|
113
|
+
memory: 256mb
|
|
114
|
+
pids_limit: 128
|
|
115
|
+
|
|
116
|
+
teardown:
|
|
117
|
+
block: docker.compose.down
|
|
118
|
+
depends_on: [query]
|
|
119
|
+
# all_done runs the teardown on success or failure, so the stack outlives neither.
|
|
120
|
+
rule: all_done
|
|
121
|
+
deadline: 5m
|
|
122
|
+
config:
|
|
123
|
+
# The database's data volume goes with it: a run's stack leaves nothing on the worker.
|
|
124
|
+
down_volumes: true
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# A bring-up that fails, and the containers it does not leave behind.
|
|
2
|
+
#
|
|
3
|
+
# THIS PIPELINE FAILS ON PURPOSE. It is here to show what a broken stack looks like from the
|
|
4
|
+
# outside, so the expected outcome is a failed run and an empty daemon.
|
|
5
|
+
#
|
|
6
|
+
# REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
|
|
7
|
+
# tcp+TLS, or the local socket in dev). docs/docker.md is the family's home.
|
|
8
|
+
#
|
|
9
|
+
# docker.compose.up declares local_execution, so the engine refuses it unless the instance
|
|
10
|
+
# allowlists its id:
|
|
11
|
+
# dg run --local examples/docker/docker-compose-failing-up.yaml --enable-unsafe docker.compose.up
|
|
12
|
+
# export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up"]'
|
|
13
|
+
#
|
|
14
|
+
# One hop. The stack has two services: "healthy" comes up and passes its check, and "broken"
|
|
15
|
+
# comes up and never will -- its healthcheck is a command that exits non-zero, so after two
|
|
16
|
+
# retries two seconds apart the daemon marks the container unhealthy. wait: true is waiting on
|
|
17
|
+
# exactly that, so compose gives up and exits non-zero, and the step fails with what compose
|
|
18
|
+
# said on stderr. wait_timeout: 30s is the backstop for a check that hangs rather than fails.
|
|
19
|
+
#
|
|
20
|
+
# WHAT A FAILED BRING-UP LEAVES BEHIND: nothing. This is the one case a block cleans up after
|
|
21
|
+
# itself, because compose has already started containers by the time it decides the stack did
|
|
22
|
+
# not come up. On a non-zero exit -- and equally on a timeout or a cancelled step -- the block
|
|
23
|
+
# runs a down of the same project before it raises, so the half-built project goes with the
|
|
24
|
+
# failure. cleanup: true is the default; it is written out below because it is the point of
|
|
25
|
+
# the example. It never touches a bring-up that SUCCEEDED: that one persists for the run, and
|
|
26
|
+
# a docker.compose.down step is what ends it.
|
|
27
|
+
#
|
|
28
|
+
# HOW TO READ THE FAILURE. The step's error is compose's own last words -- typically
|
|
29
|
+
# "container <project>-broken-1 is unhealthy" -- and the run's step list shows the one step
|
|
30
|
+
# failed with nothing after it. The whole of the CLI's two streams are artifacts of the
|
|
31
|
+
# attempt, and the block logs whether the cleanup worked, so a stack that could not be torn
|
|
32
|
+
# down says so rather than disappearing quietly. Afterwards:
|
|
33
|
+
# docker ps -a --filter label=com.docker.compose.project
|
|
34
|
+
# lists nothing from this run.
|
|
35
|
+
#
|
|
36
|
+
# TO MAKE IT YOURS: this is the failure mode to reach for when a service of yours is slow
|
|
37
|
+
# rather than broken. Raise retries and interval, or give the service a start_period, before
|
|
38
|
+
# concluding that a stack cannot come up.
|
|
39
|
+
|
|
40
|
+
format: dirigent/v1
|
|
41
|
+
kind: pipeline
|
|
42
|
+
code: docker-compose-failing-up
|
|
43
|
+
name: A stack that will not come up
|
|
44
|
+
description: A service whose healthcheck never passes, so the bring-up fails and cleans up after itself.
|
|
45
|
+
|
|
46
|
+
tags: [docker, execute]
|
|
47
|
+
|
|
48
|
+
requires:
|
|
49
|
+
blocks:
|
|
50
|
+
- docker.compose.up
|
|
51
|
+
|
|
52
|
+
steps:
|
|
53
|
+
stack:
|
|
54
|
+
block: docker.compose.up
|
|
55
|
+
deadline: 5m
|
|
56
|
+
config:
|
|
57
|
+
wait: true
|
|
58
|
+
# Short on purpose: the example should fail in seconds, not sit out a default that was
|
|
59
|
+
# chosen for stacks meant to work.
|
|
60
|
+
wait_timeout: 30s
|
|
61
|
+
timeout: 2m
|
|
62
|
+
pull: always
|
|
63
|
+
# Tear the half-built project down before failing the step. This is the default, and
|
|
64
|
+
# it is what keeps a failed run from orphaning containers.
|
|
65
|
+
cleanup: true
|
|
66
|
+
content: |
|
|
67
|
+
services:
|
|
68
|
+
healthy:
|
|
69
|
+
image: alpine:3
|
|
70
|
+
command: ["sleep", "600"]
|
|
71
|
+
healthcheck:
|
|
72
|
+
test: ["CMD-SHELL", "exit 0"]
|
|
73
|
+
interval: 2s
|
|
74
|
+
timeout: 2s
|
|
75
|
+
retries: 2
|
|
76
|
+
broken:
|
|
77
|
+
image: alpine:3
|
|
78
|
+
command: ["sleep", "600"]
|
|
79
|
+
healthcheck:
|
|
80
|
+
test: ["CMD-SHELL", "exit 1"]
|
|
81
|
+
interval: 2s
|
|
82
|
+
timeout: 2s
|
|
83
|
+
retries: 2
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# A compose stack brought up from a FILE on the worker, not from content in the document.
|
|
2
|
+
#
|
|
3
|
+
# REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
|
|
4
|
+
# tcp+TLS, or the local socket in dev). docs/docker.md is the family's home.
|
|
5
|
+
#
|
|
6
|
+
# Every block here declares local_execution, so the engine refuses them unless the instance
|
|
7
|
+
# allowlists their ids:
|
|
8
|
+
# dg run --local examples/docker/docker-compose-file.yaml \
|
|
9
|
+
# --enable-unsafe docker.compose.up,docker.compose.down,docker.run
|
|
10
|
+
# export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up","docker.compose.down","docker.run"]'
|
|
11
|
+
#
|
|
12
|
+
# Four hops, and what each one hands on:
|
|
13
|
+
# write a container writes a compose file into stack/ in the run's work directory.
|
|
14
|
+
# Output: where each declared file landed.
|
|
15
|
+
# stack compose brought up from that path. Output: the services, the networks, and
|
|
16
|
+
# compose_file -- the path the CLI was actually given.
|
|
17
|
+
# probe a container on the stack's network, fetching the service by name.
|
|
18
|
+
# teardown the same project down again, on any outcome.
|
|
19
|
+
#
|
|
20
|
+
# WHERE FILES LIVE DURING A RUN. A run has two places. Scratch -- ${run.scratch}, a prefix
|
|
21
|
+
# under the instance's artifact root -- is storage, and is where a result that outlives the
|
|
22
|
+
# run goes. The work directory is this worker's own filesystem, and is where a tool that opens
|
|
23
|
+
# a file reads it from. Compose runs a CLI, so it names a file in the work directory and
|
|
24
|
+
# nowhere else: the path is always relative to it, never absolute, and never climbs out.
|
|
25
|
+
# "stack/docker-compose.yaml" below is the same file the write step wrote there.
|
|
26
|
+
#
|
|
27
|
+
# THE TWO FORMS. docker.compose.up takes exactly one of content or file. Inline content keeps
|
|
28
|
+
# a small stack in the document, and the block writes it into the work directory for the CLI
|
|
29
|
+
# anyway; file is the form for a document an earlier step produced -- generated here, but just
|
|
30
|
+
# as often a storage.copy out of an object store, or a repository checkout. Either way the up
|
|
31
|
+
# step reports compose_file, the path it used, so a later step needing the same document -- a
|
|
32
|
+
# down that has to resolve the project by file rather than by label -- can name it without
|
|
33
|
+
# knowing which form was used.
|
|
34
|
+
#
|
|
35
|
+
# TO MAKE IT YOURS: replace the write step with whatever actually produces your compose file,
|
|
36
|
+
# and leave the rest alone. Note that a compose file with build: stanzas needs its build
|
|
37
|
+
# contexts in the work directory too; v1 targets stacks of pre-built images.
|
|
38
|
+
|
|
39
|
+
format: dirigent/v1
|
|
40
|
+
kind: pipeline
|
|
41
|
+
code: docker-compose-file
|
|
42
|
+
name: A stack from a compose file on the worker
|
|
43
|
+
description: Write a compose file into the run's work directory, bring the stack up from that path, and tear it down.
|
|
44
|
+
|
|
45
|
+
tags: [docker, execute]
|
|
46
|
+
|
|
47
|
+
requires:
|
|
48
|
+
blocks:
|
|
49
|
+
- docker.run
|
|
50
|
+
- docker.compose.up
|
|
51
|
+
- docker.compose.down
|
|
52
|
+
|
|
53
|
+
steps:
|
|
54
|
+
write:
|
|
55
|
+
block: docker.run
|
|
56
|
+
deadline: 5m
|
|
57
|
+
config:
|
|
58
|
+
image: alpine:3
|
|
59
|
+
pull: true
|
|
60
|
+
network: none
|
|
61
|
+
# Whatever the container leaves in its output mount is copied where the target names
|
|
62
|
+
# once it exits. A target with no URI scheme is a path in the run's work directory on
|
|
63
|
+
# this worker, which is where the compose CLI opens its file from; a URI would send the
|
|
64
|
+
# file to storage instead, for a result that outlives the run.
|
|
65
|
+
outputs:
|
|
66
|
+
docker-compose.yaml: "stack/docker-compose.yaml"
|
|
67
|
+
command: |
|
|
68
|
+
cat > /dirigent/outputs/docker-compose.yaml <<'COMPOSE'
|
|
69
|
+
services:
|
|
70
|
+
web:
|
|
71
|
+
image: nginx:alpine
|
|
72
|
+
healthcheck:
|
|
73
|
+
test: ["CMD-SHELL", "wget -qO- http://localhost/ >/dev/null 2>&1 || exit 1"]
|
|
74
|
+
interval: 2s
|
|
75
|
+
timeout: 2s
|
|
76
|
+
retries: 5
|
|
77
|
+
COMPOSE
|
|
78
|
+
memory: 64mb
|
|
79
|
+
pids_limit: 64
|
|
80
|
+
|
|
81
|
+
stack:
|
|
82
|
+
block: docker.compose.up
|
|
83
|
+
depends_on: [write]
|
|
84
|
+
deadline: 10m
|
|
85
|
+
config:
|
|
86
|
+
# Relative to the run's work directory, matching what the write step produced. The CLI
|
|
87
|
+
# is always run with --project-directory set to that directory, so relative paths inside
|
|
88
|
+
# the compose file resolve against it rather than against this file's directory.
|
|
89
|
+
file: stack/docker-compose.yaml
|
|
90
|
+
wait: true
|
|
91
|
+
wait_timeout: 2m
|
|
92
|
+
pull: always
|
|
93
|
+
|
|
94
|
+
probe:
|
|
95
|
+
block: docker.run
|
|
96
|
+
depends_on: [stack]
|
|
97
|
+
deadline: 5m
|
|
98
|
+
config:
|
|
99
|
+
image: alpine:3
|
|
100
|
+
pull: true
|
|
101
|
+
# The network the up step reported: joining it resolves "web" to the service.
|
|
102
|
+
network: "${steps.stack.output.default_network}"
|
|
103
|
+
argv: [wget, -qO-, "http://web/"]
|
|
104
|
+
memory: 64mb
|
|
105
|
+
pids_limit: 64
|
|
106
|
+
|
|
107
|
+
teardown:
|
|
108
|
+
block: docker.compose.down
|
|
109
|
+
depends_on: [probe]
|
|
110
|
+
rule: all_done
|
|
111
|
+
deadline: 5m
|
|
112
|
+
config:
|
|
113
|
+
# A project tears down by its label alone, so the file is optional here; passing back
|
|
114
|
+
# the path the up step reported is how a teardown that does need -f gets it, whichever
|
|
115
|
+
# form the up step was given.
|
|
116
|
+
file: "${steps.stack.output.compose_file}"
|
|
117
|
+
down_volumes: true
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# Pipeline parameters reaching into a compose stack: an interpolated variable, and a profile.
|
|
2
|
+
#
|
|
3
|
+
# REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
|
|
4
|
+
# tcp+TLS, or the local socket in dev). docs/docker.md is the family's home.
|
|
5
|
+
#
|
|
6
|
+
# Every block here declares local_execution, so the engine refuses them unless the instance
|
|
7
|
+
# allowlists their ids:
|
|
8
|
+
# dg run --local examples/docker/docker-compose-profiles-env.yaml \
|
|
9
|
+
# --enable-unsafe docker.compose.up,docker.compose.down,docker.run
|
|
10
|
+
# export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up","docker.compose.down","docker.run"]'
|
|
11
|
+
#
|
|
12
|
+
# Three hops, and what each one hands on:
|
|
13
|
+
# stack a two-service stack where one service is optional. Output: the containers the
|
|
14
|
+
# project actually has, so the profile's effect is visible in the record.
|
|
15
|
+
# present a container on the stack's network, fetching the greeting the first service is
|
|
16
|
+
# serving and asking whether the optional one is there at all. Output: its stdout,
|
|
17
|
+
# which is the parameter's value followed by a verdict on the profile.
|
|
18
|
+
# teardown the same project down again, on any outcome.
|
|
19
|
+
#
|
|
20
|
+
# TWO WAYS A PARAMETER REACHES COMPOSE, and they are not the same mechanism:
|
|
21
|
+
# env variables set for the compose CLI process itself, which compose interpolates
|
|
22
|
+
# into the file it reads. ${params.greeting} is resolved by the engine before the
|
|
23
|
+
# block sees the config, so the value is a plain string by the time compose runs.
|
|
24
|
+
# profiles the --profile flags the CLI is given. A service that declares profiles is left
|
|
25
|
+
# out of every command that does not activate one of them, so this is how a
|
|
26
|
+
# pipeline turns an optional service on and off without two compose files.
|
|
27
|
+
#
|
|
28
|
+
# WRITE $${GREETING} FOR COMPOSE, ${...} FOR THE ENGINE. Every ${...} in a document is a
|
|
29
|
+
# reference the engine resolves, and it fails the attempt on a name it does not know, so a
|
|
30
|
+
# variable compose is meant to interpolate is written with the escape: $${GREETING} reaches
|
|
31
|
+
# compose as the literal ${GREETING}, and the engine never looks the name up.
|
|
32
|
+
#
|
|
33
|
+
# project_name is set explicitly here to show the field. It defaults to a name derived from
|
|
34
|
+
# the run id -- shared by the up, the steps that drive it and the down, and unique between
|
|
35
|
+
# runs -- so an explicit one has to keep both those promises itself, which is what ${run.id}
|
|
36
|
+
# does. A fixed string would collide with the next run of the same pipeline.
|
|
37
|
+
#
|
|
38
|
+
# TO MAKE IT YOURS: the parameter names are the only thing specific to this file. Pass
|
|
39
|
+
# -p profile=none to see the same stack come up without its optional service.
|
|
40
|
+
|
|
41
|
+
format: dirigent/v1
|
|
42
|
+
kind: pipeline
|
|
43
|
+
code: docker-compose-profiles-env
|
|
44
|
+
name: Parameters into a compose stack
|
|
45
|
+
description: A compose stack whose greeting comes from a parameter and whose optional service comes from a profile.
|
|
46
|
+
|
|
47
|
+
tags: [docker, execute]
|
|
48
|
+
|
|
49
|
+
requires:
|
|
50
|
+
blocks:
|
|
51
|
+
- docker.compose.up
|
|
52
|
+
- docker.compose.down
|
|
53
|
+
- docker.run
|
|
54
|
+
|
|
55
|
+
params:
|
|
56
|
+
type: object
|
|
57
|
+
additionalProperties: false
|
|
58
|
+
properties:
|
|
59
|
+
greeting:
|
|
60
|
+
type: string
|
|
61
|
+
default: hello from a parameter
|
|
62
|
+
description: Interpolated into the compose file as $${GREETING} and served by the greeter.
|
|
63
|
+
profile:
|
|
64
|
+
type: string
|
|
65
|
+
default: extras
|
|
66
|
+
description: The compose profile to activate; anything but "extras" leaves the reporter out.
|
|
67
|
+
|
|
68
|
+
steps:
|
|
69
|
+
stack:
|
|
70
|
+
block: docker.compose.up
|
|
71
|
+
deadline: 10m
|
|
72
|
+
config:
|
|
73
|
+
wait: true
|
|
74
|
+
wait_timeout: 2m
|
|
75
|
+
pull: always
|
|
76
|
+
project_name: "dirigent-profiles-${run.id}"
|
|
77
|
+
# A whole reference resolves to the value itself, so this is a one-element list of the
|
|
78
|
+
# parameter's string rather than a list containing a template.
|
|
79
|
+
profiles: ["${params.profile}"]
|
|
80
|
+
# Set for the CLI process, which is where compose reads interpolation from.
|
|
81
|
+
env:
|
|
82
|
+
GREETING: "${params.greeting}"
|
|
83
|
+
content: |
|
|
84
|
+
services:
|
|
85
|
+
greeter:
|
|
86
|
+
image: nginx:alpine
|
|
87
|
+
# Compose substitutes GREETING out of the env above before docker sees this, so
|
|
88
|
+
# the file the daemon runs carries the parameter's value, not the variable. The
|
|
89
|
+
# greeting is written where nginx serves from, which is how a later step reads it.
|
|
90
|
+
command:
|
|
91
|
+
- sh
|
|
92
|
+
- -c
|
|
93
|
+
- echo $${GREETING} > /usr/share/nginx/html/index.html && exec nginx -g 'daemon off;'
|
|
94
|
+
healthcheck:
|
|
95
|
+
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1/ >/dev/null 2>&1 || exit 1"]
|
|
96
|
+
interval: 2s
|
|
97
|
+
timeout: 2s
|
|
98
|
+
retries: 5
|
|
99
|
+
reporter:
|
|
100
|
+
# Declared under a profile, so this service exists only when that profile is
|
|
101
|
+
# activated. The greeter, declaring none, is always part of the project.
|
|
102
|
+
profiles: [extras]
|
|
103
|
+
image: alpine:3
|
|
104
|
+
command: ["sleep", "600"]
|
|
105
|
+
|
|
106
|
+
present:
|
|
107
|
+
block: docker.run
|
|
108
|
+
depends_on: [stack]
|
|
109
|
+
deadline: 5m
|
|
110
|
+
config:
|
|
111
|
+
image: alpine:3
|
|
112
|
+
pull: true
|
|
113
|
+
network: "${steps.stack.output.default_network}"
|
|
114
|
+
# The first line proves what compose interpolated; the second proves what the profile
|
|
115
|
+
# selected, because a name that does not resolve on the project's network is a service
|
|
116
|
+
# the profile left out.
|
|
117
|
+
argv:
|
|
118
|
+
- sh
|
|
119
|
+
- -c
|
|
120
|
+
- 'wget -qO- http://greeter/; nslookup reporter >/dev/null 2>&1 && echo "reporter is up" || echo "reporter was left out"'
|
|
121
|
+
memory: 64mb
|
|
122
|
+
pids_limit: 64
|
|
123
|
+
|
|
124
|
+
teardown:
|
|
125
|
+
block: docker.compose.down
|
|
126
|
+
depends_on: [present]
|
|
127
|
+
rule: all_done
|
|
128
|
+
deadline: 5m
|
|
129
|
+
config:
|
|
130
|
+
# The project name is not defaulted here, so the teardown has to name the same one the
|
|
131
|
+
# up step built.
|
|
132
|
+
project_name: "dirigent-profiles-${run.id}"
|
|
133
|
+
down_volumes: true
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# Bringing a compose stack up, driving it, and tearing it down even on failure.
|
|
2
|
+
#
|
|
3
|
+
# REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
|
|
4
|
+
# tcp+TLS, or the local socket in dev). docs/docker.md is the family's home; docs/operations.md
|
|
5
|
+
# says what mounting the host socket grants.
|
|
6
|
+
#
|
|
7
|
+
# docker.compose.up and docker.compose.down declare local_execution, so the engine refuses
|
|
8
|
+
# them unless the instance allowlists their ids:
|
|
9
|
+
# dg run --local examples/docker/docker-compose-stack.yaml \
|
|
10
|
+
# --enable-unsafe docker.compose.up,docker.compose.down,docker.run
|
|
11
|
+
# export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up","docker.compose.down","docker.run"]'
|
|
12
|
+
#
|
|
13
|
+
# The lifecycle is the point: up brings the stack up detached and it PERSISTS, so the drive
|
|
14
|
+
# step reaches it; down runs with rule: all_done, so the stack is torn down whether the drive
|
|
15
|
+
# step passed or failed. Both default their project name from the run id, so down finds the
|
|
16
|
+
# stack up created with no wiring.
|
|
17
|
+
|
|
18
|
+
format: dirigent/v1
|
|
19
|
+
kind: pipeline
|
|
20
|
+
code: docker-compose-stack
|
|
21
|
+
name: A compose stack, driven and torn down
|
|
22
|
+
description: Bring a two-service stack up, reach it from a container, and tear it down on any outcome.
|
|
23
|
+
|
|
24
|
+
tags: [docker, execute]
|
|
25
|
+
|
|
26
|
+
requires:
|
|
27
|
+
blocks:
|
|
28
|
+
- docker.compose.up
|
|
29
|
+
- docker.compose.down
|
|
30
|
+
- docker.run
|
|
31
|
+
|
|
32
|
+
steps:
|
|
33
|
+
stack:
|
|
34
|
+
block: docker.compose.up
|
|
35
|
+
deadline: 5m
|
|
36
|
+
config:
|
|
37
|
+
# wait: true holds the step open until every service is healthy, so the drive step
|
|
38
|
+
# never races the stack coming up. pull: always fetches the images on a fresh worker.
|
|
39
|
+
wait: true
|
|
40
|
+
pull: always
|
|
41
|
+
content: |
|
|
42
|
+
services:
|
|
43
|
+
api:
|
|
44
|
+
image: nginx:alpine
|
|
45
|
+
healthcheck:
|
|
46
|
+
test: ["CMD-SHELL", "wget -qO- http://localhost/ >/dev/null 2>&1 || exit 1"]
|
|
47
|
+
interval: 2s
|
|
48
|
+
timeout: 2s
|
|
49
|
+
retries: 5
|
|
50
|
+
|
|
51
|
+
drive:
|
|
52
|
+
block: docker.run
|
|
53
|
+
depends_on: [stack]
|
|
54
|
+
deadline: 2m
|
|
55
|
+
config:
|
|
56
|
+
image: alpine:3
|
|
57
|
+
pull: true
|
|
58
|
+
# The network the up step reported: joining it is how this container resolves and
|
|
59
|
+
# reaches the "api" service by name.
|
|
60
|
+
network: "${steps.stack.output.default_network}"
|
|
61
|
+
command: wget -qO- http://api/
|
|
62
|
+
|
|
63
|
+
teardown:
|
|
64
|
+
block: docker.compose.down
|
|
65
|
+
depends_on: [drive]
|
|
66
|
+
# all_done runs the teardown on success or failure, so a stack is never left running.
|
|
67
|
+
rule: all_done
|
|
68
|
+
deadline: 2m
|
|
69
|
+
config:
|
|
70
|
+
down_volumes: true
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Running a container as a step, and what that costs in trust.
|
|
2
|
+
#
|
|
3
|
+
# REQUIRES A DOCKER SOCKET: a Docker daemon the worker can reach, at /var/run/docker.sock
|
|
4
|
+
# or wherever DOCKER_HOST points. Without one the document still validates and applies; it
|
|
5
|
+
# fails at the step, saying the daemon could not be reached.
|
|
6
|
+
#
|
|
7
|
+
# docker.run declares local_execution, so the engine refuses it unless the instance
|
|
8
|
+
# allowlists the id. Reaching the Docker socket is reaching root on the host:
|
|
9
|
+
# dg run --local examples/docker/docker-hello.yaml --enable-unsafe docker.run
|
|
10
|
+
# export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.run"]' # for the instance
|
|
11
|
+
#
|
|
12
|
+
# The block is asynchronous: execute creates the container and starts it detached, and the
|
|
13
|
+
# engine probes it. A container that runs for an hour costs one parked row, not an hour of
|
|
14
|
+
# a worker, and the deadline stays on the step.
|
|
15
|
+
#
|
|
16
|
+
# network: none is the default: an image a document named does not reach the worker's
|
|
17
|
+
# network unless the step says so. pull: true fetches the image first, so this runs on a
|
|
18
|
+
# machine that has never seen alpine.
|
|
19
|
+
|
|
20
|
+
format: dirigent/v1
|
|
21
|
+
kind: pipeline
|
|
22
|
+
code: docker-hello
|
|
23
|
+
name: Hello from a container
|
|
24
|
+
description: Run a command in a container, with no network and the image pulled first.
|
|
25
|
+
|
|
26
|
+
tags: [docker, execute]
|
|
27
|
+
|
|
28
|
+
requires:
|
|
29
|
+
blocks:
|
|
30
|
+
- docker.run
|
|
31
|
+
|
|
32
|
+
params:
|
|
33
|
+
type: object
|
|
34
|
+
additionalProperties: false
|
|
35
|
+
properties:
|
|
36
|
+
message:
|
|
37
|
+
type: string
|
|
38
|
+
default: hello from a container
|
|
39
|
+
description: What the container echoes.
|
|
40
|
+
|
|
41
|
+
steps:
|
|
42
|
+
greet:
|
|
43
|
+
block: docker.run
|
|
44
|
+
deadline: 5m
|
|
45
|
+
config:
|
|
46
|
+
image: alpine:3
|
|
47
|
+
pull: true
|
|
48
|
+
# A container that needs nothing from the network is given none, and the memory and
|
|
49
|
+
# process caps bound what a runaway image can take from the worker it landed on.
|
|
50
|
+
network: none
|
|
51
|
+
argv: [echo, "${params.message}"]
|
|
52
|
+
memory: 64mb
|
|
53
|
+
pids_limit: 64
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# Naming the daemon: the same compose stack, run on a daemon a connection points at.
|
|
2
|
+
#
|
|
3
|
+
# The compose lifecycle example (docker-compose-stack.yaml) uses whatever daemon the worker's
|
|
4
|
+
# own environment names. This one names it in the document instead, through a `docker`
|
|
5
|
+
# connection the instance holds, and that connection carries the client TLS the daemon
|
|
6
|
+
# demands.
|
|
7
|
+
#
|
|
8
|
+
# REQUIRES A DOCKER CONNECTION the instance already holds. Create it once, with the client
|
|
9
|
+
# half of the daemon's certificates -- a dind sidecar writes them to DOCKER_TLS_CERTDIR/client:
|
|
10
|
+
# dg connection create docker build-daemon \
|
|
11
|
+
# --set host=tcp://docker:2376 \
|
|
12
|
+
# --set "tls_ca=$(cat /certs/client/ca.pem)" \
|
|
13
|
+
# --set "tls_cert=$(cat /certs/client/cert.pem)" \
|
|
14
|
+
# --set "tls_key=$(cat /certs/client/key.pem)"
|
|
15
|
+
#
|
|
16
|
+
# The three blocks declare local_execution, so the engine refuses them unless the instance
|
|
17
|
+
# allowlists their ids:
|
|
18
|
+
# export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up","docker.compose.down","docker.run"]'
|
|
19
|
+
#
|
|
20
|
+
# EVERY STEP NAMES THE SAME CONNECTION, and that is the whole discipline here. A stack exists
|
|
21
|
+
# only on the daemon that was told to create it, so an `up` on one daemon and a `down` on
|
|
22
|
+
# another tear down nothing: the second daemon has no such project. The drive step is in it
|
|
23
|
+
# too, because it joins a network the first daemon owns.
|
|
24
|
+
#
|
|
25
|
+
# tls_key is sealed: encrypted at rest, redacted in every API response, and reaching the CLI
|
|
26
|
+
# only as a 0600 file under the run's work directory that goes when the step leaves. A client
|
|
27
|
+
# key for a docker daemon is a credential for root on whatever that daemon runs on.
|
|
28
|
+
#
|
|
29
|
+
# TO MAKE IT YOURS: point host at your own daemon. A daemon that needs no TLS -- an ssh://
|
|
30
|
+
# host, or a unix socket on another path -- takes host alone and none of the three PEMs.
|
|
31
|
+
|
|
32
|
+
format: dirigent/v1
|
|
33
|
+
kind: pipeline
|
|
34
|
+
code: docker-remote-daemon
|
|
35
|
+
name: A compose stack on a named daemon
|
|
36
|
+
description: Bring a stack up, drive it and tear it down on the daemon a docker connection names.
|
|
37
|
+
|
|
38
|
+
tags: [docker, execute]
|
|
39
|
+
|
|
40
|
+
requires:
|
|
41
|
+
blocks:
|
|
42
|
+
- docker.compose.up
|
|
43
|
+
- docker.compose.down
|
|
44
|
+
- docker.run
|
|
45
|
+
workers:
|
|
46
|
+
# The compose file and its build contexts are on the worker's own filesystem, so all three
|
|
47
|
+
# steps have to meet one docker-capable worker even though the daemon is named here.
|
|
48
|
+
- docker
|
|
49
|
+
connections:
|
|
50
|
+
# Named, not carried: the instance holds the connection and its sealed TLS key, and this
|
|
51
|
+
# document refuses to apply where it is absent rather than failing the first time it runs.
|
|
52
|
+
- build-daemon
|
|
53
|
+
|
|
54
|
+
steps:
|
|
55
|
+
stack:
|
|
56
|
+
block: docker.compose.up
|
|
57
|
+
deadline: 5m
|
|
58
|
+
config:
|
|
59
|
+
connection: build-daemon
|
|
60
|
+
wait: true
|
|
61
|
+
pull: always
|
|
62
|
+
content: |
|
|
63
|
+
services:
|
|
64
|
+
api:
|
|
65
|
+
image: nginx:alpine
|
|
66
|
+
healthcheck:
|
|
67
|
+
test: ["CMD-SHELL", "wget -qO- http://localhost/ >/dev/null 2>&1 || exit 1"]
|
|
68
|
+
interval: 2s
|
|
69
|
+
timeout: 2s
|
|
70
|
+
retries: 5
|
|
71
|
+
|
|
72
|
+
drive:
|
|
73
|
+
block: docker.run
|
|
74
|
+
depends_on: [stack]
|
|
75
|
+
deadline: 2m
|
|
76
|
+
config:
|
|
77
|
+
connection: build-daemon
|
|
78
|
+
image: alpine:3
|
|
79
|
+
pull: true
|
|
80
|
+
# The network the up step reported, which exists on that daemon and nowhere else.
|
|
81
|
+
network: "${steps.stack.output.default_network}"
|
|
82
|
+
command: wget -qO- http://api/
|
|
83
|
+
|
|
84
|
+
teardown:
|
|
85
|
+
block: docker.compose.down
|
|
86
|
+
depends_on: [drive]
|
|
87
|
+
# all_done runs the teardown on success or failure, so a stack is never left running.
|
|
88
|
+
rule: all_done
|
|
89
|
+
deadline: 2m
|
|
90
|
+
config:
|
|
91
|
+
connection: build-daemon
|
|
92
|
+
down_volumes: true
|