dirigent-examples 0.15.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dirigent_examples/__init__.py +22 -0
- dirigent_examples/py.typed +0 -0
- dirigent_examples/shelves/README.md +299 -0
- dirigent_examples/shelves/composition/README.md +18 -0
- dirigent_examples/shelves/composition/chained-instances.yaml +97 -0
- dirigent_examples/shelves/composition/composition-child.yaml +64 -0
- dirigent_examples/shelves/composition/composition-parent.yaml +89 -0
- dirigent_examples/shelves/connections.yaml +52 -0
- dirigent_examples/shelves/demo/README.md +19 -0
- dirigent_examples/shelves/demo/markdown-showcase.yaml +117 -0
- dirigent_examples/shelves/demo/params-showcase.yaml +92 -0
- dirigent_examples/shelves/demo/requires.yaml +65 -0
- dirigent_examples/shelves/demo/weekly-import-malawi.yaml +54 -0
- dirigent_examples/shelves/demo/weekly-import-nepal.yaml +75 -0
- dirigent_examples/shelves/docker/README.md +29 -0
- dirigent_examples/shelves/docker/docker-build-push.yaml +110 -0
- dirigent_examples/shelves/docker/docker-build-run.yaml +106 -0
- dirigent_examples/shelves/docker/docker-compose-database.yaml +124 -0
- dirigent_examples/shelves/docker/docker-compose-failing-up.yaml +83 -0
- dirigent_examples/shelves/docker/docker-compose-file.yaml +117 -0
- dirigent_examples/shelves/docker/docker-compose-profiles-env.yaml +133 -0
- dirigent_examples/shelves/docker/docker-compose-stack.yaml +70 -0
- dirigent_examples/shelves/docker/docker-hello.yaml +53 -0
- dirigent_examples/shelves/docker/docker-remote-daemon.yaml +92 -0
- dirigent_examples/shelves/docker/docker-run-failing-teardown.yaml +94 -0
- dirigent_examples/shelves/docker/docker-ticker.yaml +49 -0
- dirigent_examples/shelves/execute/README.md +16 -0
- dirigent_examples/shelves/execute/long-log.yaml +89 -0
- dirigent_examples/shelves/failure/README.md +20 -0
- dirigent_examples/shelves/failure/error-handler.yaml +89 -0
- dirigent_examples/shelves/failure/optional-step.yaml +82 -0
- dirigent_examples/shelves/failure/retries.yaml +75 -0
- dirigent_examples/shelves/failure/retry-budget.yaml +82 -0
- dirigent_examples/shelves/failure/step-timeout.yaml +96 -0
- dirigent_examples/shelves/git/README.md +32 -0
- dirigent_examples/shelves/git/git-checkout-build.yaml +125 -0
- dirigent_examples/shelves/git/git-checkout-compose.yaml +138 -0
- dirigent_examples/shelves/git/git-checkout-public.yaml +84 -0
- dirigent_examples/shelves/graph/README.md +22 -0
- dirigent_examples/shelves/graph/deep-chain.yaml +119 -0
- dirigent_examples/shelves/graph/fan-in.yaml +80 -0
- dirigent_examples/shelves/graph/fan-out.yaml +66 -0
- dirigent_examples/shelves/graph/linear.yaml +66 -0
- dirigent_examples/shelves/graph/parallel-branches.yaml +57 -0
- dirigent_examples/shelves/graph/parallel-sleep.yaml +55 -0
- dirigent_examples/shelves/graph/skip-diamond.yaml +105 -0
- dirigent_examples/shelves/graph/wide-fan.yaml +147 -0
- dirigent_examples/shelves/hello-world.yaml +30 -0
- dirigent_examples/shelves/open-data/README.md +67 -0
- dirigent_examples/shelves/open-data/feeds-composition.yaml +120 -0
- dirigent_examples/shelves/open-data/gdacs-disaster-updates.yaml +230 -0
- dirigent_examples/shelves/open-data/github-releases-relay.yaml +225 -0
- dirigent_examples/shelves/open-data/hdx-dataset-watch.yaml +211 -0
- dirigent_examples/shelves/open-data/kobo-submissions-to-csv.yaml +149 -0
- dirigent_examples/shelves/open-data/nominatim-geocode-facilities.yaml +169 -0
- dirigent_examples/shelves/open-data/odk-central-submissions.yaml +146 -0
- dirigent_examples/shelves/open-data/open-meteo-weekly-report.yaml +142 -0
- dirigent_examples/shelves/open-data/overpass-health-facilities.yaml +154 -0
- dirigent_examples/shelves/open-data/usgs-earthquakes-alert.yaml +208 -0
- dirigent_examples/shelves/open-data/who-gho-indicators-to-parquet.yaml +163 -0
- dirigent_examples/shelves/open-data/wikidata-country-reference.yaml +146 -0
- dirigent_examples/shelves/open-data/world-bank-population-trend.yaml +166 -0
- dirigent_examples/shelves/patterns/README.md +144 -0
- dirigent_examples/shelves/patterns/concurrency-queue.yaml +89 -0
- dirigent_examples/shelves/patterns/concurrency-replace.yaml +92 -0
- dirigent_examples/shelves/patterns/concurrency-skip.yaml +97 -0
- dirigent_examples/shelves/patterns/connections-referenced-vs-carried.yaml +140 -0
- dirigent_examples/shelves/patterns/deadline-on-a-sensor.yaml +106 -0
- dirigent_examples/shelves/patterns/fan-out-continue.yaml +88 -0
- dirigent_examples/shelves/patterns/fan-out-fail-fast.yaml +80 -0
- dirigent_examples/shelves/patterns/fan-out-from-params.yaml +84 -0
- dirigent_examples/shelves/patterns/fan-out-item-wise.yaml +111 -0
- dirigent_examples/shelves/patterns/fan-out-literal-list.yaml +81 -0
- dirigent_examples/shelves/patterns/fan-out-nested-objects.yaml +107 -0
- dirigent_examples/shelves/patterns/fan-out-then-join.yaml +86 -0
- dirigent_examples/shelves/patterns/log-levels.yaml +119 -0
- dirigent_examples/shelves/patterns/outputs-inline-vs-storage.yaml +140 -0
- dirigent_examples/shelves/patterns/params-every-type.yaml +259 -0
- dirigent_examples/shelves/patterns/params-validation-refuses.yaml +131 -0
- dirigent_examples/shelves/patterns/pipeline-run-child.yaml +96 -0
- dirigent_examples/shelves/patterns/pipeline-run-fire-and-forget.yaml +99 -0
- dirigent_examples/shelves/patterns/pipeline-run-strict.yaml +103 -0
- dirigent_examples/shelves/patterns/pipeline-run-wait.yaml +107 -0
- dirigent_examples/shelves/patterns/pipeline-run-with-params.yaml +124 -0
- dirigent_examples/shelves/patterns/poll-cadence.yaml +102 -0
- dirigent_examples/shelves/patterns/priority-layered.yaml +120 -0
- dirigent_examples/shelves/patterns/references-cheat-sheet.yaml +186 -0
- dirigent_examples/shelves/patterns/retry-budget-exhausted.yaml +92 -0
- dirigent_examples/shelves/patterns/retry-exponential-backoff.yaml +88 -0
- dirigent_examples/shelves/patterns/retry-only-transient.yaml +108 -0
- dirigent_examples/shelves/patterns/retry-with-jitter.yaml +102 -0
- dirigent_examples/shelves/patterns/rule-all-done.yaml +83 -0
- dirigent_examples/shelves/patterns/rule-all-success.yaml +79 -0
- dirigent_examples/shelves/patterns/rule-always.yaml +92 -0
- dirigent_examples/shelves/patterns/rule-one-failed.yaml +86 -0
- dirigent_examples/shelves/patterns/schedule-at-once.yaml +110 -0
- dirigent_examples/shelves/patterns/schedule-cron-timezone.yaml +121 -0
- dirigent_examples/shelves/patterns/schedule-interval.yaml +109 -0
- dirigent_examples/shelves/patterns/schedule-window-half-open.yaml +105 -0
- dirigent_examples/shelves/patterns/sensor-http-ready.yaml +121 -0
- dirigent_examples/shelves/patterns/sensor-storage-exists.yaml +134 -0
- dirigent_examples/shelves/patterns/step-names-and-keys.yaml +99 -0
- dirigent_examples/shelves/patterns/timeout-fails-the-step.yaml +94 -0
- dirigent_examples/shelves/patterns/timeout-skips-the-step.yaml +102 -0
- dirigent_examples/shelves/patterns/webhook-mapping-nested-payload.yaml +125 -0
- dirigent_examples/shelves/patterns/webhook-signed.yaml +144 -0
- dirigent_examples/shelves/preview/s3-parquet-to-ingestion.yaml +92 -0
- dirigent_examples/shelves/python/README.md +31 -0
- dirigent_examples/shelves/python/apply_and_run.py +52 -0
- dirigent_examples/shelves/python/ci_gate.py +76 -0
- dirigent_examples/shelves/python/connections.py +61 -0
- dirigent_examples/shelves/python/error_handling.py +84 -0
- dirigent_examples/shelves/python/follow_logs.py +39 -0
- dirigent_examples/shelves/python/list_and_filter.py +52 -0
- dirigent_examples/shelves/queues/README.md +59 -0
- dirigent_examples/shelves/queues/kafka-consume-then-transform.yaml +105 -0
- dirigent_examples/shelves/queues/kafka-produce-then-consume.yaml +124 -0
- dirigent_examples/shelves/queues/rabbitmq-consume-ack-on-success.yaml +102 -0
- dirigent_examples/shelves/queues/report-to-kafka.yaml +105 -0
- dirigent_examples/shelves/queues/report-to-rabbitmq.yaml +105 -0
- dirigent_examples/shelves/recipes/README.md +130 -0
- dirigent_examples/shelves/recipes/csv-header-rules.yaml +147 -0
- dirigent_examples/shelves/recipes/csv-to-ndjson.yaml +107 -0
- dirigent_examples/shelves/recipes/etl-csv-clean-validate-parquet.yaml +207 -0
- dirigent_examples/shelves/recipes/filter-by-predicate.yaml +116 -0
- dirigent_examples/shelves/recipes/filter-then-map-then-reduce.yaml +109 -0
- dirigent_examples/shelves/recipes/http-fetch-validate-post.yaml +142 -0
- dirigent_examples/shelves/recipes/http-follow-redirects.yaml +100 -0
- dirigent_examples/shelves/recipes/http-get-with-query.yaml +96 -0
- dirigent_examples/shelves/recipes/http-headers-and-auth-connection.yaml +110 -0
- dirigent_examples/shelves/recipes/http-post-file-from-storage.yaml +117 -0
- dirigent_examples/shelves/recipes/http-post-json-echo.yaml +105 -0
- dirigent_examples/shelves/recipes/http-post-report.yaml +183 -0
- dirigent_examples/shelves/recipes/http-save-body-to-storage.yaml +106 -0
- dirigent_examples/shelves/recipes/http-success-status-list.yaml +80 -0
- dirigent_examples/shelves/recipes/http-timeout-override.yaml +104 -0
- dirigent_examples/shelves/recipes/jq-dedupe-by-key.yaml +78 -0
- dirigent_examples/shelves/recipes/jq-defaults-and-nulls.yaml +91 -0
- dirigent_examples/shelves/recipes/jq-group-by-and-sum.yaml +74 -0
- dirigent_examples/shelves/recipes/jq-join-two-lists.yaml +77 -0
- dirigent_examples/shelves/recipes/jq-long-to-wide.yaml +76 -0
- dirigent_examples/shelves/recipes/jq-nested-to-flat.yaml +89 -0
- dirigent_examples/shelves/recipes/jq-pivot-wide-to-long.yaml +65 -0
- dirigent_examples/shelves/recipes/jq-running-totals.yaml +82 -0
- dirigent_examples/shelves/recipes/jq-string-cleaning.yaml +88 -0
- dirigent_examples/shelves/recipes/jq-top-n.yaml +85 -0
- dirigent_examples/shelves/recipes/jq-validate-in-jq-vs-schema.yaml +124 -0
- dirigent_examples/shelves/recipes/jq-window-dates.yaml +82 -0
- dirigent_examples/shelves/recipes/json-to-csv-flattening.yaml +141 -0
- dirigent_examples/shelves/recipes/large-output-to-storage.yaml +134 -0
- dirigent_examples/shelves/recipes/map-enrich-with-lookup.yaml +96 -0
- dirigent_examples/shelves/recipes/ndjson-to-parquet.yaml +130 -0
- dirigent_examples/shelves/recipes/pagination-by-fan-out.yaml +115 -0
- dirigent_examples/shelves/recipes/parquet-round-trip-types.yaml +163 -0
- dirigent_examples/shelves/recipes/reconcile-two-sources.yaml +159 -0
- dirigent_examples/shelves/recipes/report-built-in.yaml +72 -0
- dirigent_examples/shelves/recipes/report-daily-digest.yaml +186 -0
- dirigent_examples/shelves/recipes/report-to-file.yaml +131 -0
- dirigent_examples/shelves/recipes/report-to-webhook.yaml +136 -0
- dirigent_examples/shelves/recipes/schema-carried.yaml +112 -0
- dirigent_examples/shelves/recipes/schema-formats.yaml +107 -0
- dirigent_examples/shelves/recipes/schema-referenced.yaml +86 -0
- dirigent_examples/shelves/recipes/schema-refuses-then-rule.yaml +127 -0
- dirigent_examples/shelves/recipes/storage-copy-dated-archive.yaml +114 -0
- dirigent_examples/shelves/recipes/storage-exists-gate.yaml +127 -0
- dirigent_examples/shelves/recipes/storage-manifest-of-a-fan-out.yaml +104 -0
- dirigent_examples/shelves/recipes/storage-write-then-read.yaml +119 -0
- dirigent_examples/shelves/recipes/webhook-post-hmac.yaml +132 -0
- dirigent_examples/shelves/recipes/webhook-post-summary.yaml +142 -0
- dirigent_examples/shelves/s3/README.md +34 -0
- dirigent_examples/shelves/s3/report-to-s3.yaml +93 -0
- dirigent_examples/shelves/s3/s3-copy-and-verify.yaml +105 -0
- dirigent_examples/shelves/s3/s3-csv-report.yaml +87 -0
- dirigent_examples/shelves/s3/s3-parquet-report.yaml +77 -0
- dirigent_examples/shelves/s3/s3-round-trip.yaml +125 -0
- dirigent_examples/shelves/schemas/README.md +36 -0
- dirigent_examples/shelves/schemas/echo-reading.json +18 -0
- dirigent_examples/shelves/schemas/ou-record.json +13 -0
- dirigent_examples/shelves/schemas/station-reading.json +13 -0
- dirigent_examples/shelves/sensors/README.md +16 -0
- dirigent_examples/shelves/sensors/sensor-gate.yaml +65 -0
- dirigent_examples/shelves/sensors/time-window.yaml +61 -0
- dirigent_examples/shelves/sql/README.md +52 -0
- dirigent_examples/shelves/sql/duckdb-parquet-to-report.yaml +146 -0
- dirigent_examples/shelves/sql/sql-postgres-readonly.yaml +111 -0
- dirigent_examples/shelves/sql/sql-query-to-storage.yaml +85 -0
- dirigent_examples/shelves/sql/sql-sqlite-roundtrip.yaml +114 -0
- dirigent_examples/shelves/sql/warehouse.sql +42 -0
- dirigent_examples/shelves/transform/README.md +36 -0
- dirigent_examples/shelves/transform/csv-report.yaml +55 -0
- dirigent_examples/shelves/transform/jq-filter-and-map.yaml +70 -0
- dirigent_examples/shelves/transform/jq-group-and-aggregate.yaml +70 -0
- dirigent_examples/shelves/transform/jq-join-two-sources.yaml +98 -0
- dirigent_examples/shelves/transform/jq-reshape.yaml +91 -0
- dirigent_examples/shelves/transform/jq-stream-through-storage.yaml +112 -0
- dirigent_examples/shelves/transform/ndjson-round-trip.yaml +56 -0
- dirigent_examples/shelves/transform/parquet-round-trip.yaml +68 -0
- dirigent_examples/shelves/transform/std-convert-fan-out.yaml +142 -0
- dirigent_examples/shelves/transform/xml-feed-to-ndjson.yaml +116 -0
- dirigent_examples/shelves/transform/yaml-config-to-json.yaml +104 -0
- dirigent_examples/shelves/triggers/README.md +45 -0
- dirigent_examples/shelves/triggers/at-one-time.yaml +78 -0
- dirigent_examples/shelves/triggers/cron-nightly.yaml +79 -0
- dirigent_examples/shelves/triggers/cron-windowed.yaml +86 -0
- dirigent_examples/shelves/triggers/document-nightly.yaml +80 -0
- dirigent_examples/shelves/triggers/interval-rolling.yaml +88 -0
- dirigent_examples/shelves/triggers/managed-and-manual.yaml +109 -0
- dirigent_examples/shelves/triggers/webhook-trigger.yaml +75 -0
- dirigent_examples/shelves/validate/README.md +31 -0
- dirigent_examples/shelves/validate/expects-a-shape.yaml +56 -0
- dirigent_examples/shelves/validate/the-shape-is-wrong.yaml +46 -0
- dirigent_examples-0.15.0.dist-info/METADATA +21 -0
- dirigent_examples-0.15.0.dist-info/RECORD +216 -0
- dirigent_examples-0.15.0.dist-info/WHEEL +4 -0
- dirigent_examples-0.15.0.dist-info/entry_points.txt +3 -0
- dirigent_examples-0.15.0.dist-info/licenses/LICENSE +18 -0
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# A description that exercises every markdown construct the UI claims to render.
|
|
2
|
+
#
|
|
3
|
+
# A pipeline's description is long-form and markdown-capable, and it is the body of the
|
|
4
|
+
# pipeline's screen. Most examples use one paragraph of it. This one uses all of it, so that
|
|
5
|
+
# a renderer change has something to fail against: headings, both kinds of list, nested
|
|
6
|
+
# items, a task list, inline code, a fenced block with a language, a fenced block without
|
|
7
|
+
# one, a table with alignment, a blockquote, a horizontal rule, emphasis, a link, and a long
|
|
8
|
+
# paragraph that has to wrap.
|
|
9
|
+
#
|
|
10
|
+
# It is a real pipeline rather than a stub, because a document that cannot run has no
|
|
11
|
+
# business in examples/ -- the two steps below reshape a small payload and need no network
|
|
12
|
+
# and no allowlist entry. The description is the exhibit; the steps are what make it legal.
|
|
13
|
+
#
|
|
14
|
+
# The rendering rule the corpus follows: a screen's title is the name if there is one and
|
|
15
|
+
# the code otherwise, the code is always on screen in mono and never drawn twice, and the
|
|
16
|
+
# description is the body. So this document carries all three, and the body is what to look
|
|
17
|
+
# at.
|
|
18
|
+
#
|
|
19
|
+
# dg run --local examples/demo/markdown-showcase.yaml
|
|
20
|
+
# dg pipelines show markdown-showcase # the description, rendered
|
|
21
|
+
|
|
22
|
+
format: dirigent/v1
|
|
23
|
+
kind: pipeline
|
|
24
|
+
code: markdown-showcase
|
|
25
|
+
name: Markdown showcase
|
|
26
|
+
description: |
|
|
27
|
+
A pipeline whose **description** is the point. Everything below is one markdown document,
|
|
28
|
+
rendered as the body of this pipeline's screen.
|
|
29
|
+
|
|
30
|
+
## Headings, and what they are for
|
|
31
|
+
|
|
32
|
+
A description long enough to need sections should have them. This is an `h2`; the format
|
|
33
|
+
puts no ceiling on depth, and a renderer is expected to keep the hierarchy visible.
|
|
34
|
+
|
|
35
|
+
### A third level
|
|
36
|
+
|
|
37
|
+
Which exists mainly so the scale between levels can be judged.
|
|
38
|
+
|
|
39
|
+
## Lists
|
|
40
|
+
|
|
41
|
+
Unordered, with a nested level:
|
|
42
|
+
|
|
43
|
+
- Extract, from a source that may or may not be awake
|
|
44
|
+
- Reshape, which is where most of the work actually is
|
|
45
|
+
- `transform.jq` for one whole value
|
|
46
|
+
- `map.jq` for one element at a time
|
|
47
|
+
- `filter.jq` when the answer is a shorter list
|
|
48
|
+
- Load, and then say so
|
|
49
|
+
|
|
50
|
+
Ordered, because order sometimes matters:
|
|
51
|
+
|
|
52
|
+
1. Apply the document
|
|
53
|
+
2. Run it once by hand
|
|
54
|
+
3. Only then give it a clock
|
|
55
|
+
|
|
56
|
+
## Code
|
|
57
|
+
|
|
58
|
+
Inline code such as `dg run markdown-showcase --watch` sits inside a sentence. A fenced
|
|
59
|
+
block with a language gets highlighted:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
dg apply examples/demo/markdown-showcase.yaml
|
|
63
|
+
dg run markdown-showcase --watch
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
And one without a language does not, which is the correct treatment for output:
|
|
67
|
+
|
|
68
|
+
```
|
|
69
|
+
{"kind": "run", "status": "succeeded", "duration_ms": 412}
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## A table
|
|
73
|
+
|
|
74
|
+
| Block | Reads | Writes | Needs the allowlist |
|
|
75
|
+
| --- | :---: | ---: | --- |
|
|
76
|
+
| `transform.jq` | one value | one value | no |
|
|
77
|
+
| `http.request` | a URL | an inline body or storage | no |
|
|
78
|
+
| `shell.run` | a command | stdout and stderr | **yes** |
|
|
79
|
+
| `storage.copy` | a URI | a URI | no |
|
|
80
|
+
|
|
81
|
+
## A quote, and a rule
|
|
82
|
+
|
|
83
|
+
> A run is a row, not a process. It survives every restart, and it is queryable long
|
|
84
|
+
> after the machine that executed it is gone.
|
|
85
|
+
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
## Emphasis, links, and wrapping
|
|
89
|
+
|
|
90
|
+
Text can be *emphasised*, **strong**, or ***both at once***, and it can carry a link to
|
|
91
|
+
[the transforms page](../docs/transforms.md). This last paragraph is deliberately long so
|
|
92
|
+
that soft wrapping has something to do with it: a description is written as prose in a
|
|
93
|
+
file where the lines are hard-wrapped at a sensible width for reading in an editor, and
|
|
94
|
+
it must arrive on screen as one flowing paragraph rather than as a column of short lines,
|
|
95
|
+
because the editor's line breaks are an accident of the file and not a fact about the
|
|
96
|
+
text.
|
|
97
|
+
|
|
98
|
+
tags: [demo, transform]
|
|
99
|
+
|
|
100
|
+
requires:
|
|
101
|
+
blocks:
|
|
102
|
+
- transform.jq
|
|
103
|
+
|
|
104
|
+
steps:
|
|
105
|
+
collect:
|
|
106
|
+
block: transform.jq
|
|
107
|
+
config:
|
|
108
|
+
input:
|
|
109
|
+
constructs: [headings, lists, code, table, quote, rule, emphasis]
|
|
110
|
+
program: "{counted: (.constructs | length), constructs: .constructs}"
|
|
111
|
+
|
|
112
|
+
summarise:
|
|
113
|
+
block: transform.jq
|
|
114
|
+
depends_on: [collect]
|
|
115
|
+
config:
|
|
116
|
+
input: "${steps.collect.output.value}"
|
|
117
|
+
program: "{note: \"\\(.counted) markdown constructs, all of them in the description\"}"
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# The parameter schema is plain JSON Schema, and it is load-bearing in three places.
|
|
2
|
+
#
|
|
3
|
+
# It validates every run's parameters, whoever started it: the CLI, the API, a schedule's
|
|
4
|
+
# overrides, or a webhook's mapped payload. It is what the UI renders the run dialog from,
|
|
5
|
+
# so an enum becomes a select and a date format becomes a date picker without any UI code
|
|
6
|
+
# knowing this pipeline exists. And apply checks every ${params.x} in the document against
|
|
7
|
+
# it, so a typo is caught when the document is applied rather than when it fires at 05:00.
|
|
8
|
+
#
|
|
9
|
+
# Defaults are filled in before validation, so a parameter with a default is optional.
|
|
10
|
+
#
|
|
11
|
+
# The CLI reads the schema before it sends anything, so a value is coerced to the type the
|
|
12
|
+
# schema declares. Three ways to pass parameters, all ending in the same validated object:
|
|
13
|
+
#
|
|
14
|
+
# dg run --local examples/demo/params-showcase.yaml -p day=2026-01-01 -p dataset=climate
|
|
15
|
+
#
|
|
16
|
+
# # dotted keys address a nested leaf, and the declared type decides the coercion:
|
|
17
|
+
# # batch_size is an integer, so -p batch_size=1000 sends 1000, not "1000"
|
|
18
|
+
# dg run --local examples/demo/params-showcase.yaml \
|
|
19
|
+
# -p day=2026-01-01 -p dataset=climate -p batch_size=1000 -p dry_run=true
|
|
20
|
+
#
|
|
21
|
+
# # an array or an object is given whole, inline, in JSON or YAML:
|
|
22
|
+
# dg run --local examples/demo/params-showcase.yaml \
|
|
23
|
+
# -p day=2026-01-01 -p dataset=climate -p regions='[east, west, north]'
|
|
24
|
+
#
|
|
25
|
+
# # or a whole payload from a file, with flags overriding it:
|
|
26
|
+
# dg run --local examples/demo/params-showcase.yaml -P params.yaml -p dataset=incidence
|
|
27
|
+
#
|
|
28
|
+
# Array indices are not addressable from a dotted key: -p regions.0=east and
|
|
29
|
+
# -p regions[0]=east are both refused, because a dotted path cannot tell an index from an
|
|
30
|
+
# object key named "0", and brackets are reserved in case element addressing is ever added.
|
|
31
|
+
|
|
32
|
+
format: dirigent/v1
|
|
33
|
+
kind: pipeline
|
|
34
|
+
code: params-showcase
|
|
35
|
+
name: Every parameter shape
|
|
36
|
+
description: Everything a run form is built from, in one parameter schema.
|
|
37
|
+
|
|
38
|
+
tags: [demo, http]
|
|
39
|
+
|
|
40
|
+
params:
|
|
41
|
+
type: object
|
|
42
|
+
required: [day, dataset]
|
|
43
|
+
properties:
|
|
44
|
+
day:
|
|
45
|
+
type: string
|
|
46
|
+
description: The logical date this run covers.
|
|
47
|
+
format: date
|
|
48
|
+
|
|
49
|
+
dataset:
|
|
50
|
+
type: string
|
|
51
|
+
description: Which dataset to process.
|
|
52
|
+
enum: [climate, population, incidence]
|
|
53
|
+
|
|
54
|
+
regions:
|
|
55
|
+
type: array
|
|
56
|
+
description: Regions to process; every one becomes its own run item.
|
|
57
|
+
default: [east, west]
|
|
58
|
+
items:
|
|
59
|
+
type: string
|
|
60
|
+
|
|
61
|
+
batch_size:
|
|
62
|
+
type: integer
|
|
63
|
+
description: How many records to send per request.
|
|
64
|
+
default: 500
|
|
65
|
+
minimum: 1
|
|
66
|
+
maximum: 10000
|
|
67
|
+
|
|
68
|
+
dry_run:
|
|
69
|
+
type: boolean
|
|
70
|
+
description: Validate the payload without writing anything downstream.
|
|
71
|
+
default: false
|
|
72
|
+
|
|
73
|
+
steps:
|
|
74
|
+
process:
|
|
75
|
+
block: http.request
|
|
76
|
+
# The fan-out is over a parameter, so how wide the run is is decided when it starts
|
|
77
|
+
# rather than when the document is written.
|
|
78
|
+
for_each: "${params.regions}"
|
|
79
|
+
items: continue
|
|
80
|
+
config:
|
|
81
|
+
url: https://postman-echo.com/post
|
|
82
|
+
method: POST
|
|
83
|
+
query:
|
|
84
|
+
day: "${params.day}"
|
|
85
|
+
dataset: "${params.dataset}"
|
|
86
|
+
region: "${item}"
|
|
87
|
+
batch_size: "${params.batch_size}"
|
|
88
|
+
dry_run: "${params.dry_run}"
|
|
89
|
+
body:
|
|
90
|
+
day: "${params.day}"
|
|
91
|
+
dataset: "${params.dataset}"
|
|
92
|
+
region: "${item}"
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# The requires preflight, and referencing a connection by name.
|
|
2
|
+
#
|
|
3
|
+
# Documents are portable because everything is referenced by name -- block ids, connection
|
|
4
|
+
# names, other pipelines -- and never by id. That portability only works if applying a
|
|
5
|
+
# document to an instance that is missing something says so once, with the whole list,
|
|
6
|
+
# rather than one unknown-block error at a time.
|
|
7
|
+
#
|
|
8
|
+
# apply checks requires first: everything named here must exist on the target instance.
|
|
9
|
+
#
|
|
10
|
+
# The connection carries the credentials, and the document carries only its name. Create it
|
|
11
|
+
# once on a server:
|
|
12
|
+
# dg connection create http postman-echo --set base_url=https://postman-echo.com
|
|
13
|
+
#
|
|
14
|
+
# Or hand it to a local run, which has no server to hold it:
|
|
15
|
+
# dg run --local examples/demo/requires.yaml --connections examples/connections.yaml
|
|
16
|
+
|
|
17
|
+
format: dirigent/v1
|
|
18
|
+
kind: pipeline
|
|
19
|
+
code: requires
|
|
20
|
+
name: Declared requirements
|
|
21
|
+
description: Declare the blocks and connections this document cannot run without.
|
|
22
|
+
|
|
23
|
+
tags: [demo, http, storage]
|
|
24
|
+
|
|
25
|
+
requires:
|
|
26
|
+
blocks:
|
|
27
|
+
- http.request
|
|
28
|
+
- storage.write
|
|
29
|
+
- storage.copy
|
|
30
|
+
connections:
|
|
31
|
+
- postman-echo
|
|
32
|
+
|
|
33
|
+
steps:
|
|
34
|
+
authenticate:
|
|
35
|
+
block: http.request
|
|
36
|
+
config:
|
|
37
|
+
# The connection holds the base URL and the credential, so the document names a path
|
|
38
|
+
# and never a secret. What is stored is which connection, not what is in it.
|
|
39
|
+
connection: postman-echo
|
|
40
|
+
path: /basic-auth
|
|
41
|
+
method: GET
|
|
42
|
+
|
|
43
|
+
publish:
|
|
44
|
+
block: http.request
|
|
45
|
+
depends_on: [authenticate]
|
|
46
|
+
config:
|
|
47
|
+
connection: postman-echo
|
|
48
|
+
path: /post
|
|
49
|
+
method: POST
|
|
50
|
+
body:
|
|
51
|
+
authenticated: "${steps.authenticate.output.status}"
|
|
52
|
+
|
|
53
|
+
receipt:
|
|
54
|
+
block: storage.write
|
|
55
|
+
depends_on: [publish]
|
|
56
|
+
config:
|
|
57
|
+
target: "${run.scratch}/reports/receipt.json"
|
|
58
|
+
value: "${steps.publish.output.body}"
|
|
59
|
+
|
|
60
|
+
archive:
|
|
61
|
+
block: storage.copy
|
|
62
|
+
depends_on: [receipt]
|
|
63
|
+
config:
|
|
64
|
+
source: "${steps.receipt.output.uri}"
|
|
65
|
+
target: "${run.scratch}/archive/receipt.json"
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# The other half of the shared-name pair; weekly-import-nepal.yaml carries the explanation.
|
|
2
|
+
#
|
|
3
|
+
# Both documents declare `name: Weekly import`. Applying both leaves an instance holding two
|
|
4
|
+
# pipelines with the same title and different codes, which is exactly the situation the
|
|
5
|
+
# naming rules allow and the screens are drawn to survive.
|
|
6
|
+
#
|
|
7
|
+
# Kept as a separate file because a dirigent document holds exactly one pipeline. That is
|
|
8
|
+
# also why composition is two files, and why a schedule example carries its own pipeline.
|
|
9
|
+
#
|
|
10
|
+
# dg apply examples/demo/weekly-import-malawi.yaml
|
|
11
|
+
# dg run weekly-import-malawi -p week=2026-W02
|
|
12
|
+
|
|
13
|
+
format: dirigent/v1
|
|
14
|
+
kind: pipeline
|
|
15
|
+
code: weekly-import-malawi
|
|
16
|
+
# The same title as weekly-import-nepal.yaml, on purpose.
|
|
17
|
+
name: Weekly import
|
|
18
|
+
description: |
|
|
19
|
+
The Malawi half of a deliberately ambiguous pair.
|
|
20
|
+
|
|
21
|
+
Same display name as `weekly-import-nepal`, different `code`. The listing shows one title
|
|
22
|
+
twice and two codes once each, which is how a person tells them apart and how every
|
|
23
|
+
reference already did.
|
|
24
|
+
|
|
25
|
+
tags: [demo, transform]
|
|
26
|
+
|
|
27
|
+
requires:
|
|
28
|
+
blocks:
|
|
29
|
+
- transform.jq
|
|
30
|
+
|
|
31
|
+
params:
|
|
32
|
+
type: object
|
|
33
|
+
properties:
|
|
34
|
+
week:
|
|
35
|
+
type: string
|
|
36
|
+
description: The ISO week being imported.
|
|
37
|
+
default: 2026-W01
|
|
38
|
+
|
|
39
|
+
steps:
|
|
40
|
+
read_manifest:
|
|
41
|
+
block: transform.jq
|
|
42
|
+
config:
|
|
43
|
+
input:
|
|
44
|
+
country: malawi
|
|
45
|
+
week: "${params.week}"
|
|
46
|
+
districts: [balaka, chikwawa, dedza, karonga, mzimba, salima, zomba]
|
|
47
|
+
program: "{country, week, districts: (.districts | length)}"
|
|
48
|
+
|
|
49
|
+
summarise:
|
|
50
|
+
block: transform.jq
|
|
51
|
+
depends_on: [read_manifest]
|
|
52
|
+
config:
|
|
53
|
+
input: "${steps.read_manifest.output.value}"
|
|
54
|
+
program: "{note: \"\\(.country) \\(.week): \\(.districts) districts imported\"}"
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# Two pipelines, one display name, and only the code telling them apart.
|
|
2
|
+
#
|
|
3
|
+
# This document and weekly-import-malawi.yaml both carry `name: Weekly import`. That is legal,
|
|
4
|
+
# it is not a mistake, and an instance holding both is behaving correctly.
|
|
5
|
+
#
|
|
6
|
+
# The four fields every addressable thing carries each have exactly one job:
|
|
7
|
+
#
|
|
8
|
+
# id the uuid a machine holds
|
|
9
|
+
# code the addressable key: constrained, unique, and what a URL, a reference and
|
|
10
|
+
# `dg run` all carry
|
|
11
|
+
# name an optional human title, free-form, with NO identity semantics at all --
|
|
12
|
+
# nothing may ever be referenced by it, and two rows may share one
|
|
13
|
+
# description the long-form body
|
|
14
|
+
#
|
|
15
|
+
# The temptation this pair exists to head off is treating a name as a key: looking a pipeline
|
|
16
|
+
# up by it, keying a dictionary on it, or assuming a listing's titles are distinct. Names are
|
|
17
|
+
# for reading. `dg run weekly-import-nepal` names a pipeline; "Weekly import" names two, and
|
|
18
|
+
# an instance asked to run that has no way to choose.
|
|
19
|
+
#
|
|
20
|
+
# So the UI's rule pays for itself here. A screen shows the name as the title AND the code in
|
|
21
|
+
# mono beside it, always, never drawn twice -- which is what makes two identically titled
|
|
22
|
+
# rows in a listing tellable apart at a glance instead of a support question. Apply both and
|
|
23
|
+
# look at the pipelines list.
|
|
24
|
+
#
|
|
25
|
+
# Nothing here reaches the network or runs code on the worker.
|
|
26
|
+
#
|
|
27
|
+
# dg apply examples/demo/weekly-import-nepal.yaml
|
|
28
|
+
# dg apply examples/demo/weekly-import-malawi.yaml
|
|
29
|
+
# dg pipelines list # two rows, one title, two codes
|
|
30
|
+
|
|
31
|
+
format: dirigent/v1
|
|
32
|
+
kind: pipeline
|
|
33
|
+
code: weekly-import-nepal
|
|
34
|
+
# Shared, deliberately, with weekly-import-malawi.yaml. The code above is what differs, and
|
|
35
|
+
# the code is the only thing that was ever identity.
|
|
36
|
+
name: Weekly import
|
|
37
|
+
description: |
|
|
38
|
+
The Nepal half of a deliberately ambiguous pair.
|
|
39
|
+
|
|
40
|
+
This pipeline and `weekly-import-malawi` share the display name **Weekly import**. Only
|
|
41
|
+
the `code` is identity: `name` is free-form, carries no uniqueness, and nothing may be
|
|
42
|
+
referenced by it.
|
|
43
|
+
|
|
44
|
+
Two rows may share a title. Neither may share a code.
|
|
45
|
+
|
|
46
|
+
tags: [demo, transform]
|
|
47
|
+
|
|
48
|
+
requires:
|
|
49
|
+
blocks:
|
|
50
|
+
- transform.jq
|
|
51
|
+
|
|
52
|
+
params:
|
|
53
|
+
type: object
|
|
54
|
+
properties:
|
|
55
|
+
week:
|
|
56
|
+
type: string
|
|
57
|
+
description: The ISO week being imported.
|
|
58
|
+
default: 2026-W01
|
|
59
|
+
|
|
60
|
+
steps:
|
|
61
|
+
read_manifest:
|
|
62
|
+
block: transform.jq
|
|
63
|
+
config:
|
|
64
|
+
input:
|
|
65
|
+
country: nepal
|
|
66
|
+
week: "${params.week}"
|
|
67
|
+
districts: [bagmati, gandaki, karnali, koshi, lumbini, madhesh, sudurpashchim]
|
|
68
|
+
program: "{country, week, districts: (.districts | length)}"
|
|
69
|
+
|
|
70
|
+
summarise:
|
|
71
|
+
block: transform.jq
|
|
72
|
+
depends_on: [read_manifest]
|
|
73
|
+
config:
|
|
74
|
+
input: "${steps.read_manifest.output.value}"
|
|
75
|
+
program: "{note: \"\\(.country) \\(.week): \\(.districts) districts imported\"}"
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Docker examples
|
|
2
|
+
|
|
3
|
+
The `docker.*` family runs containers on the worker, which is why each block needs its id in
|
|
4
|
+
`DIRIGENT_ENABLED_UNSAFE_BLOCKS` before an instance will run it, and a daemon the worker can
|
|
5
|
+
reach. [docs/docker.md](../../docs/docker.md) is the family's home, and
|
|
6
|
+
[docs/security.md](../../docs/security.md) says what mounting the socket grants.
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
dg run --local examples/docker/docker-hello.yaml --enable-unsafe docker.run
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
The last two documents on the shelf **fail on purpose**: they are here for what a broken
|
|
13
|
+
stack and a failed drive step leave behind, which in both cases is nothing.
|
|
14
|
+
|
|
15
|
+
## Pipelines
|
|
16
|
+
|
|
17
|
+
| File | What it teaches |
|
|
18
|
+
| --- | --- |
|
|
19
|
+
| [docker-hello.yaml](docker-hello.yaml) | One command in one container: the image pulled first, no network, the streams collected. |
|
|
20
|
+
| [docker-ticker.yaml](docker-ticker.yaml) | The live log: a container talking while it works, each probe appending what arrived since the last one. |
|
|
21
|
+
| [docker-compose-stack.yaml](docker-compose-stack.yaml) | A whole stack: `docker.compose.up` brings it up, a `docker.run` step joins its network, and `docker.compose.down` with `rule: all_done` tears it down on any outcome. |
|
|
22
|
+
| [docker-compose-database.yaml](docker-compose-database.yaml) | A database driven for real: `wait: true` turns a healthcheck into a gate, a `psql` container writes and reads rows, and the volumes go with the teardown. |
|
|
23
|
+
| [docker-compose-file.yaml](docker-compose-file.yaml) | The other compose form: a file an earlier step wrote into the run's work directory, named by its path relative to it. |
|
|
24
|
+
| [docker-compose-profiles-env.yaml](docker-compose-profiles-env.yaml) | Parameters reaching the stack: a variable compose interpolates, and a profile that decides whether the optional service exists. |
|
|
25
|
+
| [docker-build-run.yaml](docker-build-run.yaml) | An image built on the worker and run by tag, because a built image lives only in the store of the daemon that built it. |
|
|
26
|
+
| [docker-build-push.yaml](docker-build-push.yaml) | Pushing what was built: a `docker` connection holding a registry credential, and the tags and digests that come back. |
|
|
27
|
+
| [docker-remote-daemon.yaml](docker-remote-daemon.yaml) | Naming the daemon: a `docker` connection with a host and its client TLS, on every step of one stack's lifecycle. |
|
|
28
|
+
| [docker-compose-failing-up.yaml](docker-compose-failing-up.yaml) | A bring-up that fails, and the containers it does not leave behind: the block tears its own half-built project down. |
|
|
29
|
+
| [docker-run-failing-teardown.yaml](docker-run-failing-teardown.yaml) | A drive step that fails: the run is failed, the teardown ran anyway, and the daemon holds nothing. |
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# Building an image and pushing it, which is the one thing a tag alone cannot do.
|
|
2
|
+
#
|
|
3
|
+
# docker-build-run.yaml builds an image and runs it on the same worker, where the tag is the
|
|
4
|
+
# whole contract and the image never leaves that daemon's store. This document is the other
|
|
5
|
+
# half: the image is pushed, so a later run on another worker, another cluster or another
|
|
6
|
+
# machine can pull it by name.
|
|
7
|
+
#
|
|
8
|
+
# IT CANNOT RUN WITHOUT A REGISTRY. There is no public registry that will take an anonymous
|
|
9
|
+
# push, so this document does nothing at all until it has a real registry and a credential
|
|
10
|
+
# for it. Create the connection once, with a token that may write packages:
|
|
11
|
+
# dg connection create docker ghcr \
|
|
12
|
+
# --set registry=ghcr.io \
|
|
13
|
+
# --set username=your-account \
|
|
14
|
+
# --set "password=$GITHUB_TOKEN"
|
|
15
|
+
#
|
|
16
|
+
# docker.build declares local_execution, so the engine refuses it unless the instance
|
|
17
|
+
# allowlists its id:
|
|
18
|
+
# export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.run","docker.build"]'
|
|
19
|
+
#
|
|
20
|
+
# PUSHING NEEDS A CONNECTION AND SAYS SO. push: true with no connection is refused when the
|
|
21
|
+
# document is applied, not when it first runs, and a connection carrying no username and
|
|
22
|
+
# password is refused when the step starts. Neither is a build that silently produced a local
|
|
23
|
+
# image nobody can reach.
|
|
24
|
+
#
|
|
25
|
+
# ONE DAEMON, THEREFORE ONE WORKER. The build and the step that wrote its context are on the
|
|
26
|
+
# worker's own filesystem, so requires.workers keeps them on one docker-capable worker. The push
|
|
27
|
+
# is what frees the image from that worker: once it is in the registry, anything can pull it.
|
|
28
|
+
#
|
|
29
|
+
# THE CREDENTIAL NEVER BECOMES AN ARGUMENT. The password is sealed on the connection, reaches
|
|
30
|
+
# `docker login` on stdin, and lives in a DOCKER_CONFIG directory of its own under the run's
|
|
31
|
+
# work directory -- never the worker's own docker config. The step logs out and the directory
|
|
32
|
+
# goes when it leaves, so nothing about the login survives the step, and the password is
|
|
33
|
+
# scrubbed out of every log line and every failure message.
|
|
34
|
+
#
|
|
35
|
+
# THE TAG IS A REGISTRY ADDRESS NOW. A pushed tag is fully qualified: registry, owner, name.
|
|
36
|
+
# The build's output says which tags reached the registry (pushed) and the digest the registry
|
|
37
|
+
# gave each of them (digests), and a digest is the only exact name for what was pushed -- a
|
|
38
|
+
# tag can be moved, a digest cannot.
|
|
39
|
+
#
|
|
40
|
+
# TO MAKE IT YOURS: point image_tag at your own registry and repository, replace the Dockerfile
|
|
41
|
+
# the first step writes with your own, and add a second tag if you publish both a version and
|
|
42
|
+
# a moving "latest".
|
|
43
|
+
|
|
44
|
+
format: dirigent/v1
|
|
45
|
+
kind: pipeline
|
|
46
|
+
code: docker-build-push
|
|
47
|
+
name: Build an image and push it
|
|
48
|
+
description: Build an image from a context in the work directory and push every tag to the registry a docker connection names.
|
|
49
|
+
|
|
50
|
+
tags: [docker, execute]
|
|
51
|
+
|
|
52
|
+
requires:
|
|
53
|
+
blocks:
|
|
54
|
+
- docker.run
|
|
55
|
+
- docker.build
|
|
56
|
+
workers:
|
|
57
|
+
# An image built on worker A is invisible to worker B, so both steps have to meet one
|
|
58
|
+
# docker-capable worker: give those the tag (dg worker --tag docker) and name it here.
|
|
59
|
+
- docker
|
|
60
|
+
connections:
|
|
61
|
+
# Named, not carried: the instance holds the registry credential, and this document
|
|
62
|
+
# refuses to apply where it is absent rather than failing the first time it runs.
|
|
63
|
+
- ghcr
|
|
64
|
+
|
|
65
|
+
params:
|
|
66
|
+
type: object
|
|
67
|
+
additionalProperties: false
|
|
68
|
+
properties:
|
|
69
|
+
image_tag:
|
|
70
|
+
type: string
|
|
71
|
+
default: ghcr.io/your-account/dirigent-greeter:1.0.0
|
|
72
|
+
description: The fully qualified tag the build gives the image and the push sends.
|
|
73
|
+
|
|
74
|
+
steps:
|
|
75
|
+
dockerfile:
|
|
76
|
+
block: docker.run
|
|
77
|
+
deadline: 5m
|
|
78
|
+
config:
|
|
79
|
+
image: alpine:3
|
|
80
|
+
pull: true
|
|
81
|
+
network: none
|
|
82
|
+
# A target with no URI scheme lands in the run's work directory on this worker, which
|
|
83
|
+
# is where the build below reads its context from; a URI would send the file to storage.
|
|
84
|
+
outputs:
|
|
85
|
+
Dockerfile: "context/Dockerfile"
|
|
86
|
+
command: |
|
|
87
|
+
cat > /dirigent/outputs/Dockerfile <<'DOCKERFILE'
|
|
88
|
+
FROM alpine:3
|
|
89
|
+
RUN echo "baked in at build time" > /greeting.txt
|
|
90
|
+
CMD ["cat", "/greeting.txt"]
|
|
91
|
+
DOCKERFILE
|
|
92
|
+
memory: 64mb
|
|
93
|
+
pids_limit: 64
|
|
94
|
+
|
|
95
|
+
build:
|
|
96
|
+
block: docker.build
|
|
97
|
+
depends_on: [dockerfile]
|
|
98
|
+
deadline: 15m
|
|
99
|
+
config:
|
|
100
|
+
context: context
|
|
101
|
+
dockerfile: Dockerfile
|
|
102
|
+
# A pushed tag carries its registry, so this name is where the image ends up as much as
|
|
103
|
+
# what it is called.
|
|
104
|
+
tags: ["${params.image_tag}"]
|
|
105
|
+
pull: true
|
|
106
|
+
# The connection holds the registry, the username and the sealed password. It could hold
|
|
107
|
+
# a daemon too: one connection can name both, and the steps that build and the steps that
|
|
108
|
+
# run would then all name it.
|
|
109
|
+
connection: ghcr
|
|
110
|
+
push: true
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# Building an image on the worker, then running it, and why both steps must meet one daemon.
|
|
2
|
+
#
|
|
3
|
+
# REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
|
|
4
|
+
# tcp+TLS, or the local socket in dev). docs/docker.md is the family's home.
|
|
5
|
+
#
|
|
6
|
+
# docker.run and docker.build both declare local_execution, so the engine refuses them unless
|
|
7
|
+
# the instance allowlists their ids:
|
|
8
|
+
# dg run --local examples/docker/docker-build-run.yaml --enable-unsafe docker.run,docker.build
|
|
9
|
+
# export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.run","docker.build"]'
|
|
10
|
+
#
|
|
11
|
+
# Three hops, and what each one hands on:
|
|
12
|
+
# dockerfile a container writes a Dockerfile into context/ in the run's work directory.
|
|
13
|
+
# docker.build takes a context DIRECTORY on the worker's filesystem and has no
|
|
14
|
+
# inline form, so getting the context there is the pipeline's job.
|
|
15
|
+
# build buildx builds that directory and loads the result into the worker's own daemon
|
|
16
|
+
# store. The output carries the image id, its size in bytes, and its tags.
|
|
17
|
+
# greet docker.run runs the image by tag with no CMD of its own, so the image's own
|
|
18
|
+
# CMD is what runs. The output carries its stdout, exit code and container id.
|
|
19
|
+
#
|
|
20
|
+
# THE TAG IS THE CONTRACT. Nothing is pushed here, so the image exists only in the store of the
|
|
21
|
+
# daemon that built it, under the tags the build gave it. That name is all the run step has,
|
|
22
|
+
# which is why both steps read it from one parameter. docker-build-push.yaml is the other half:
|
|
23
|
+
# a docker connection with a registry credential, and an image that leaves the worker.
|
|
24
|
+
#
|
|
25
|
+
# ONE DAEMON, THEREFORE ONE WORKER. An image built on worker A is invisible to worker B, so
|
|
26
|
+
# give the docker-capable workers a `docker` tag (dg worker --tag docker) and declare
|
|
27
|
+
# requires.workers: [docker] below, which is what keeps both steps on one of them.
|
|
28
|
+
#
|
|
29
|
+
# TO MAKE IT YOURS: replace the Dockerfile the first step writes with your own, or drop that
|
|
30
|
+
# step and point the build at a context an upstream storage.copy put in the work directory
|
|
31
|
+
# instead, and give the image a tag of your own.
|
|
32
|
+
|
|
33
|
+
format: dirigent/v1
|
|
34
|
+
kind: pipeline
|
|
35
|
+
code: docker-build-run
|
|
36
|
+
name: Build an image, then run it
|
|
37
|
+
description: Write a Dockerfile into the run's work directory, build it with buildx, and run the tag.
|
|
38
|
+
|
|
39
|
+
tags: [docker, execute, starter]
|
|
40
|
+
|
|
41
|
+
requires:
|
|
42
|
+
blocks:
|
|
43
|
+
- docker.run
|
|
44
|
+
- docker.build
|
|
45
|
+
workers:
|
|
46
|
+
- docker
|
|
47
|
+
|
|
48
|
+
params:
|
|
49
|
+
type: object
|
|
50
|
+
additionalProperties: false
|
|
51
|
+
properties:
|
|
52
|
+
image_tag:
|
|
53
|
+
type: string
|
|
54
|
+
default: dirigent-greeter:local
|
|
55
|
+
description: The tag the build gives the image, and the only name the run step has for it.
|
|
56
|
+
|
|
57
|
+
steps:
|
|
58
|
+
dockerfile:
|
|
59
|
+
block: docker.run
|
|
60
|
+
deadline: 5m
|
|
61
|
+
config:
|
|
62
|
+
image: alpine:3
|
|
63
|
+
pull: true
|
|
64
|
+
network: none
|
|
65
|
+
# Whatever the container leaves in its output mount is copied where the target names
|
|
66
|
+
# once it exits. A target with no URI scheme is a path in the run's work directory on
|
|
67
|
+
# this worker, which is where a build context is read from; a URI would send the file to
|
|
68
|
+
# storage instead, for a result that outlives the run.
|
|
69
|
+
outputs:
|
|
70
|
+
Dockerfile: "context/Dockerfile"
|
|
71
|
+
command: |
|
|
72
|
+
cat > /dirigent/outputs/Dockerfile <<'DOCKERFILE'
|
|
73
|
+
FROM alpine:3
|
|
74
|
+
RUN echo "baked in at build time" > /greeting.txt
|
|
75
|
+
CMD ["cat", "/greeting.txt"]
|
|
76
|
+
DOCKERFILE
|
|
77
|
+
memory: 64mb
|
|
78
|
+
pids_limit: 64
|
|
79
|
+
|
|
80
|
+
build:
|
|
81
|
+
block: docker.build
|
|
82
|
+
depends_on: [dockerfile]
|
|
83
|
+
deadline: 10m
|
|
84
|
+
config:
|
|
85
|
+
# A directory inside the run's work directory, never an absolute path: the same
|
|
86
|
+
# "context/" the step above wrote into, with the Dockerfile named relative to it.
|
|
87
|
+
context: context
|
|
88
|
+
dockerfile: Dockerfile
|
|
89
|
+
# The image is loaded into the worker's daemon store under this tag, and the id comes
|
|
90
|
+
# back from buildx's --iidfile rather than scraped out of the build log.
|
|
91
|
+
tags: ["${params.image_tag}"]
|
|
92
|
+
# Fetch a newer alpine:3 rather than building on whatever the daemon happens to hold.
|
|
93
|
+
pull: true
|
|
94
|
+
|
|
95
|
+
greet:
|
|
96
|
+
block: docker.run
|
|
97
|
+
depends_on: [build]
|
|
98
|
+
deadline: 5m
|
|
99
|
+
config:
|
|
100
|
+
image: "${params.image_tag}"
|
|
101
|
+
# pull stays off, and must: the image was never pushed, so a pull would go looking for
|
|
102
|
+
# this tag in a registry and fail with a name the daemon already has locally.
|
|
103
|
+
pull: false
|
|
104
|
+
network: none
|
|
105
|
+
memory: 64mb
|
|
106
|
+
pids_limit: 64
|