dirigent-examples 0.15.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dirigent_examples/__init__.py +22 -0
- dirigent_examples/py.typed +0 -0
- dirigent_examples/shelves/README.md +299 -0
- dirigent_examples/shelves/composition/README.md +18 -0
- dirigent_examples/shelves/composition/chained-instances.yaml +97 -0
- dirigent_examples/shelves/composition/composition-child.yaml +64 -0
- dirigent_examples/shelves/composition/composition-parent.yaml +89 -0
- dirigent_examples/shelves/connections.yaml +52 -0
- dirigent_examples/shelves/demo/README.md +19 -0
- dirigent_examples/shelves/demo/markdown-showcase.yaml +117 -0
- dirigent_examples/shelves/demo/params-showcase.yaml +92 -0
- dirigent_examples/shelves/demo/requires.yaml +65 -0
- dirigent_examples/shelves/demo/weekly-import-malawi.yaml +54 -0
- dirigent_examples/shelves/demo/weekly-import-nepal.yaml +75 -0
- dirigent_examples/shelves/docker/README.md +29 -0
- dirigent_examples/shelves/docker/docker-build-push.yaml +110 -0
- dirigent_examples/shelves/docker/docker-build-run.yaml +106 -0
- dirigent_examples/shelves/docker/docker-compose-database.yaml +124 -0
- dirigent_examples/shelves/docker/docker-compose-failing-up.yaml +83 -0
- dirigent_examples/shelves/docker/docker-compose-file.yaml +117 -0
- dirigent_examples/shelves/docker/docker-compose-profiles-env.yaml +133 -0
- dirigent_examples/shelves/docker/docker-compose-stack.yaml +70 -0
- dirigent_examples/shelves/docker/docker-hello.yaml +53 -0
- dirigent_examples/shelves/docker/docker-remote-daemon.yaml +92 -0
- dirigent_examples/shelves/docker/docker-run-failing-teardown.yaml +94 -0
- dirigent_examples/shelves/docker/docker-ticker.yaml +49 -0
- dirigent_examples/shelves/execute/README.md +16 -0
- dirigent_examples/shelves/execute/long-log.yaml +89 -0
- dirigent_examples/shelves/failure/README.md +20 -0
- dirigent_examples/shelves/failure/error-handler.yaml +89 -0
- dirigent_examples/shelves/failure/optional-step.yaml +82 -0
- dirigent_examples/shelves/failure/retries.yaml +75 -0
- dirigent_examples/shelves/failure/retry-budget.yaml +82 -0
- dirigent_examples/shelves/failure/step-timeout.yaml +96 -0
- dirigent_examples/shelves/git/README.md +32 -0
- dirigent_examples/shelves/git/git-checkout-build.yaml +125 -0
- dirigent_examples/shelves/git/git-checkout-compose.yaml +138 -0
- dirigent_examples/shelves/git/git-checkout-public.yaml +84 -0
- dirigent_examples/shelves/graph/README.md +22 -0
- dirigent_examples/shelves/graph/deep-chain.yaml +119 -0
- dirigent_examples/shelves/graph/fan-in.yaml +80 -0
- dirigent_examples/shelves/graph/fan-out.yaml +66 -0
- dirigent_examples/shelves/graph/linear.yaml +66 -0
- dirigent_examples/shelves/graph/parallel-branches.yaml +57 -0
- dirigent_examples/shelves/graph/parallel-sleep.yaml +55 -0
- dirigent_examples/shelves/graph/skip-diamond.yaml +105 -0
- dirigent_examples/shelves/graph/wide-fan.yaml +147 -0
- dirigent_examples/shelves/hello-world.yaml +30 -0
- dirigent_examples/shelves/open-data/README.md +67 -0
- dirigent_examples/shelves/open-data/feeds-composition.yaml +120 -0
- dirigent_examples/shelves/open-data/gdacs-disaster-updates.yaml +230 -0
- dirigent_examples/shelves/open-data/github-releases-relay.yaml +225 -0
- dirigent_examples/shelves/open-data/hdx-dataset-watch.yaml +211 -0
- dirigent_examples/shelves/open-data/kobo-submissions-to-csv.yaml +149 -0
- dirigent_examples/shelves/open-data/nominatim-geocode-facilities.yaml +169 -0
- dirigent_examples/shelves/open-data/odk-central-submissions.yaml +146 -0
- dirigent_examples/shelves/open-data/open-meteo-weekly-report.yaml +142 -0
- dirigent_examples/shelves/open-data/overpass-health-facilities.yaml +154 -0
- dirigent_examples/shelves/open-data/usgs-earthquakes-alert.yaml +208 -0
- dirigent_examples/shelves/open-data/who-gho-indicators-to-parquet.yaml +163 -0
- dirigent_examples/shelves/open-data/wikidata-country-reference.yaml +146 -0
- dirigent_examples/shelves/open-data/world-bank-population-trend.yaml +166 -0
- dirigent_examples/shelves/patterns/README.md +144 -0
- dirigent_examples/shelves/patterns/concurrency-queue.yaml +89 -0
- dirigent_examples/shelves/patterns/concurrency-replace.yaml +92 -0
- dirigent_examples/shelves/patterns/concurrency-skip.yaml +97 -0
- dirigent_examples/shelves/patterns/connections-referenced-vs-carried.yaml +140 -0
- dirigent_examples/shelves/patterns/deadline-on-a-sensor.yaml +106 -0
- dirigent_examples/shelves/patterns/fan-out-continue.yaml +88 -0
- dirigent_examples/shelves/patterns/fan-out-fail-fast.yaml +80 -0
- dirigent_examples/shelves/patterns/fan-out-from-params.yaml +84 -0
- dirigent_examples/shelves/patterns/fan-out-item-wise.yaml +111 -0
- dirigent_examples/shelves/patterns/fan-out-literal-list.yaml +81 -0
- dirigent_examples/shelves/patterns/fan-out-nested-objects.yaml +107 -0
- dirigent_examples/shelves/patterns/fan-out-then-join.yaml +86 -0
- dirigent_examples/shelves/patterns/log-levels.yaml +119 -0
- dirigent_examples/shelves/patterns/outputs-inline-vs-storage.yaml +140 -0
- dirigent_examples/shelves/patterns/params-every-type.yaml +259 -0
- dirigent_examples/shelves/patterns/params-validation-refuses.yaml +131 -0
- dirigent_examples/shelves/patterns/pipeline-run-child.yaml +96 -0
- dirigent_examples/shelves/patterns/pipeline-run-fire-and-forget.yaml +99 -0
- dirigent_examples/shelves/patterns/pipeline-run-strict.yaml +103 -0
- dirigent_examples/shelves/patterns/pipeline-run-wait.yaml +107 -0
- dirigent_examples/shelves/patterns/pipeline-run-with-params.yaml +124 -0
- dirigent_examples/shelves/patterns/poll-cadence.yaml +102 -0
- dirigent_examples/shelves/patterns/priority-layered.yaml +120 -0
- dirigent_examples/shelves/patterns/references-cheat-sheet.yaml +186 -0
- dirigent_examples/shelves/patterns/retry-budget-exhausted.yaml +92 -0
- dirigent_examples/shelves/patterns/retry-exponential-backoff.yaml +88 -0
- dirigent_examples/shelves/patterns/retry-only-transient.yaml +108 -0
- dirigent_examples/shelves/patterns/retry-with-jitter.yaml +102 -0
- dirigent_examples/shelves/patterns/rule-all-done.yaml +83 -0
- dirigent_examples/shelves/patterns/rule-all-success.yaml +79 -0
- dirigent_examples/shelves/patterns/rule-always.yaml +92 -0
- dirigent_examples/shelves/patterns/rule-one-failed.yaml +86 -0
- dirigent_examples/shelves/patterns/schedule-at-once.yaml +110 -0
- dirigent_examples/shelves/patterns/schedule-cron-timezone.yaml +121 -0
- dirigent_examples/shelves/patterns/schedule-interval.yaml +109 -0
- dirigent_examples/shelves/patterns/schedule-window-half-open.yaml +105 -0
- dirigent_examples/shelves/patterns/sensor-http-ready.yaml +121 -0
- dirigent_examples/shelves/patterns/sensor-storage-exists.yaml +134 -0
- dirigent_examples/shelves/patterns/step-names-and-keys.yaml +99 -0
- dirigent_examples/shelves/patterns/timeout-fails-the-step.yaml +94 -0
- dirigent_examples/shelves/patterns/timeout-skips-the-step.yaml +102 -0
- dirigent_examples/shelves/patterns/webhook-mapping-nested-payload.yaml +125 -0
- dirigent_examples/shelves/patterns/webhook-signed.yaml +144 -0
- dirigent_examples/shelves/preview/s3-parquet-to-ingestion.yaml +92 -0
- dirigent_examples/shelves/python/README.md +31 -0
- dirigent_examples/shelves/python/apply_and_run.py +52 -0
- dirigent_examples/shelves/python/ci_gate.py +76 -0
- dirigent_examples/shelves/python/connections.py +61 -0
- dirigent_examples/shelves/python/error_handling.py +84 -0
- dirigent_examples/shelves/python/follow_logs.py +39 -0
- dirigent_examples/shelves/python/list_and_filter.py +52 -0
- dirigent_examples/shelves/queues/README.md +59 -0
- dirigent_examples/shelves/queues/kafka-consume-then-transform.yaml +105 -0
- dirigent_examples/shelves/queues/kafka-produce-then-consume.yaml +124 -0
- dirigent_examples/shelves/queues/rabbitmq-consume-ack-on-success.yaml +102 -0
- dirigent_examples/shelves/queues/report-to-kafka.yaml +105 -0
- dirigent_examples/shelves/queues/report-to-rabbitmq.yaml +105 -0
- dirigent_examples/shelves/recipes/README.md +130 -0
- dirigent_examples/shelves/recipes/csv-header-rules.yaml +147 -0
- dirigent_examples/shelves/recipes/csv-to-ndjson.yaml +107 -0
- dirigent_examples/shelves/recipes/etl-csv-clean-validate-parquet.yaml +207 -0
- dirigent_examples/shelves/recipes/filter-by-predicate.yaml +116 -0
- dirigent_examples/shelves/recipes/filter-then-map-then-reduce.yaml +109 -0
- dirigent_examples/shelves/recipes/http-fetch-validate-post.yaml +142 -0
- dirigent_examples/shelves/recipes/http-follow-redirects.yaml +100 -0
- dirigent_examples/shelves/recipes/http-get-with-query.yaml +96 -0
- dirigent_examples/shelves/recipes/http-headers-and-auth-connection.yaml +110 -0
- dirigent_examples/shelves/recipes/http-post-file-from-storage.yaml +117 -0
- dirigent_examples/shelves/recipes/http-post-json-echo.yaml +105 -0
- dirigent_examples/shelves/recipes/http-post-report.yaml +183 -0
- dirigent_examples/shelves/recipes/http-save-body-to-storage.yaml +106 -0
- dirigent_examples/shelves/recipes/http-success-status-list.yaml +80 -0
- dirigent_examples/shelves/recipes/http-timeout-override.yaml +104 -0
- dirigent_examples/shelves/recipes/jq-dedupe-by-key.yaml +78 -0
- dirigent_examples/shelves/recipes/jq-defaults-and-nulls.yaml +91 -0
- dirigent_examples/shelves/recipes/jq-group-by-and-sum.yaml +74 -0
- dirigent_examples/shelves/recipes/jq-join-two-lists.yaml +77 -0
- dirigent_examples/shelves/recipes/jq-long-to-wide.yaml +76 -0
- dirigent_examples/shelves/recipes/jq-nested-to-flat.yaml +89 -0
- dirigent_examples/shelves/recipes/jq-pivot-wide-to-long.yaml +65 -0
- dirigent_examples/shelves/recipes/jq-running-totals.yaml +82 -0
- dirigent_examples/shelves/recipes/jq-string-cleaning.yaml +88 -0
- dirigent_examples/shelves/recipes/jq-top-n.yaml +85 -0
- dirigent_examples/shelves/recipes/jq-validate-in-jq-vs-schema.yaml +124 -0
- dirigent_examples/shelves/recipes/jq-window-dates.yaml +82 -0
- dirigent_examples/shelves/recipes/json-to-csv-flattening.yaml +141 -0
- dirigent_examples/shelves/recipes/large-output-to-storage.yaml +134 -0
- dirigent_examples/shelves/recipes/map-enrich-with-lookup.yaml +96 -0
- dirigent_examples/shelves/recipes/ndjson-to-parquet.yaml +130 -0
- dirigent_examples/shelves/recipes/pagination-by-fan-out.yaml +115 -0
- dirigent_examples/shelves/recipes/parquet-round-trip-types.yaml +163 -0
- dirigent_examples/shelves/recipes/reconcile-two-sources.yaml +159 -0
- dirigent_examples/shelves/recipes/report-built-in.yaml +72 -0
- dirigent_examples/shelves/recipes/report-daily-digest.yaml +186 -0
- dirigent_examples/shelves/recipes/report-to-file.yaml +131 -0
- dirigent_examples/shelves/recipes/report-to-webhook.yaml +136 -0
- dirigent_examples/shelves/recipes/schema-carried.yaml +112 -0
- dirigent_examples/shelves/recipes/schema-formats.yaml +107 -0
- dirigent_examples/shelves/recipes/schema-referenced.yaml +86 -0
- dirigent_examples/shelves/recipes/schema-refuses-then-rule.yaml +127 -0
- dirigent_examples/shelves/recipes/storage-copy-dated-archive.yaml +114 -0
- dirigent_examples/shelves/recipes/storage-exists-gate.yaml +127 -0
- dirigent_examples/shelves/recipes/storage-manifest-of-a-fan-out.yaml +104 -0
- dirigent_examples/shelves/recipes/storage-write-then-read.yaml +119 -0
- dirigent_examples/shelves/recipes/webhook-post-hmac.yaml +132 -0
- dirigent_examples/shelves/recipes/webhook-post-summary.yaml +142 -0
- dirigent_examples/shelves/s3/README.md +34 -0
- dirigent_examples/shelves/s3/report-to-s3.yaml +93 -0
- dirigent_examples/shelves/s3/s3-copy-and-verify.yaml +105 -0
- dirigent_examples/shelves/s3/s3-csv-report.yaml +87 -0
- dirigent_examples/shelves/s3/s3-parquet-report.yaml +77 -0
- dirigent_examples/shelves/s3/s3-round-trip.yaml +125 -0
- dirigent_examples/shelves/schemas/README.md +36 -0
- dirigent_examples/shelves/schemas/echo-reading.json +18 -0
- dirigent_examples/shelves/schemas/ou-record.json +13 -0
- dirigent_examples/shelves/schemas/station-reading.json +13 -0
- dirigent_examples/shelves/sensors/README.md +16 -0
- dirigent_examples/shelves/sensors/sensor-gate.yaml +65 -0
- dirigent_examples/shelves/sensors/time-window.yaml +61 -0
- dirigent_examples/shelves/sql/README.md +52 -0
- dirigent_examples/shelves/sql/duckdb-parquet-to-report.yaml +146 -0
- dirigent_examples/shelves/sql/sql-postgres-readonly.yaml +111 -0
- dirigent_examples/shelves/sql/sql-query-to-storage.yaml +85 -0
- dirigent_examples/shelves/sql/sql-sqlite-roundtrip.yaml +114 -0
- dirigent_examples/shelves/sql/warehouse.sql +42 -0
- dirigent_examples/shelves/transform/README.md +36 -0
- dirigent_examples/shelves/transform/csv-report.yaml +55 -0
- dirigent_examples/shelves/transform/jq-filter-and-map.yaml +70 -0
- dirigent_examples/shelves/transform/jq-group-and-aggregate.yaml +70 -0
- dirigent_examples/shelves/transform/jq-join-two-sources.yaml +98 -0
- dirigent_examples/shelves/transform/jq-reshape.yaml +91 -0
- dirigent_examples/shelves/transform/jq-stream-through-storage.yaml +112 -0
- dirigent_examples/shelves/transform/ndjson-round-trip.yaml +56 -0
- dirigent_examples/shelves/transform/parquet-round-trip.yaml +68 -0
- dirigent_examples/shelves/transform/std-convert-fan-out.yaml +142 -0
- dirigent_examples/shelves/transform/xml-feed-to-ndjson.yaml +116 -0
- dirigent_examples/shelves/transform/yaml-config-to-json.yaml +104 -0
- dirigent_examples/shelves/triggers/README.md +45 -0
- dirigent_examples/shelves/triggers/at-one-time.yaml +78 -0
- dirigent_examples/shelves/triggers/cron-nightly.yaml +79 -0
- dirigent_examples/shelves/triggers/cron-windowed.yaml +86 -0
- dirigent_examples/shelves/triggers/document-nightly.yaml +80 -0
- dirigent_examples/shelves/triggers/interval-rolling.yaml +88 -0
- dirigent_examples/shelves/triggers/managed-and-manual.yaml +109 -0
- dirigent_examples/shelves/triggers/webhook-trigger.yaml +75 -0
- dirigent_examples/shelves/validate/README.md +31 -0
- dirigent_examples/shelves/validate/expects-a-shape.yaml +56 -0
- dirigent_examples/shelves/validate/the-shape-is-wrong.yaml +46 -0
- dirigent_examples-0.15.0.dist-info/METADATA +21 -0
- dirigent_examples-0.15.0.dist-info/RECORD +216 -0
- dirigent_examples-0.15.0.dist-info/WHEEL +4 -0
- dirigent_examples-0.15.0.dist-info/entry_points.txt +3 -0
- dirigent_examples-0.15.0.dist-info/licenses/LICENSE +18 -0
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# A step that fails on time rather than on work, and the three clocks that could have done it.
|
|
2
|
+
#
|
|
3
|
+
# The step below asks for thirty seconds of sleep against a two-second budget. It is designed
|
|
4
|
+
# to lose. The run fails in about two seconds, and the interesting part is which of the three
|
|
5
|
+
# available clocks stopped it, because they are not interchangeable:
|
|
6
|
+
#
|
|
7
|
+
# timeout (a step field) How long ONE block call may take before the engine abandons it.
|
|
8
|
+
# This is the one that fires here.
|
|
9
|
+
# deadline (a step field) How long a step may keep WAITING on a sensor or a remote job.
|
|
10
|
+
# A sensor's individual pokes are fast; what needs bounding is
|
|
11
|
+
# the waiting between them. sensor-gate.yaml is that example.
|
|
12
|
+
# timeout (block config) The block's own limit on its own work -- here, when
|
|
13
|
+
# shell.run kills the child process it started.
|
|
14
|
+
#
|
|
15
|
+
# The two on this step are set deliberately far apart. shell.run would let the process run for
|
|
16
|
+
# 300 seconds by default, and the engine gives up after two, so the engine is what fails the
|
|
17
|
+
# step. Widen the step's `timeout` past thirty and the sleep finishes and the step goes green;
|
|
18
|
+
# narrow the block's own `timeout` below two and the block kills its own child first and
|
|
19
|
+
# reports that instead. Same red step, different reason, and the error class in the attempt says which.
|
|
20
|
+
#
|
|
21
|
+
# A timeout is classified as a transient failure, which means a retry budget would spend
|
|
22
|
+
# itself on it: three attempts against a sleep that is always too long is six seconds of
|
|
23
|
+
# waiting and three identical failures. There is no retry here for exactly that reason -- a
|
|
24
|
+
# deadline that cannot be met is not a flake.
|
|
25
|
+
#
|
|
26
|
+
# on_timeout decides what a timeout means. The default is fail, which is what this shows;
|
|
27
|
+
# on_timeout: skip is the other answer, and it is what sensor-gate.yaml uses to let a run
|
|
28
|
+
# carry on past a drop that never landed.
|
|
29
|
+
#
|
|
30
|
+
# This one uses shell.run, which executes code on the worker and is refused unless the
|
|
31
|
+
# instance allowlists it: export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["shell.run"]', or pass
|
|
32
|
+
# --enable-unsafe shell.run to a local run.
|
|
33
|
+
#
|
|
34
|
+
# dg run --local examples/failure/step-timeout.yaml --enable-unsafe shell.run # fails, in ~2s
|
|
35
|
+
# dg run --local examples/failure/step-timeout.yaml --enable-unsafe shell.run -p seconds=1 # succeeds
|
|
36
|
+
|
|
37
|
+
format: dirigent/v1
|
|
38
|
+
kind: pipeline
|
|
39
|
+
code: step-timeout
|
|
40
|
+
name: Step timeout
|
|
41
|
+
description: |
|
|
42
|
+
A sleep that is asked to take thirty seconds, given two.
|
|
43
|
+
|
|
44
|
+
The run fails in about two seconds. It fails **on time, not on work**: the command was
|
|
45
|
+
healthy and would have finished, and the engine abandoned it anyway because the step
|
|
46
|
+
declared how long it was allowed to take.
|
|
47
|
+
|
|
48
|
+
Three clocks could stop a step, and they answer different questions:
|
|
49
|
+
|
|
50
|
+
| Field | Where it lives | What it bounds |
|
|
51
|
+
| --- | --- | --- |
|
|
52
|
+
| `timeout` | the step | one block call |
|
|
53
|
+
| `deadline` | the step | how long a sensor may keep waiting |
|
|
54
|
+
| `timeout` | the block's config | the block's own work |
|
|
55
|
+
|
|
56
|
+
tags: [failure, execute, retry]
|
|
57
|
+
|
|
58
|
+
requires:
|
|
59
|
+
blocks:
|
|
60
|
+
- shell.run
|
|
61
|
+
|
|
62
|
+
params:
|
|
63
|
+
type: object
|
|
64
|
+
properties:
|
|
65
|
+
seconds:
|
|
66
|
+
type: integer
|
|
67
|
+
description: How long the command sleeps. Anything above the two-second budget loses.
|
|
68
|
+
default: 30
|
|
69
|
+
minimum: 0
|
|
70
|
+
maximum: 300
|
|
71
|
+
|
|
72
|
+
steps:
|
|
73
|
+
start:
|
|
74
|
+
block: shell.run
|
|
75
|
+
config:
|
|
76
|
+
argv: [echo, "starting work that will not be allowed to finish"]
|
|
77
|
+
|
|
78
|
+
slow_work:
|
|
79
|
+
block: shell.run
|
|
80
|
+
depends_on: [start]
|
|
81
|
+
# The budget. The command below wants thirty seconds; it gets two, and the engine takes
|
|
82
|
+
# the attempt away from it.
|
|
83
|
+
timeout: 2s
|
|
84
|
+
config:
|
|
85
|
+
# 300 seconds is this block's own default, left as it is so the engine's clock is
|
|
86
|
+
# plainly the shorter one and plainly the one that fires.
|
|
87
|
+
timeout: 5m
|
|
88
|
+
command: "echo 'sleeping ${params.seconds}s'; sleep ${params.seconds}; echo 'never printed'"
|
|
89
|
+
|
|
90
|
+
# Skipped when the sleep is cut short, because the default all_success edge is not
|
|
91
|
+
# satisfied by a step the engine abandoned.
|
|
92
|
+
after:
|
|
93
|
+
block: shell.run
|
|
94
|
+
depends_on: [slow_work]
|
|
95
|
+
config:
|
|
96
|
+
argv: [echo, "only reached when the work fits in its budget"]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Git examples
|
|
2
|
+
|
|
3
|
+
`git.checkout` puts an existing project into a run. It clones a repository at a ref into the
|
|
4
|
+
run's work directory and reports the commit it landed on, so a `docker.build` names its context,
|
|
5
|
+
a `docker.compose.up` names its compose file, and a transform names its files, all by a path
|
|
6
|
+
relative to the checkout. [docs/git.md](../../docs/git.md) is the family's home.
|
|
7
|
+
|
|
8
|
+
It is an **ordinary** block: it writes only under the run's work directory and reaches only the
|
|
9
|
+
remote its connection names, so no id has to be allowlisted to run it.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
dg run --local examples/git/git-checkout-public.yaml
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
The other two documents drive the `docker.*` family off a checkout, so they need a Docker
|
|
16
|
+
daemon and the usual allowlist; each says so in its own header.
|
|
17
|
+
|
|
18
|
+
Every document here carries its `git` connection in a `connections:` section, because a
|
|
19
|
+
`--local` run has no instance to hold one. On a server the connection is created once and the
|
|
20
|
+
document names it:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
dg connection create git my-project --set url=https://github.com/owner/repo.git
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Pipelines
|
|
27
|
+
|
|
28
|
+
| File | What it teaches |
|
|
29
|
+
| --- | --- |
|
|
30
|
+
| [git-checkout-public.yaml](git-checkout-public.yaml) | The block on its own: a public remote, a shallow checkout at the default branch, and the commit, ref, target and remote a downstream step reads. |
|
|
31
|
+
| [git-checkout-build.yaml](git-checkout-build.yaml) | Starting an existing project: the repository's own Dockerfile built from the checkout, and the image run once to prove it starts. |
|
|
32
|
+
| [git-checkout-compose.yaml](git-checkout-compose.yaml) | A stack from somebody else's compose file: the checkout is what puts it where the compose CLI can open it, and the docker family's up/drive/down lifecycle is unchanged. |
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# Starting an existing project: check the repository out, build its Dockerfile, run the image.
|
|
2
|
+
#
|
|
3
|
+
# This is the answer to "I have a project on GitHub, how do I get it into a pipeline". Nothing
|
|
4
|
+
# is copied by hand and no Dockerfile is written into the document: the repository's own build
|
|
5
|
+
# is what runs.
|
|
6
|
+
#
|
|
7
|
+
# REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
|
|
8
|
+
# tcp+TLS, or the local socket in dev), and network from the worker to github.com.
|
|
9
|
+
# docker.build and docker.run declare local_execution, so the engine refuses them unless the
|
|
10
|
+
# instance allowlists their ids -- git.checkout is ordinary and needs no allowlisting:
|
|
11
|
+
# dg run --local examples/git/git-checkout-build.yaml --enable-unsafe docker.run,docker.build
|
|
12
|
+
# export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.run","docker.build"]'
|
|
13
|
+
#
|
|
14
|
+
# Three hops, and what each one hands on:
|
|
15
|
+
# source the repository, shallow, into source/ in the run's work directory. Output: the
|
|
16
|
+
# commit, and the path the build step names next.
|
|
17
|
+
# build buildx builds that directory with the repository's own Dockerfile and loads the
|
|
18
|
+
# image into the worker's daemon store. Output: the image id, its size and its tags.
|
|
19
|
+
# smoke the built image, run once, proving the project starts.
|
|
20
|
+
#
|
|
21
|
+
# THE CHECKOUT IS WHAT MAKES THE BUILD POSSIBLE. docker.build takes a context DIRECTORY on the
|
|
22
|
+
# worker's filesystem and has no inline form, so before this block existed a pipeline had to
|
|
23
|
+
# put every file there itself, one storage.copy at a time. A checkout puts a whole repository
|
|
24
|
+
# there in one step, and the build names it by the same work-directory-relative path the
|
|
25
|
+
# checkout reported.
|
|
26
|
+
#
|
|
27
|
+
# THE TAG IS THE CONTRACT. Nothing is pushed anywhere, so the image exists only in the store of
|
|
28
|
+
# the daemon that built it, under the tag the build gave it. Both steps read that name from one
|
|
29
|
+
# parameter, and both must land on the same docker-capable worker: an image built on worker A
|
|
30
|
+
# is invisible to worker B, which is what requires.workers below routes around.
|
|
31
|
+
#
|
|
32
|
+
# WHY THE RUN STEP OVERRIDES THE COMMAND. The image's own CMD starts an HTTP server and never
|
|
33
|
+
# returns, and a step has to end. So the smoke step starts the project's server, asks it for a
|
|
34
|
+
# page, and exits with whether it answered -- which is what "it builds and it runs" means for
|
|
35
|
+
# a service. Running a stack that stays up for the rest of a run is the compose example's job.
|
|
36
|
+
#
|
|
37
|
+
# TO MAKE IT YOURS: point the connection at your own repository, pin a ref: to a tag or a
|
|
38
|
+
# commit rather than tracking a branch, and replace the smoke step with your project's own
|
|
39
|
+
# check -- its test suite, its migration, its one-shot job.
|
|
40
|
+
|
|
41
|
+
format: dirigent/v1
|
|
42
|
+
kind: pipeline
|
|
43
|
+
code: git-checkout-build
|
|
44
|
+
name: Build a checked-out project
|
|
45
|
+
description: Check a public repository out, build the Dockerfile it ships, and run the image once.
|
|
46
|
+
|
|
47
|
+
tags: [git, execute]
|
|
48
|
+
|
|
49
|
+
requires:
|
|
50
|
+
blocks:
|
|
51
|
+
- git.checkout
|
|
52
|
+
- docker.build
|
|
53
|
+
- docker.run
|
|
54
|
+
workers:
|
|
55
|
+
- docker
|
|
56
|
+
|
|
57
|
+
connections:
|
|
58
|
+
helloworld-demo-node:
|
|
59
|
+
kind: git
|
|
60
|
+
config:
|
|
61
|
+
# Docker's own sample project: a Node server with no dependencies and a Dockerfile at
|
|
62
|
+
# the repository root, which is the shape docker.build expects by default.
|
|
63
|
+
url: https://github.com/dockersamples/helloworld-demo-node.git
|
|
64
|
+
|
|
65
|
+
params:
|
|
66
|
+
type: object
|
|
67
|
+
additionalProperties: false
|
|
68
|
+
properties:
|
|
69
|
+
image_tag:
|
|
70
|
+
type: string
|
|
71
|
+
default: dirigent-helloworld:local
|
|
72
|
+
description: The tag the build gives the image, and the only name the smoke step has for it.
|
|
73
|
+
|
|
74
|
+
steps:
|
|
75
|
+
source:
|
|
76
|
+
block: git.checkout
|
|
77
|
+
deadline: 5m
|
|
78
|
+
config:
|
|
79
|
+
connection: helloworld-demo-node
|
|
80
|
+
# Named rather than defaulted, so the build step below can say "source" out loud instead
|
|
81
|
+
# of depending on what this step happens to be called.
|
|
82
|
+
target: source
|
|
83
|
+
depth: 1
|
|
84
|
+
|
|
85
|
+
build:
|
|
86
|
+
block: docker.build
|
|
87
|
+
depends_on: [source]
|
|
88
|
+
deadline: 15m
|
|
89
|
+
config:
|
|
90
|
+
# The checkout's directory, relative to the run's work directory. The Dockerfile path is
|
|
91
|
+
# relative to it, and
|
|
92
|
+
# "Dockerfile" is already the default; it is written out here because the whole point is
|
|
93
|
+
# that this is the repository's file and not one this pipeline invented.
|
|
94
|
+
context: source
|
|
95
|
+
dockerfile: Dockerfile
|
|
96
|
+
tags: ["${params.image_tag}"]
|
|
97
|
+
# The Dockerfile's npm ci runs inside the build, so the build needs the network the
|
|
98
|
+
# daemon has; nothing else about it is dirigent's business.
|
|
99
|
+
pull: true
|
|
100
|
+
|
|
101
|
+
smoke:
|
|
102
|
+
block: docker.run
|
|
103
|
+
depends_on: [build]
|
|
104
|
+
deadline: 5m
|
|
105
|
+
config:
|
|
106
|
+
image: "${params.image_tag}"
|
|
107
|
+
# pull stays off, and must: the image was never pushed, so a pull would look for this
|
|
108
|
+
# tag in a registry and fail on a name the daemon already holds locally.
|
|
109
|
+
pull: false
|
|
110
|
+
# none, not bridge: the server and the request that proves it are both in this
|
|
111
|
+
# container, so the loopback interface is the whole network this step needs.
|
|
112
|
+
network: none
|
|
113
|
+
# The shell form, because this is a small script rather than one command: start the
|
|
114
|
+
# project's server in the background, then ask it for a page until it answers. Ten
|
|
115
|
+
# tries at a second apart is generous for a process that binds a port and nothing else.
|
|
116
|
+
command: |
|
|
117
|
+
node app.js &
|
|
118
|
+
for _ in 1 2 3 4 5 6 7 8 9 10; do
|
|
119
|
+
wget -qO- http://127.0.0.1:8080/ && exit 0
|
|
120
|
+
sleep 1
|
|
121
|
+
done
|
|
122
|
+
echo "the server never answered" >&2
|
|
123
|
+
exit 1
|
|
124
|
+
memory: 256mb
|
|
125
|
+
pids_limit: 128
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# A stack brought up from a compose file that lives in someone else's repository.
|
|
2
|
+
#
|
|
3
|
+
# The compose file is not in this document and was not written into the work directory by an
|
|
4
|
+
# earlier
|
|
5
|
+
# step: it is a file in a public repository, and the checkout is what puts it somewhere the
|
|
6
|
+
# compose CLI can open.
|
|
7
|
+
#
|
|
8
|
+
# REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
|
|
9
|
+
# tcp+TLS, or the local socket in dev), and network from the worker to github.com. The three
|
|
10
|
+
# docker blocks declare local_execution, so the engine refuses them unless the instance
|
|
11
|
+
# allowlists their ids -- git.checkout is ordinary and needs no allowlisting:
|
|
12
|
+
# dg run --local examples/git/git-checkout-compose.yaml \
|
|
13
|
+
# --enable-unsafe docker.compose.up,docker.compose.down,docker.run
|
|
14
|
+
# export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up","docker.compose.down","docker.run"]'
|
|
15
|
+
#
|
|
16
|
+
# Four hops, and what each one hands on:
|
|
17
|
+
# source the repository, shallow, into source/ in the run's work directory. Output: the
|
|
18
|
+
# commit and the target the compose step names a file inside.
|
|
19
|
+
# stack compose brought up from that file, detached, and left running for the rest of
|
|
20
|
+
# the run. Output: the services, the networks, and default_network.
|
|
21
|
+
# probe a container on the stack's network, asking the service for a page.
|
|
22
|
+
# teardown the same project down again, volumes included, on any outcome.
|
|
23
|
+
#
|
|
24
|
+
# THE PATH IS WORK-DIRECTORY-RELATIVE ALL THE WAY THROUGH. docker.compose.up names a file
|
|
25
|
+
# inside the run's work directory, never an absolute path and never one that climbs out, and
|
|
26
|
+
# the checkout landed the whole repository at "source". So the compose file is
|
|
27
|
+
# "source/<directory>/compose.yaml" -- the repository's own layout, prefixed by where the
|
|
28
|
+
# checkout put it. The CLI is always run with --project-directory set to the work directory,
|
|
29
|
+
# so a compose file with relative paths of its own resolves them against the run's directory
|
|
30
|
+
# rather than its own: a stack of pre-built images, like this one, is what that suits.
|
|
31
|
+
#
|
|
32
|
+
# PIN THE REF FOR ANYTHING REAL. This one tracks the repository's default branch, which is
|
|
33
|
+
# honest for an example and wrong for a pipeline you rely on: somebody else's commit would
|
|
34
|
+
# change what your stack is. A ref: naming a tag or a full commit sha is how a checkout stops
|
|
35
|
+
# being a moving target, and the commit output is what to record when it is not.
|
|
36
|
+
#
|
|
37
|
+
# THE LIFECYCLE IS THE DOCKER FAMILY'S, UNCHANGED. up brings the stack up and it persists, a
|
|
38
|
+
# step drives it, and a down with rule: all_done tears it down whether the drive step passed
|
|
39
|
+
# or failed. Both compose steps default their project name from the run id, so the down
|
|
40
|
+
# addresses exactly the project the up created with nothing wired between them.
|
|
41
|
+
#
|
|
42
|
+
# TO MAKE IT YOURS: point the connection at the repository whose stack you actually run, name
|
|
43
|
+
# its compose file under compose_path, and replace the probe with something that proves your
|
|
44
|
+
# services are up -- a request that must return 200, a query that must return rows.
|
|
45
|
+
|
|
46
|
+
format: dirigent/v1
|
|
47
|
+
kind: pipeline
|
|
48
|
+
code: git-checkout-compose
|
|
49
|
+
name: A stack from a repository's compose file
|
|
50
|
+
description: Check a repository out, bring up the compose file it ships, probe it, and tear it down.
|
|
51
|
+
|
|
52
|
+
tags: [git, execute]
|
|
53
|
+
|
|
54
|
+
requires:
|
|
55
|
+
blocks:
|
|
56
|
+
- git.checkout
|
|
57
|
+
- docker.compose.up
|
|
58
|
+
- docker.compose.down
|
|
59
|
+
- docker.run
|
|
60
|
+
|
|
61
|
+
connections:
|
|
62
|
+
awesome-compose:
|
|
63
|
+
kind: git
|
|
64
|
+
config:
|
|
65
|
+
# Docker's own collection of sample stacks. It is public, so no credential is sealed
|
|
66
|
+
# here; a private repository would carry a token or a deploy key on this connection and
|
|
67
|
+
# the steps below would not change at all.
|
|
68
|
+
url: https://github.com/docker/awesome-compose.git
|
|
69
|
+
|
|
70
|
+
params:
|
|
71
|
+
type: object
|
|
72
|
+
additionalProperties: false
|
|
73
|
+
properties:
|
|
74
|
+
compose_path:
|
|
75
|
+
type: string
|
|
76
|
+
default: source/wordpress-mysql/compose.yaml
|
|
77
|
+
description: The compose file to bring up, as a path under the run's work directory.
|
|
78
|
+
|
|
79
|
+
steps:
|
|
80
|
+
source:
|
|
81
|
+
block: git.checkout
|
|
82
|
+
deadline: 10m
|
|
83
|
+
config:
|
|
84
|
+
connection: awesome-compose
|
|
85
|
+
target: source
|
|
86
|
+
# One commit and none of its history: this pipeline reads two files out of the tree and
|
|
87
|
+
# has no use for the years of log behind them.
|
|
88
|
+
depth: 1
|
|
89
|
+
|
|
90
|
+
stack:
|
|
91
|
+
block: docker.compose.up
|
|
92
|
+
depends_on: [source]
|
|
93
|
+
deadline: 15m
|
|
94
|
+
config:
|
|
95
|
+
# A file, not content: this is the form for a compose document something else produced,
|
|
96
|
+
# and a checkout is the most direct producer there is.
|
|
97
|
+
file: "${params.compose_path}"
|
|
98
|
+
# The stack's services declare no healthcheck, so --wait can only wait for their
|
|
99
|
+
# containers to be running; the probe below is what waits for the application itself.
|
|
100
|
+
wait: true
|
|
101
|
+
wait_timeout: 5m
|
|
102
|
+
pull: always
|
|
103
|
+
|
|
104
|
+
probe:
|
|
105
|
+
block: docker.run
|
|
106
|
+
depends_on: [stack]
|
|
107
|
+
deadline: 10m
|
|
108
|
+
config:
|
|
109
|
+
image: alpine:3
|
|
110
|
+
pull: true
|
|
111
|
+
# The network the up step reported: joining it resolves the service by the name the
|
|
112
|
+
# compose file gave it.
|
|
113
|
+
network: "${steps.stack.output.default_network}"
|
|
114
|
+
# The web service answers seconds after its container is running, and only once the
|
|
115
|
+
# database behind it does, so the probe retries rather than racing it. This is what a
|
|
116
|
+
# stack without a healthcheck costs the step that drives it.
|
|
117
|
+
command: |
|
|
118
|
+
for _ in $(seq 1 90); do
|
|
119
|
+
wget -q --spider http://wordpress/ && exit 0
|
|
120
|
+
sleep 2
|
|
121
|
+
done
|
|
122
|
+
echo "the service never answered" >&2
|
|
123
|
+
exit 1
|
|
124
|
+
memory: 64mb
|
|
125
|
+
pids_limit: 64
|
|
126
|
+
|
|
127
|
+
teardown:
|
|
128
|
+
block: docker.compose.down
|
|
129
|
+
depends_on: [probe]
|
|
130
|
+
# all_done runs the teardown on success or failure, so the stack never outlives the run.
|
|
131
|
+
rule: all_done
|
|
132
|
+
deadline: 10m
|
|
133
|
+
config:
|
|
134
|
+
# The same file the up step used, passed back from its output: a teardown resolves the
|
|
135
|
+
# project by label alone in most cases, and needs -f in the ones where it cannot.
|
|
136
|
+
file: "${steps.stack.output.compose_file}"
|
|
137
|
+
# The stack's named volumes go with it, so a run leaves nothing on the worker.
|
|
138
|
+
down_volumes: true
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# A public repository checked out into the run's work directory, and the commit it landed on.
|
|
2
|
+
#
|
|
3
|
+
# NEEDS NETWORK from the worker to github.com, and nothing else: git.checkout is an ordinary
|
|
4
|
+
# block, so no id has to be allowlisted and no daemon has to be reachable.
|
|
5
|
+
# dg run --local examples/git/git-checkout-public.yaml
|
|
6
|
+
# docs/git.md is the family's home.
|
|
7
|
+
#
|
|
8
|
+
# Two hops, and what each one hands on:
|
|
9
|
+
# checkout the repository, shallow, at its default branch, at checkout/ in the run's
|
|
10
|
+
# work directory.
|
|
11
|
+
# Output: commit, ref, target and remote.
|
|
12
|
+
# report a constant step that reads those three fields, which is all it takes to prove
|
|
13
|
+
# they are references any downstream step can name.
|
|
14
|
+
#
|
|
15
|
+
# THE CONNECTION HOLDS THE REMOTE, THE DOCUMENT HOLDS THE REF. A checkout step never writes a
|
|
16
|
+
# URL: it names a git connection, and the connection is what carries the remote and, when the
|
|
17
|
+
# repository is private, the sealed token or deploy key that reaches it. Moving a pipeline
|
|
18
|
+
# from a fork to the real repository is then an edit to one connection and to no pipeline.
|
|
19
|
+
# This one is carried in the document because a --local run has no instance to hold it; on a
|
|
20
|
+
# server it would be created once, with `dg connection create git ...`, and the document would
|
|
21
|
+
# name it and stop there. Public repositories need no credential at all, which is why there is
|
|
22
|
+
# none here.
|
|
23
|
+
#
|
|
24
|
+
# WHERE IT LANDS. target is a path inside the run's work directory -- never absolute, never
|
|
25
|
+
# climbing out -- and left unset it is the step's own name, so this checkout is at
|
|
26
|
+
# checkout/. A checkout is a directory a tool opens rather than bytes in storage, which is
|
|
27
|
+
# why it lands there and not under ${run.scratch}. That relative path is the whole point of
|
|
28
|
+
# the block: a downstream docker.build names it as a context, a docker.compose.up names a
|
|
29
|
+
# file inside it, and a transform names its files, none of them knowing where the run's work
|
|
30
|
+
# directory actually is.
|
|
31
|
+
#
|
|
32
|
+
# SHALLOW BY DEFAULT. depth is 1 unless a document says otherwise, which fetches the one
|
|
33
|
+
# commit the ref points at and none of its history. That is what a build wants and it is a
|
|
34
|
+
# fraction of the bytes. Set depth: 0 for the whole history, which is what a step that reads
|
|
35
|
+
# the log or runs `git describe` needs.
|
|
36
|
+
#
|
|
37
|
+
# TO MAKE IT YOURS: point the connection at your own repository, add a ref: if the default
|
|
38
|
+
# branch is not what you want, and replace the report step with whatever actually reads the
|
|
39
|
+
# checkout.
|
|
40
|
+
|
|
41
|
+
format: dirigent/v1
|
|
42
|
+
kind: pipeline
|
|
43
|
+
code: git-checkout-public
|
|
44
|
+
name: Check a public repository out
|
|
45
|
+
description: Clone a public repository at its default branch into the run's work directory and report the commit.
|
|
46
|
+
|
|
47
|
+
tags: [git]
|
|
48
|
+
|
|
49
|
+
requires:
|
|
50
|
+
blocks:
|
|
51
|
+
- git.checkout
|
|
52
|
+
- value.const
|
|
53
|
+
|
|
54
|
+
connections:
|
|
55
|
+
octocat-hello-world:
|
|
56
|
+
kind: git
|
|
57
|
+
config:
|
|
58
|
+
# GitHub's own smallest repository: one README and two commits. Nothing is sealed here
|
|
59
|
+
# because nothing needs to be -- a public remote takes no credential.
|
|
60
|
+
url: https://github.com/octocat/Hello-World.git
|
|
61
|
+
|
|
62
|
+
steps:
|
|
63
|
+
checkout:
|
|
64
|
+
block: git.checkout
|
|
65
|
+
deadline: 5m
|
|
66
|
+
config:
|
|
67
|
+
connection: octocat-hello-world
|
|
68
|
+
# No ref, so the remote's own default branch is what lands, and the output reports which
|
|
69
|
+
# one that turned out to be.
|
|
70
|
+
depth: 1
|
|
71
|
+
|
|
72
|
+
report:
|
|
73
|
+
block: value.const
|
|
74
|
+
depends_on: [checkout]
|
|
75
|
+
config:
|
|
76
|
+
# The commit is the only exact name for what was checked out: a branch moves, a tag can
|
|
77
|
+
# be moved, and a sha cannot. A step that records what a pipeline built records this.
|
|
78
|
+
value:
|
|
79
|
+
commit: "${steps.checkout.output.commit}"
|
|
80
|
+
ref: "${steps.checkout.output.ref}"
|
|
81
|
+
# The remote as the output reports it, which is the URL with any credential stripped.
|
|
82
|
+
remote: "${steps.checkout.output.remote}"
|
|
83
|
+
# Relative to the run's work directory, which is the form every downstream block wants.
|
|
84
|
+
target: "${steps.checkout.output.target}"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Graph examples
|
|
2
|
+
|
|
3
|
+
The shapes a DAG takes: a straight line, branches that split and join, a fan over a list,
|
|
4
|
+
and a chain deep enough to watch the engine walk it. Every one runs with `dg run --local`,
|
|
5
|
+
and the ones that call out use only Postman Echo.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
dg run --local examples/graph/linear.yaml -p day=2026-01-01 --enable-unsafe shell.run
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Pipelines
|
|
12
|
+
|
|
13
|
+
| File | What it teaches |
|
|
14
|
+
| --- | --- |
|
|
15
|
+
| [linear.yaml](linear.yaml) | The straight line: fetch, archive, report, each step reading the one before it. |
|
|
16
|
+
| [parallel-branches.yaml](parallel-branches.yaml) | Two branches from one root, joined by a step that reads both. |
|
|
17
|
+
| [fan-out.yaml](fan-out.yaml) | `for_each` over a list: one item per region, and a refusal that fails the item rather than the fan. |
|
|
18
|
+
| [fan-in.yaml](fan-in.yaml) | The other direction: parallel fetches whose whole fan one step reads back as a list. |
|
|
19
|
+
| [wide-fan.yaml](wide-fan.yaml) | A fan wide enough to fill every worker slot, for watching concurrency rather than reading about it. |
|
|
20
|
+
| [deep-chain.yaml](deep-chain.yaml) | Ten steps in one line, each reading the one above it. |
|
|
21
|
+
| [skip-diamond.yaml](skip-diamond.yaml) | A diamond where one road is taken and the other settles as skipped, not failed. |
|
|
22
|
+
| [parallel-sleep.yaml](parallel-sleep.yaml) | Three independent waits at once, and the join that waits for all of them. |
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# Graph height: ten steps in a single file, where every one waits for the one before it.
|
|
2
|
+
#
|
|
3
|
+
# This document exists to be looked at rather than to compute anything. Height is the length
|
|
4
|
+
# of a DAG's longest path, and it is the one dimension no amount of hardware shortens: each
|
|
5
|
+
# step here reads the step above it, so the tenth cannot start until the ninth has settled.
|
|
6
|
+
# Ten workers finish this run no faster than one does.
|
|
7
|
+
#
|
|
8
|
+
# wide-fan.yaml is the same idea turned ninety degrees -- ten steps that are all claimable at
|
|
9
|
+
# once. Run both and watch the graph: this one draws a column, that one draws a row. Between
|
|
10
|
+
# them they are the two numbers worth knowing about a pipeline's shape, because height is
|
|
11
|
+
# what sets the floor on a run's duration and width is what sets the demand on the workers.
|
|
12
|
+
#
|
|
13
|
+
# The value threaded down the chain is a running total, so each step's output visibly depends
|
|
14
|
+
# on its predecessor's rather than merely coming after it in the drawing. transform.jq opens
|
|
15
|
+
# no socket and runs nothing on the worker, so the whole column needs no allowlist entry and
|
|
16
|
+
# no network.
|
|
17
|
+
#
|
|
18
|
+
# dg run --local examples/graph/deep-chain.yaml
|
|
19
|
+
# dg run --local examples/graph/deep-chain.yaml -p start=100
|
|
20
|
+
|
|
21
|
+
format: dirigent/v1
|
|
22
|
+
kind: pipeline
|
|
23
|
+
code: deep-chain
|
|
24
|
+
name: Deep chain
|
|
25
|
+
description: |
|
|
26
|
+
Ten steps in one line, each reading the one above it.
|
|
27
|
+
|
|
28
|
+
The longest path through a DAG is its **height**, and it is the floor under the run's
|
|
29
|
+
duration: no number of workers makes a chain shorter. Its opposite is
|
|
30
|
+
`wide-fan`, where all ten steps are claimable at once.
|
|
31
|
+
|
|
32
|
+
tags: [graph, transform]
|
|
33
|
+
|
|
34
|
+
requires:
|
|
35
|
+
blocks:
|
|
36
|
+
- transform.jq
|
|
37
|
+
|
|
38
|
+
params:
|
|
39
|
+
type: object
|
|
40
|
+
properties:
|
|
41
|
+
start:
|
|
42
|
+
type: integer
|
|
43
|
+
description: What the first link adds to.
|
|
44
|
+
default: 0
|
|
45
|
+
|
|
46
|
+
steps:
|
|
47
|
+
# The first link is the only one with no depends_on, so it is the only root: the run has
|
|
48
|
+
# exactly one claimable step at the moment it is created, and never more than one after.
|
|
49
|
+
link_1:
|
|
50
|
+
block: transform.jq
|
|
51
|
+
config:
|
|
52
|
+
input:
|
|
53
|
+
total: "${params.start}"
|
|
54
|
+
program: "{total: (.total + 1), depth: 1}"
|
|
55
|
+
|
|
56
|
+
link_2:
|
|
57
|
+
block: transform.jq
|
|
58
|
+
depends_on: [link_1]
|
|
59
|
+
config:
|
|
60
|
+
input: "${steps.link_1.output.value}"
|
|
61
|
+
program: "{total: (.total + 2), depth: 2}"
|
|
62
|
+
|
|
63
|
+
link_3:
|
|
64
|
+
block: transform.jq
|
|
65
|
+
depends_on: [link_2]
|
|
66
|
+
config:
|
|
67
|
+
input: "${steps.link_2.output.value}"
|
|
68
|
+
program: "{total: (.total + 3), depth: 3}"
|
|
69
|
+
|
|
70
|
+
link_4:
|
|
71
|
+
block: transform.jq
|
|
72
|
+
depends_on: [link_3]
|
|
73
|
+
config:
|
|
74
|
+
input: "${steps.link_3.output.value}"
|
|
75
|
+
program: "{total: (.total + 4), depth: 4}"
|
|
76
|
+
|
|
77
|
+
link_5:
|
|
78
|
+
block: transform.jq
|
|
79
|
+
depends_on: [link_4]
|
|
80
|
+
config:
|
|
81
|
+
input: "${steps.link_4.output.value}"
|
|
82
|
+
program: "{total: (.total + 5), depth: 5}"
|
|
83
|
+
|
|
84
|
+
link_6:
|
|
85
|
+
block: transform.jq
|
|
86
|
+
depends_on: [link_5]
|
|
87
|
+
config:
|
|
88
|
+
input: "${steps.link_5.output.value}"
|
|
89
|
+
program: "{total: (.total + 6), depth: 6}"
|
|
90
|
+
|
|
91
|
+
link_7:
|
|
92
|
+
block: transform.jq
|
|
93
|
+
depends_on: [link_6]
|
|
94
|
+
config:
|
|
95
|
+
input: "${steps.link_6.output.value}"
|
|
96
|
+
program: "{total: (.total + 7), depth: 7}"
|
|
97
|
+
|
|
98
|
+
link_8:
|
|
99
|
+
block: transform.jq
|
|
100
|
+
depends_on: [link_7]
|
|
101
|
+
config:
|
|
102
|
+
input: "${steps.link_7.output.value}"
|
|
103
|
+
program: "{total: (.total + 8), depth: 8}"
|
|
104
|
+
|
|
105
|
+
link_9:
|
|
106
|
+
block: transform.jq
|
|
107
|
+
depends_on: [link_8]
|
|
108
|
+
config:
|
|
109
|
+
input: "${steps.link_8.output.value}"
|
|
110
|
+
program: "{total: (.total + 9), depth: 9}"
|
|
111
|
+
|
|
112
|
+
# The last link reports the height it sits at alongside the total, so the run's output says
|
|
113
|
+
# what the drawing says.
|
|
114
|
+
link_10:
|
|
115
|
+
block: transform.jq
|
|
116
|
+
depends_on: [link_9]
|
|
117
|
+
config:
|
|
118
|
+
input: "${steps.link_9.output.value}"
|
|
119
|
+
program: "{total: (.total + 10), depth: 10, note: \"55 added over ten dependent steps\"}"
|