dirigent-examples 0.15.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. dirigent_examples/__init__.py +22 -0
  2. dirigent_examples/py.typed +0 -0
  3. dirigent_examples/shelves/README.md +299 -0
  4. dirigent_examples/shelves/composition/README.md +18 -0
  5. dirigent_examples/shelves/composition/chained-instances.yaml +97 -0
  6. dirigent_examples/shelves/composition/composition-child.yaml +64 -0
  7. dirigent_examples/shelves/composition/composition-parent.yaml +89 -0
  8. dirigent_examples/shelves/connections.yaml +52 -0
  9. dirigent_examples/shelves/demo/README.md +19 -0
  10. dirigent_examples/shelves/demo/markdown-showcase.yaml +117 -0
  11. dirigent_examples/shelves/demo/params-showcase.yaml +92 -0
  12. dirigent_examples/shelves/demo/requires.yaml +65 -0
  13. dirigent_examples/shelves/demo/weekly-import-malawi.yaml +54 -0
  14. dirigent_examples/shelves/demo/weekly-import-nepal.yaml +75 -0
  15. dirigent_examples/shelves/docker/README.md +29 -0
  16. dirigent_examples/shelves/docker/docker-build-push.yaml +110 -0
  17. dirigent_examples/shelves/docker/docker-build-run.yaml +106 -0
  18. dirigent_examples/shelves/docker/docker-compose-database.yaml +124 -0
  19. dirigent_examples/shelves/docker/docker-compose-failing-up.yaml +83 -0
  20. dirigent_examples/shelves/docker/docker-compose-file.yaml +117 -0
  21. dirigent_examples/shelves/docker/docker-compose-profiles-env.yaml +133 -0
  22. dirigent_examples/shelves/docker/docker-compose-stack.yaml +70 -0
  23. dirigent_examples/shelves/docker/docker-hello.yaml +53 -0
  24. dirigent_examples/shelves/docker/docker-remote-daemon.yaml +92 -0
  25. dirigent_examples/shelves/docker/docker-run-failing-teardown.yaml +94 -0
  26. dirigent_examples/shelves/docker/docker-ticker.yaml +49 -0
  27. dirigent_examples/shelves/execute/README.md +16 -0
  28. dirigent_examples/shelves/execute/long-log.yaml +89 -0
  29. dirigent_examples/shelves/failure/README.md +20 -0
  30. dirigent_examples/shelves/failure/error-handler.yaml +89 -0
  31. dirigent_examples/shelves/failure/optional-step.yaml +82 -0
  32. dirigent_examples/shelves/failure/retries.yaml +75 -0
  33. dirigent_examples/shelves/failure/retry-budget.yaml +82 -0
  34. dirigent_examples/shelves/failure/step-timeout.yaml +96 -0
  35. dirigent_examples/shelves/git/README.md +32 -0
  36. dirigent_examples/shelves/git/git-checkout-build.yaml +125 -0
  37. dirigent_examples/shelves/git/git-checkout-compose.yaml +138 -0
  38. dirigent_examples/shelves/git/git-checkout-public.yaml +84 -0
  39. dirigent_examples/shelves/graph/README.md +22 -0
  40. dirigent_examples/shelves/graph/deep-chain.yaml +119 -0
  41. dirigent_examples/shelves/graph/fan-in.yaml +80 -0
  42. dirigent_examples/shelves/graph/fan-out.yaml +66 -0
  43. dirigent_examples/shelves/graph/linear.yaml +66 -0
  44. dirigent_examples/shelves/graph/parallel-branches.yaml +57 -0
  45. dirigent_examples/shelves/graph/parallel-sleep.yaml +55 -0
  46. dirigent_examples/shelves/graph/skip-diamond.yaml +105 -0
  47. dirigent_examples/shelves/graph/wide-fan.yaml +147 -0
  48. dirigent_examples/shelves/hello-world.yaml +30 -0
  49. dirigent_examples/shelves/open-data/README.md +67 -0
  50. dirigent_examples/shelves/open-data/feeds-composition.yaml +120 -0
  51. dirigent_examples/shelves/open-data/gdacs-disaster-updates.yaml +230 -0
  52. dirigent_examples/shelves/open-data/github-releases-relay.yaml +225 -0
  53. dirigent_examples/shelves/open-data/hdx-dataset-watch.yaml +211 -0
  54. dirigent_examples/shelves/open-data/kobo-submissions-to-csv.yaml +149 -0
  55. dirigent_examples/shelves/open-data/nominatim-geocode-facilities.yaml +169 -0
  56. dirigent_examples/shelves/open-data/odk-central-submissions.yaml +146 -0
  57. dirigent_examples/shelves/open-data/open-meteo-weekly-report.yaml +142 -0
  58. dirigent_examples/shelves/open-data/overpass-health-facilities.yaml +154 -0
  59. dirigent_examples/shelves/open-data/usgs-earthquakes-alert.yaml +208 -0
  60. dirigent_examples/shelves/open-data/who-gho-indicators-to-parquet.yaml +163 -0
  61. dirigent_examples/shelves/open-data/wikidata-country-reference.yaml +146 -0
  62. dirigent_examples/shelves/open-data/world-bank-population-trend.yaml +166 -0
  63. dirigent_examples/shelves/patterns/README.md +144 -0
  64. dirigent_examples/shelves/patterns/concurrency-queue.yaml +89 -0
  65. dirigent_examples/shelves/patterns/concurrency-replace.yaml +92 -0
  66. dirigent_examples/shelves/patterns/concurrency-skip.yaml +97 -0
  67. dirigent_examples/shelves/patterns/connections-referenced-vs-carried.yaml +140 -0
  68. dirigent_examples/shelves/patterns/deadline-on-a-sensor.yaml +106 -0
  69. dirigent_examples/shelves/patterns/fan-out-continue.yaml +88 -0
  70. dirigent_examples/shelves/patterns/fan-out-fail-fast.yaml +80 -0
  71. dirigent_examples/shelves/patterns/fan-out-from-params.yaml +84 -0
  72. dirigent_examples/shelves/patterns/fan-out-item-wise.yaml +111 -0
  73. dirigent_examples/shelves/patterns/fan-out-literal-list.yaml +81 -0
  74. dirigent_examples/shelves/patterns/fan-out-nested-objects.yaml +107 -0
  75. dirigent_examples/shelves/patterns/fan-out-then-join.yaml +86 -0
  76. dirigent_examples/shelves/patterns/log-levels.yaml +119 -0
  77. dirigent_examples/shelves/patterns/outputs-inline-vs-storage.yaml +140 -0
  78. dirigent_examples/shelves/patterns/params-every-type.yaml +259 -0
  79. dirigent_examples/shelves/patterns/params-validation-refuses.yaml +131 -0
  80. dirigent_examples/shelves/patterns/pipeline-run-child.yaml +96 -0
  81. dirigent_examples/shelves/patterns/pipeline-run-fire-and-forget.yaml +99 -0
  82. dirigent_examples/shelves/patterns/pipeline-run-strict.yaml +103 -0
  83. dirigent_examples/shelves/patterns/pipeline-run-wait.yaml +107 -0
  84. dirigent_examples/shelves/patterns/pipeline-run-with-params.yaml +124 -0
  85. dirigent_examples/shelves/patterns/poll-cadence.yaml +102 -0
  86. dirigent_examples/shelves/patterns/priority-layered.yaml +120 -0
  87. dirigent_examples/shelves/patterns/references-cheat-sheet.yaml +186 -0
  88. dirigent_examples/shelves/patterns/retry-budget-exhausted.yaml +92 -0
  89. dirigent_examples/shelves/patterns/retry-exponential-backoff.yaml +88 -0
  90. dirigent_examples/shelves/patterns/retry-only-transient.yaml +108 -0
  91. dirigent_examples/shelves/patterns/retry-with-jitter.yaml +102 -0
  92. dirigent_examples/shelves/patterns/rule-all-done.yaml +83 -0
  93. dirigent_examples/shelves/patterns/rule-all-success.yaml +79 -0
  94. dirigent_examples/shelves/patterns/rule-always.yaml +92 -0
  95. dirigent_examples/shelves/patterns/rule-one-failed.yaml +86 -0
  96. dirigent_examples/shelves/patterns/schedule-at-once.yaml +110 -0
  97. dirigent_examples/shelves/patterns/schedule-cron-timezone.yaml +121 -0
  98. dirigent_examples/shelves/patterns/schedule-interval.yaml +109 -0
  99. dirigent_examples/shelves/patterns/schedule-window-half-open.yaml +105 -0
  100. dirigent_examples/shelves/patterns/sensor-http-ready.yaml +121 -0
  101. dirigent_examples/shelves/patterns/sensor-storage-exists.yaml +134 -0
  102. dirigent_examples/shelves/patterns/step-names-and-keys.yaml +99 -0
  103. dirigent_examples/shelves/patterns/timeout-fails-the-step.yaml +94 -0
  104. dirigent_examples/shelves/patterns/timeout-skips-the-step.yaml +102 -0
  105. dirigent_examples/shelves/patterns/webhook-mapping-nested-payload.yaml +125 -0
  106. dirigent_examples/shelves/patterns/webhook-signed.yaml +144 -0
  107. dirigent_examples/shelves/preview/s3-parquet-to-ingestion.yaml +92 -0
  108. dirigent_examples/shelves/python/README.md +31 -0
  109. dirigent_examples/shelves/python/apply_and_run.py +52 -0
  110. dirigent_examples/shelves/python/ci_gate.py +76 -0
  111. dirigent_examples/shelves/python/connections.py +61 -0
  112. dirigent_examples/shelves/python/error_handling.py +84 -0
  113. dirigent_examples/shelves/python/follow_logs.py +39 -0
  114. dirigent_examples/shelves/python/list_and_filter.py +52 -0
  115. dirigent_examples/shelves/queues/README.md +59 -0
  116. dirigent_examples/shelves/queues/kafka-consume-then-transform.yaml +105 -0
  117. dirigent_examples/shelves/queues/kafka-produce-then-consume.yaml +124 -0
  118. dirigent_examples/shelves/queues/rabbitmq-consume-ack-on-success.yaml +102 -0
  119. dirigent_examples/shelves/queues/report-to-kafka.yaml +105 -0
  120. dirigent_examples/shelves/queues/report-to-rabbitmq.yaml +105 -0
  121. dirigent_examples/shelves/recipes/README.md +130 -0
  122. dirigent_examples/shelves/recipes/csv-header-rules.yaml +147 -0
  123. dirigent_examples/shelves/recipes/csv-to-ndjson.yaml +107 -0
  124. dirigent_examples/shelves/recipes/etl-csv-clean-validate-parquet.yaml +207 -0
  125. dirigent_examples/shelves/recipes/filter-by-predicate.yaml +116 -0
  126. dirigent_examples/shelves/recipes/filter-then-map-then-reduce.yaml +109 -0
  127. dirigent_examples/shelves/recipes/http-fetch-validate-post.yaml +142 -0
  128. dirigent_examples/shelves/recipes/http-follow-redirects.yaml +100 -0
  129. dirigent_examples/shelves/recipes/http-get-with-query.yaml +96 -0
  130. dirigent_examples/shelves/recipes/http-headers-and-auth-connection.yaml +110 -0
  131. dirigent_examples/shelves/recipes/http-post-file-from-storage.yaml +117 -0
  132. dirigent_examples/shelves/recipes/http-post-json-echo.yaml +105 -0
  133. dirigent_examples/shelves/recipes/http-post-report.yaml +183 -0
  134. dirigent_examples/shelves/recipes/http-save-body-to-storage.yaml +106 -0
  135. dirigent_examples/shelves/recipes/http-success-status-list.yaml +80 -0
  136. dirigent_examples/shelves/recipes/http-timeout-override.yaml +104 -0
  137. dirigent_examples/shelves/recipes/jq-dedupe-by-key.yaml +78 -0
  138. dirigent_examples/shelves/recipes/jq-defaults-and-nulls.yaml +91 -0
  139. dirigent_examples/shelves/recipes/jq-group-by-and-sum.yaml +74 -0
  140. dirigent_examples/shelves/recipes/jq-join-two-lists.yaml +77 -0
  141. dirigent_examples/shelves/recipes/jq-long-to-wide.yaml +76 -0
  142. dirigent_examples/shelves/recipes/jq-nested-to-flat.yaml +89 -0
  143. dirigent_examples/shelves/recipes/jq-pivot-wide-to-long.yaml +65 -0
  144. dirigent_examples/shelves/recipes/jq-running-totals.yaml +82 -0
  145. dirigent_examples/shelves/recipes/jq-string-cleaning.yaml +88 -0
  146. dirigent_examples/shelves/recipes/jq-top-n.yaml +85 -0
  147. dirigent_examples/shelves/recipes/jq-validate-in-jq-vs-schema.yaml +124 -0
  148. dirigent_examples/shelves/recipes/jq-window-dates.yaml +82 -0
  149. dirigent_examples/shelves/recipes/json-to-csv-flattening.yaml +141 -0
  150. dirigent_examples/shelves/recipes/large-output-to-storage.yaml +134 -0
  151. dirigent_examples/shelves/recipes/map-enrich-with-lookup.yaml +96 -0
  152. dirigent_examples/shelves/recipes/ndjson-to-parquet.yaml +130 -0
  153. dirigent_examples/shelves/recipes/pagination-by-fan-out.yaml +115 -0
  154. dirigent_examples/shelves/recipes/parquet-round-trip-types.yaml +163 -0
  155. dirigent_examples/shelves/recipes/reconcile-two-sources.yaml +159 -0
  156. dirigent_examples/shelves/recipes/report-built-in.yaml +72 -0
  157. dirigent_examples/shelves/recipes/report-daily-digest.yaml +186 -0
  158. dirigent_examples/shelves/recipes/report-to-file.yaml +131 -0
  159. dirigent_examples/shelves/recipes/report-to-webhook.yaml +136 -0
  160. dirigent_examples/shelves/recipes/schema-carried.yaml +112 -0
  161. dirigent_examples/shelves/recipes/schema-formats.yaml +107 -0
  162. dirigent_examples/shelves/recipes/schema-referenced.yaml +86 -0
  163. dirigent_examples/shelves/recipes/schema-refuses-then-rule.yaml +127 -0
  164. dirigent_examples/shelves/recipes/storage-copy-dated-archive.yaml +114 -0
  165. dirigent_examples/shelves/recipes/storage-exists-gate.yaml +127 -0
  166. dirigent_examples/shelves/recipes/storage-manifest-of-a-fan-out.yaml +104 -0
  167. dirigent_examples/shelves/recipes/storage-write-then-read.yaml +119 -0
  168. dirigent_examples/shelves/recipes/webhook-post-hmac.yaml +132 -0
  169. dirigent_examples/shelves/recipes/webhook-post-summary.yaml +142 -0
  170. dirigent_examples/shelves/s3/README.md +34 -0
  171. dirigent_examples/shelves/s3/report-to-s3.yaml +93 -0
  172. dirigent_examples/shelves/s3/s3-copy-and-verify.yaml +105 -0
  173. dirigent_examples/shelves/s3/s3-csv-report.yaml +87 -0
  174. dirigent_examples/shelves/s3/s3-parquet-report.yaml +77 -0
  175. dirigent_examples/shelves/s3/s3-round-trip.yaml +125 -0
  176. dirigent_examples/shelves/schemas/README.md +36 -0
  177. dirigent_examples/shelves/schemas/echo-reading.json +18 -0
  178. dirigent_examples/shelves/schemas/ou-record.json +13 -0
  179. dirigent_examples/shelves/schemas/station-reading.json +13 -0
  180. dirigent_examples/shelves/sensors/README.md +16 -0
  181. dirigent_examples/shelves/sensors/sensor-gate.yaml +65 -0
  182. dirigent_examples/shelves/sensors/time-window.yaml +61 -0
  183. dirigent_examples/shelves/sql/README.md +52 -0
  184. dirigent_examples/shelves/sql/duckdb-parquet-to-report.yaml +146 -0
  185. dirigent_examples/shelves/sql/sql-postgres-readonly.yaml +111 -0
  186. dirigent_examples/shelves/sql/sql-query-to-storage.yaml +85 -0
  187. dirigent_examples/shelves/sql/sql-sqlite-roundtrip.yaml +114 -0
  188. dirigent_examples/shelves/sql/warehouse.sql +42 -0
  189. dirigent_examples/shelves/transform/README.md +36 -0
  190. dirigent_examples/shelves/transform/csv-report.yaml +55 -0
  191. dirigent_examples/shelves/transform/jq-filter-and-map.yaml +70 -0
  192. dirigent_examples/shelves/transform/jq-group-and-aggregate.yaml +70 -0
  193. dirigent_examples/shelves/transform/jq-join-two-sources.yaml +98 -0
  194. dirigent_examples/shelves/transform/jq-reshape.yaml +91 -0
  195. dirigent_examples/shelves/transform/jq-stream-through-storage.yaml +112 -0
  196. dirigent_examples/shelves/transform/ndjson-round-trip.yaml +56 -0
  197. dirigent_examples/shelves/transform/parquet-round-trip.yaml +68 -0
  198. dirigent_examples/shelves/transform/std-convert-fan-out.yaml +142 -0
  199. dirigent_examples/shelves/transform/xml-feed-to-ndjson.yaml +116 -0
  200. dirigent_examples/shelves/transform/yaml-config-to-json.yaml +104 -0
  201. dirigent_examples/shelves/triggers/README.md +45 -0
  202. dirigent_examples/shelves/triggers/at-one-time.yaml +78 -0
  203. dirigent_examples/shelves/triggers/cron-nightly.yaml +79 -0
  204. dirigent_examples/shelves/triggers/cron-windowed.yaml +86 -0
  205. dirigent_examples/shelves/triggers/document-nightly.yaml +80 -0
  206. dirigent_examples/shelves/triggers/interval-rolling.yaml +88 -0
  207. dirigent_examples/shelves/triggers/managed-and-manual.yaml +109 -0
  208. dirigent_examples/shelves/triggers/webhook-trigger.yaml +75 -0
  209. dirigent_examples/shelves/validate/README.md +31 -0
  210. dirigent_examples/shelves/validate/expects-a-shape.yaml +56 -0
  211. dirigent_examples/shelves/validate/the-shape-is-wrong.yaml +46 -0
  212. dirigent_examples-0.15.0.dist-info/METADATA +21 -0
  213. dirigent_examples-0.15.0.dist-info/RECORD +216 -0
  214. dirigent_examples-0.15.0.dist-info/WHEEL +4 -0
  215. dirigent_examples-0.15.0.dist-info/entry_points.txt +3 -0
  216. dirigent_examples-0.15.0.dist-info/licenses/LICENSE +18 -0
@@ -0,0 +1,96 @@
1
+ # A step that fails on time rather than on work, and the three clocks that could have done it.
2
+ #
3
+ # The step below asks for thirty seconds of sleep against a two-second budget. It is designed
4
+ # to lose. The run fails in about two seconds, and the interesting part is which of the three
5
+ # available clocks stopped it, because they are not interchangeable:
6
+ #
7
+ # timeout (a step field) How long ONE block call may take before the engine abandons it.
8
+ # This is the one that fires here.
9
+ # deadline (a step field) How long a step may keep WAITING on a sensor or a remote job.
10
+ # A sensor's individual pokes are fast; what needs bounding is
11
+ # the waiting between them. sensor-gate.yaml is that example.
12
+ # timeout (block config) The block's own limit on its own work -- here, when
13
+ # shell.run kills the child process it started.
14
+ #
15
+ # The two on this step are set deliberately far apart. shell.run would let the process run for
16
+ # 300 seconds by default, and the engine gives up after two, so the engine is what fails the
17
+ # step. Widen the step's `timeout` past thirty and the sleep finishes and the step goes green;
18
+ # narrow the block's own `timeout` below two and the block kills its own child first and
19
+ # reports that instead. Same red step, different reason, and the error class in the attempt says which.
20
+ #
21
+ # A timeout is classified as a transient failure, which means a retry budget would spend
22
+ # itself on it: three attempts against a sleep that is always too long is six seconds of
23
+ # waiting and three identical failures. There is no retry here for exactly that reason -- a
24
+ # deadline that cannot be met is not a flake.
25
+ #
26
+ # on_timeout decides what a timeout means. The default is fail, which is what this shows;
27
+ # on_timeout: skip is the other answer, and it is what sensor-gate.yaml uses to let a run
28
+ # carry on past a drop that never landed.
29
+ #
30
+ # This one uses shell.run, which executes code on the worker and is refused unless the
31
+ # instance allowlists it: export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["shell.run"]', or pass
32
+ # --enable-unsafe shell.run to a local run.
33
+ #
34
+ # dg run --local examples/failure/step-timeout.yaml --enable-unsafe shell.run # fails, in ~2s
35
+ # dg run --local examples/failure/step-timeout.yaml --enable-unsafe shell.run -p seconds=1 # succeeds
36
+
37
+ format: dirigent/v1
38
+ kind: pipeline
39
+ code: step-timeout
40
+ name: Step timeout
41
+ description: |
42
+ A sleep that is asked to take thirty seconds, given two.
43
+
44
+ The run fails in about two seconds. It fails **on time, not on work**: the command was
45
+ healthy and would have finished, and the engine abandoned it anyway because the step
46
+ declared how long it was allowed to take.
47
+
48
+ Three clocks could stop a step, and they answer different questions:
49
+
50
+ | Field | Where it lives | What it bounds |
51
+ | --- | --- | --- |
52
+ | `timeout` | the step | one block call |
53
+ | `deadline` | the step | how long a sensor may keep waiting |
54
+ | `timeout` | the block's config | the block's own work |
55
+
56
+ tags: [failure, execute, retry]
57
+
58
+ requires:
59
+ blocks:
60
+ - shell.run
61
+
62
+ params:
63
+ type: object
64
+ properties:
65
+ seconds:
66
+ type: integer
67
+ description: How long the command sleeps. Anything above the two-second budget loses.
68
+ default: 30
69
+ minimum: 0
70
+ maximum: 300
71
+
72
+ steps:
73
+ start:
74
+ block: shell.run
75
+ config:
76
+ argv: [echo, "starting work that will not be allowed to finish"]
77
+
78
+ slow_work:
79
+ block: shell.run
80
+ depends_on: [start]
81
+ # The budget. The command below wants thirty seconds; it gets two, and the engine takes
82
+ # the attempt away from it.
83
+ timeout: 2s
84
+ config:
85
+ # 300 seconds is this block's own default, left as it is so the engine's clock is
86
+ # plainly the shorter one and plainly the one that fires.
87
+ timeout: 5m
88
+ command: "echo 'sleeping ${params.seconds}s'; sleep ${params.seconds}; echo 'never printed'"
89
+
90
+ # Skipped when the sleep is cut short, because the default all_success edge is not
91
+ # satisfied by a step the engine abandoned.
92
+ after:
93
+ block: shell.run
94
+ depends_on: [slow_work]
95
+ config:
96
+ argv: [echo, "only reached when the work fits in its budget"]
@@ -0,0 +1,32 @@
1
+ # Git examples
2
+
3
+ `git.checkout` puts an existing project into a run. It clones a repository at a ref into the
4
+ run's work directory and reports the commit it landed on, so a `docker.build` names its context,
5
+ a `docker.compose.up` names its compose file, and a transform names its files, all by a path
6
+ relative to the checkout. [docs/git.md](../../docs/git.md) is the family's home.
7
+
8
+ It is an **ordinary** block: it writes only under the run's work directory and reaches only the
9
+ remote its connection names, so no id has to be allowlisted to run it.
10
+
11
+ ```bash
12
+ dg run --local examples/git/git-checkout-public.yaml
13
+ ```
14
+
15
+ The other two documents drive the `docker.*` family off a checkout, so they need a Docker
16
+ daemon and the usual allowlist; each says so in its own header.
17
+
18
+ Every document here carries its `git` connection in a `connections:` section, because a
19
+ `--local` run has no instance to hold one. On a server the connection is created once and the
20
+ document names it:
21
+
22
+ ```bash
23
+ dg connection create git my-project --set url=https://github.com/owner/repo.git
24
+ ```
25
+
26
+ ## Pipelines
27
+
28
+ | File | What it teaches |
29
+ | --- | --- |
30
+ | [git-checkout-public.yaml](git-checkout-public.yaml) | The block on its own: a public remote, a shallow checkout at the default branch, and the commit, ref, target and remote a downstream step reads. |
31
+ | [git-checkout-build.yaml](git-checkout-build.yaml) | Starting an existing project: the repository's own Dockerfile built from the checkout, and the image run once to prove it starts. |
32
+ | [git-checkout-compose.yaml](git-checkout-compose.yaml) | A stack from somebody else's compose file: the checkout is what puts it where the compose CLI can open it, and the docker family's up/drive/down lifecycle is unchanged. |
@@ -0,0 +1,125 @@
1
+ # Starting an existing project: check the repository out, build its Dockerfile, run the image.
2
+ #
3
+ # This is the answer to "I have a project on GitHub, how do I get it into a pipeline". Nothing
4
+ # is copied by hand and no Dockerfile is written into the document: the repository's own build
5
+ # is what runs.
6
+ #
7
+ # REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
8
+ # tcp+TLS, or the local socket in dev), and network from the worker to github.com.
9
+ # docker.build and docker.run declare local_execution, so the engine refuses them unless the
10
+ # instance allowlists their ids -- git.checkout is ordinary and needs no allowlisting:
11
+ # dg run --local examples/git/git-checkout-build.yaml --enable-unsafe docker.run,docker.build
12
+ # export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.run","docker.build"]'
13
+ #
14
+ # Three hops, and what each one hands on:
15
+ # source the repository, shallow, into source/ in the run's work directory. Output: the
16
+ # commit, and the path the build step names next.
17
+ # build buildx builds that directory with the repository's own Dockerfile and loads the
18
+ # image into the worker's daemon store. Output: the image id, its size and its tags.
19
+ # smoke the built image, run once, proving the project starts.
20
+ #
21
+ # THE CHECKOUT IS WHAT MAKES THE BUILD POSSIBLE. docker.build takes a context DIRECTORY on the
22
+ # worker's filesystem and has no inline form, so before this block existed a pipeline had to
23
+ # put every file there itself, one storage.copy at a time. A checkout puts a whole repository
24
+ # there in one step, and the build names it by the same work-directory-relative path the
25
+ # checkout reported.
26
+ #
27
+ # THE TAG IS THE CONTRACT. Nothing is pushed anywhere, so the image exists only in the store of
28
+ # the daemon that built it, under the tag the build gave it. Both steps read that name from one
29
+ # parameter, and both must land on the same docker-capable worker: an image built on worker A
30
+ # is invisible to worker B, which is what requires.workers below routes around.
31
+ #
32
+ # WHY THE RUN STEP OVERRIDES THE COMMAND. The image's own CMD starts an HTTP server and never
33
+ # returns, and a step has to end. So the smoke step starts the project's server, asks it for a
34
+ # page, and exits with whether it answered -- which is what "it builds and it runs" means for
35
+ # a service. Running a stack that stays up for the rest of a run is the compose example's job.
36
+ #
37
+ # TO MAKE IT YOURS: point the connection at your own repository, pin a ref: to a tag or a
38
+ # commit rather than tracking a branch, and replace the smoke step with your project's own
39
+ # check -- its test suite, its migration, its one-shot job.
40
+
41
+ format: dirigent/v1
42
+ kind: pipeline
43
+ code: git-checkout-build
44
+ name: Build a checked-out project
45
+ description: Check a public repository out, build the Dockerfile it ships, and run the image once.
46
+
47
+ tags: [git, execute]
48
+
49
+ requires:
50
+ blocks:
51
+ - git.checkout
52
+ - docker.build
53
+ - docker.run
54
+ workers:
55
+ - docker
56
+
57
+ connections:
58
+ helloworld-demo-node:
59
+ kind: git
60
+ config:
61
+ # Docker's own sample project: a Node server with no dependencies and a Dockerfile at
62
+ # the repository root, which is the shape docker.build expects by default.
63
+ url: https://github.com/dockersamples/helloworld-demo-node.git
64
+
65
+ params:
66
+ type: object
67
+ additionalProperties: false
68
+ properties:
69
+ image_tag:
70
+ type: string
71
+ default: dirigent-helloworld:local
72
+ description: The tag the build gives the image, and the only name the smoke step has for it.
73
+
74
+ steps:
75
+ source:
76
+ block: git.checkout
77
+ deadline: 5m
78
+ config:
79
+ connection: helloworld-demo-node
80
+ # Named rather than defaulted, so the build step below can say "source" out loud instead
81
+ # of depending on what this step happens to be called.
82
+ target: source
83
+ depth: 1
84
+
85
+ build:
86
+ block: docker.build
87
+ depends_on: [source]
88
+ deadline: 15m
89
+ config:
90
+ # The checkout's directory, relative to the run's work directory. The Dockerfile path is
91
+ # relative to it, and
92
+ # "Dockerfile" is already the default; it is written out here because the whole point is
93
+ # that this is the repository's file and not one this pipeline invented.
94
+ context: source
95
+ dockerfile: Dockerfile
96
+ tags: ["${params.image_tag}"]
97
+ # The Dockerfile's npm ci runs inside the build, so the build needs the network the
98
+ # daemon has; nothing else about it is dirigent's business.
99
+ pull: true
100
+
101
+ smoke:
102
+ block: docker.run
103
+ depends_on: [build]
104
+ deadline: 5m
105
+ config:
106
+ image: "${params.image_tag}"
107
+ # pull stays off, and must: the image was never pushed, so a pull would look for this
108
+ # tag in a registry and fail on a name the daemon already holds locally.
109
+ pull: false
110
+ # none, not bridge: the server and the request that proves it are both in this
111
+ # container, so the loopback interface is the whole network this step needs.
112
+ network: none
113
+ # The shell form, because this is a small script rather than one command: start the
114
+ # project's server in the background, then ask it for a page until it answers. Ten
115
+ # tries at a second apart is generous for a process that binds a port and nothing else.
116
+ command: |
117
+ node app.js &
118
+ for _ in 1 2 3 4 5 6 7 8 9 10; do
119
+ wget -qO- http://127.0.0.1:8080/ && exit 0
120
+ sleep 1
121
+ done
122
+ echo "the server never answered" >&2
123
+ exit 1
124
+ memory: 256mb
125
+ pids_limit: 128
@@ -0,0 +1,138 @@
1
+ # A stack brought up from a compose file that lives in someone else's repository.
2
+ #
3
+ # The compose file is not in this document and was not written into the work directory by an
4
+ # earlier
5
+ # step: it is a file in a public repository, and the checkout is what puts it somewhere the
6
+ # compose CLI can open.
7
+ #
8
+ # REQUIRES A DOCKER DAEMON the worker can reach, named by DOCKER_HOST (a dind sidecar over
9
+ # tcp+TLS, or the local socket in dev), and network from the worker to github.com. The three
10
+ # docker blocks declare local_execution, so the engine refuses them unless the instance
11
+ # allowlists their ids -- git.checkout is ordinary and needs no allowlisting:
12
+ # dg run --local examples/git/git-checkout-compose.yaml \
13
+ # --enable-unsafe docker.compose.up,docker.compose.down,docker.run
14
+ # export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["docker.compose.up","docker.compose.down","docker.run"]'
15
+ #
16
+ # Four hops, and what each one hands on:
17
+ # source the repository, shallow, into source/ in the run's work directory. Output: the
18
+ # commit and the target the compose step names a file inside.
19
+ # stack compose brought up from that file, detached, and left running for the rest of
20
+ # the run. Output: the services, the networks, and default_network.
21
+ # probe a container on the stack's network, asking the service for a page.
22
+ # teardown the same project down again, volumes included, on any outcome.
23
+ #
24
+ # THE PATH IS WORK-DIRECTORY-RELATIVE ALL THE WAY THROUGH. docker.compose.up names a file
25
+ # inside the run's work directory, never an absolute path and never one that climbs out, and
26
+ # the checkout landed the whole repository at "source". So the compose file is
27
+ # "source/<directory>/compose.yaml" -- the repository's own layout, prefixed by where the
28
+ # checkout put it. The CLI is always run with --project-directory set to the work directory,
29
+ # so a compose file with relative paths of its own resolves them against the run's directory
30
+ # rather than its own: a stack of pre-built images, like this one, is what that suits.
31
+ #
32
+ # PIN THE REF FOR ANYTHING REAL. This one tracks the repository's default branch, which is
33
+ # honest for an example and wrong for a pipeline you rely on: somebody else's commit would
34
+ # change what your stack is. A ref: naming a tag or a full commit sha is how a checkout stops
35
+ # being a moving target, and the commit output is what to record when it is not.
36
+ #
37
+ # THE LIFECYCLE IS THE DOCKER FAMILY'S, UNCHANGED. up brings the stack up and it persists, a
38
+ # step drives it, and a down with rule: all_done tears it down whether the drive step passed
39
+ # or failed. Both compose steps default their project name from the run id, so the down
40
+ # addresses exactly the project the up created with nothing wired between them.
41
+ #
42
+ # TO MAKE IT YOURS: point the connection at the repository whose stack you actually run, name
43
+ # its compose file under compose_path, and replace the probe with something that proves your
44
+ # services are up -- a request that must return 200, a query that must return rows.
45
+
46
+ format: dirigent/v1
47
+ kind: pipeline
48
+ code: git-checkout-compose
49
+ name: A stack from a repository's compose file
50
+ description: Check a repository out, bring up the compose file it ships, probe it, and tear it down.
51
+
52
+ tags: [git, execute]
53
+
54
+ requires:
55
+ blocks:
56
+ - git.checkout
57
+ - docker.compose.up
58
+ - docker.compose.down
59
+ - docker.run
60
+
61
+ connections:
62
+ awesome-compose:
63
+ kind: git
64
+ config:
65
+ # Docker's own collection of sample stacks. It is public, so no credential is sealed
66
+ # here; a private repository would carry a token or a deploy key on this connection and
67
+ # the steps below would not change at all.
68
+ url: https://github.com/docker/awesome-compose.git
69
+
70
+ params:
71
+ type: object
72
+ additionalProperties: false
73
+ properties:
74
+ compose_path:
75
+ type: string
76
+ default: source/wordpress-mysql/compose.yaml
77
+ description: The compose file to bring up, as a path under the run's work directory.
78
+
79
+ steps:
80
+ source:
81
+ block: git.checkout
82
+ deadline: 10m
83
+ config:
84
+ connection: awesome-compose
85
+ target: source
86
+ # One commit and none of its history: this pipeline reads two files out of the tree and
87
+ # has no use for the years of log behind them.
88
+ depth: 1
89
+
90
+ stack:
91
+ block: docker.compose.up
92
+ depends_on: [source]
93
+ deadline: 15m
94
+ config:
95
+ # A file, not content: this is the form for a compose document something else produced,
96
+ # and a checkout is the most direct producer there is.
97
+ file: "${params.compose_path}"
98
+ # The stack's services declare no healthcheck, so --wait can only wait for their
99
+ # containers to be running; the probe below is what waits for the application itself.
100
+ wait: true
101
+ wait_timeout: 5m
102
+ pull: always
103
+
104
+ probe:
105
+ block: docker.run
106
+ depends_on: [stack]
107
+ deadline: 10m
108
+ config:
109
+ image: alpine:3
110
+ pull: true
111
+ # The network the up step reported: joining it resolves the service by the name the
112
+ # compose file gave it.
113
+ network: "${steps.stack.output.default_network}"
114
+ # The web service answers seconds after its container is running, and only once the
115
+ # database behind it does, so the probe retries rather than racing it. This is what a
116
+ # stack without a healthcheck costs the step that drives it.
117
+ command: |
118
+ for _ in $(seq 1 90); do
119
+ wget -q --spider http://wordpress/ && exit 0
120
+ sleep 2
121
+ done
122
+ echo "the service never answered" >&2
123
+ exit 1
124
+ memory: 64mb
125
+ pids_limit: 64
126
+
127
+ teardown:
128
+ block: docker.compose.down
129
+ depends_on: [probe]
130
+ # all_done runs the teardown on success or failure, so the stack never outlives the run.
131
+ rule: all_done
132
+ deadline: 10m
133
+ config:
134
+ # The same file the up step used, passed back from its output: a teardown resolves the
135
+ # project by label alone in most cases, and needs -f in the ones where it cannot.
136
+ file: "${steps.stack.output.compose_file}"
137
+ # The stack's named volumes go with it, so a run leaves nothing on the worker.
138
+ down_volumes: true
@@ -0,0 +1,84 @@
1
+ # A public repository checked out into the run's work directory, and the commit it landed on.
2
+ #
3
+ # NEEDS NETWORK from the worker to github.com, and nothing else: git.checkout is an ordinary
4
+ # block, so no id has to be allowlisted and no daemon has to be reachable.
5
+ # dg run --local examples/git/git-checkout-public.yaml
6
+ # docs/git.md is the family's home.
7
+ #
8
+ # Two hops, and what each one hands on:
9
+ # checkout the repository, shallow, at its default branch, at checkout/ in the run's
10
+ # work directory.
11
+ # Output: commit, ref, target and remote.
12
+ # report a constant step that reads those three fields, which is all it takes to prove
13
+ # they are references any downstream step can name.
14
+ #
15
+ # THE CONNECTION HOLDS THE REMOTE, THE DOCUMENT HOLDS THE REF. A checkout step never writes a
16
+ # URL: it names a git connection, and the connection is what carries the remote and, when the
17
+ # repository is private, the sealed token or deploy key that reaches it. Moving a pipeline
18
+ # from a fork to the real repository is then an edit to one connection and to no pipeline.
19
+ # This one is carried in the document because a --local run has no instance to hold it; on a
20
+ # server it would be created once, with `dg connection create git ...`, and the document would
21
+ # name it and stop there. Public repositories need no credential at all, which is why there is
22
+ # none here.
23
+ #
24
+ # WHERE IT LANDS. target is a path inside the run's work directory -- never absolute, never
25
+ # climbing out -- and left unset it is the step's own name, so this checkout is at
26
+ # checkout/. A checkout is a directory a tool opens rather than bytes in storage, which is
27
+ # why it lands there and not under ${run.scratch}. That relative path is the whole point of
28
+ # the block: a downstream docker.build names it as a context, a docker.compose.up names a
29
+ # file inside it, and a transform names its files, none of them knowing where the run's work
30
+ # directory actually is.
31
+ #
32
+ # SHALLOW BY DEFAULT. depth is 1 unless a document says otherwise, which fetches the one
33
+ # commit the ref points at and none of its history. That is what a build wants and it is a
34
+ # fraction of the bytes. Set depth: 0 for the whole history, which is what a step that reads
35
+ # the log or runs `git describe` needs.
36
+ #
37
+ # TO MAKE IT YOURS: point the connection at your own repository, add a ref: if the default
38
+ # branch is not what you want, and replace the report step with whatever actually reads the
39
+ # checkout.
40
+
41
+ format: dirigent/v1
42
+ kind: pipeline
43
+ code: git-checkout-public
44
+ name: Check a public repository out
45
+ description: Clone a public repository at its default branch into the run's work directory and report the commit.
46
+
47
+ tags: [git]
48
+
49
+ requires:
50
+ blocks:
51
+ - git.checkout
52
+ - value.const
53
+
54
+ connections:
55
+ octocat-hello-world:
56
+ kind: git
57
+ config:
58
+ # GitHub's own smallest repository: one README and two commits. Nothing is sealed here
59
+ # because nothing needs to be -- a public remote takes no credential.
60
+ url: https://github.com/octocat/Hello-World.git
61
+
62
+ steps:
63
+ checkout:
64
+ block: git.checkout
65
+ deadline: 5m
66
+ config:
67
+ connection: octocat-hello-world
68
+ # No ref, so the remote's own default branch is what lands, and the output reports which
69
+ # one that turned out to be.
70
+ depth: 1
71
+
72
+ report:
73
+ block: value.const
74
+ depends_on: [checkout]
75
+ config:
76
+ # The commit is the only exact name for what was checked out: a branch moves, a tag can
77
+ # be moved, and a sha cannot. A step that records what a pipeline built records this.
78
+ value:
79
+ commit: "${steps.checkout.output.commit}"
80
+ ref: "${steps.checkout.output.ref}"
81
+ # The remote as the output reports it, which is the URL with any credential stripped.
82
+ remote: "${steps.checkout.output.remote}"
83
+ # Relative to the run's work directory, which is the form every downstream block wants.
84
+ target: "${steps.checkout.output.target}"
@@ -0,0 +1,22 @@
1
+ # Graph examples
2
+
3
+ The shapes a DAG takes: a straight line, branches that split and join, a fan over a list,
4
+ and a chain deep enough to watch the engine walk it. Every one runs with `dg run --local`,
5
+ and the ones that call out use only Postman Echo.
6
+
7
+ ```bash
8
+ dg run --local examples/graph/linear.yaml -p day=2026-01-01 --enable-unsafe shell.run
9
+ ```
10
+
11
+ ## Pipelines
12
+
13
+ | File | What it teaches |
14
+ | --- | --- |
15
+ | [linear.yaml](linear.yaml) | The straight line: fetch, archive, report, each step reading the one before it. |
16
+ | [parallel-branches.yaml](parallel-branches.yaml) | Two branches from one root, joined by a step that reads both. |
17
+ | [fan-out.yaml](fan-out.yaml) | `for_each` over a list: one item per region, and a refusal that fails the item rather than the fan. |
18
+ | [fan-in.yaml](fan-in.yaml) | The other direction: parallel fetches whose whole fan one step reads back as a list. |
19
+ | [wide-fan.yaml](wide-fan.yaml) | A fan wide enough to fill every worker slot, for watching concurrency rather than reading about it. |
20
+ | [deep-chain.yaml](deep-chain.yaml) | Ten steps in one line, each reading the one above it. |
21
+ | [skip-diamond.yaml](skip-diamond.yaml) | A diamond where one road is taken and the other settles as skipped, not failed. |
22
+ | [parallel-sleep.yaml](parallel-sleep.yaml) | Three independent waits at once, and the join that waits for all of them. |
@@ -0,0 +1,119 @@
1
+ # Graph height: ten steps in a single file, where every one waits for the one before it.
2
+ #
3
+ # This document exists to be looked at rather than to compute anything. Height is the length
4
+ # of a DAG's longest path, and it is the one dimension no amount of hardware shortens: each
5
+ # step here reads the step above it, so the tenth cannot start until the ninth has settled.
6
+ # Ten workers finish this run no faster than one does.
7
+ #
8
+ # wide-fan.yaml is the same idea turned ninety degrees -- ten steps that are all claimable at
9
+ # once. Run both and watch the graph: this one draws a column, that one draws a row. Between
10
+ # them they are the two numbers worth knowing about a pipeline's shape, because height is
11
+ # what sets the floor on a run's duration and width is what sets the demand on the workers.
12
+ #
13
+ # The value threaded down the chain is a running total, so each step's output visibly depends
14
+ # on its predecessor's rather than merely coming after it in the drawing. transform.jq opens
15
+ # no socket and runs nothing on the worker, so the whole column needs no allowlist entry and
16
+ # no network.
17
+ #
18
+ # dg run --local examples/graph/deep-chain.yaml
19
+ # dg run --local examples/graph/deep-chain.yaml -p start=100
20
+
21
+ format: dirigent/v1
22
+ kind: pipeline
23
+ code: deep-chain
24
+ name: Deep chain
25
+ description: |
26
+ Ten steps in one line, each reading the one above it.
27
+
28
+ The longest path through a DAG is its **height**, and it is the floor under the run's
29
+ duration: no number of workers makes a chain shorter. Its opposite is
30
+ `wide-fan`, where all ten steps are claimable at once.
31
+
32
+ tags: [graph, transform]
33
+
34
+ requires:
35
+ blocks:
36
+ - transform.jq
37
+
38
+ params:
39
+ type: object
40
+ properties:
41
+ start:
42
+ type: integer
43
+ description: What the first link adds to.
44
+ default: 0
45
+
46
+ steps:
47
+ # The first link is the only one with no depends_on, so it is the only root: the run has
48
+ # exactly one claimable step at the moment it is created, and never more than one after.
49
+ link_1:
50
+ block: transform.jq
51
+ config:
52
+ input:
53
+ total: "${params.start}"
54
+ program: "{total: (.total + 1), depth: 1}"
55
+
56
+ link_2:
57
+ block: transform.jq
58
+ depends_on: [link_1]
59
+ config:
60
+ input: "${steps.link_1.output.value}"
61
+ program: "{total: (.total + 2), depth: 2}"
62
+
63
+ link_3:
64
+ block: transform.jq
65
+ depends_on: [link_2]
66
+ config:
67
+ input: "${steps.link_2.output.value}"
68
+ program: "{total: (.total + 3), depth: 3}"
69
+
70
+ link_4:
71
+ block: transform.jq
72
+ depends_on: [link_3]
73
+ config:
74
+ input: "${steps.link_3.output.value}"
75
+ program: "{total: (.total + 4), depth: 4}"
76
+
77
+ link_5:
78
+ block: transform.jq
79
+ depends_on: [link_4]
80
+ config:
81
+ input: "${steps.link_4.output.value}"
82
+ program: "{total: (.total + 5), depth: 5}"
83
+
84
+ link_6:
85
+ block: transform.jq
86
+ depends_on: [link_5]
87
+ config:
88
+ input: "${steps.link_5.output.value}"
89
+ program: "{total: (.total + 6), depth: 6}"
90
+
91
+ link_7:
92
+ block: transform.jq
93
+ depends_on: [link_6]
94
+ config:
95
+ input: "${steps.link_6.output.value}"
96
+ program: "{total: (.total + 7), depth: 7}"
97
+
98
+ link_8:
99
+ block: transform.jq
100
+ depends_on: [link_7]
101
+ config:
102
+ input: "${steps.link_7.output.value}"
103
+ program: "{total: (.total + 8), depth: 8}"
104
+
105
+ link_9:
106
+ block: transform.jq
107
+ depends_on: [link_8]
108
+ config:
109
+ input: "${steps.link_8.output.value}"
110
+ program: "{total: (.total + 9), depth: 9}"
111
+
112
+ # The last link reports the height it sits at alongside the total, so the run's output says
113
+ # what the drawing says.
114
+ link_10:
115
+ block: transform.jq
116
+ depends_on: [link_9]
117
+ config:
118
+ input: "${steps.link_9.output.value}"
119
+ program: "{total: (.total + 10), depth: 10, note: \"55 added over ten dependent steps\"}"