dirigent-examples 0.15.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dirigent_examples/__init__.py +22 -0
- dirigent_examples/py.typed +0 -0
- dirigent_examples/shelves/README.md +299 -0
- dirigent_examples/shelves/composition/README.md +18 -0
- dirigent_examples/shelves/composition/chained-instances.yaml +97 -0
- dirigent_examples/shelves/composition/composition-child.yaml +64 -0
- dirigent_examples/shelves/composition/composition-parent.yaml +89 -0
- dirigent_examples/shelves/connections.yaml +52 -0
- dirigent_examples/shelves/demo/README.md +19 -0
- dirigent_examples/shelves/demo/markdown-showcase.yaml +117 -0
- dirigent_examples/shelves/demo/params-showcase.yaml +92 -0
- dirigent_examples/shelves/demo/requires.yaml +65 -0
- dirigent_examples/shelves/demo/weekly-import-malawi.yaml +54 -0
- dirigent_examples/shelves/demo/weekly-import-nepal.yaml +75 -0
- dirigent_examples/shelves/docker/README.md +29 -0
- dirigent_examples/shelves/docker/docker-build-push.yaml +110 -0
- dirigent_examples/shelves/docker/docker-build-run.yaml +106 -0
- dirigent_examples/shelves/docker/docker-compose-database.yaml +124 -0
- dirigent_examples/shelves/docker/docker-compose-failing-up.yaml +83 -0
- dirigent_examples/shelves/docker/docker-compose-file.yaml +117 -0
- dirigent_examples/shelves/docker/docker-compose-profiles-env.yaml +133 -0
- dirigent_examples/shelves/docker/docker-compose-stack.yaml +70 -0
- dirigent_examples/shelves/docker/docker-hello.yaml +53 -0
- dirigent_examples/shelves/docker/docker-remote-daemon.yaml +92 -0
- dirigent_examples/shelves/docker/docker-run-failing-teardown.yaml +94 -0
- dirigent_examples/shelves/docker/docker-ticker.yaml +49 -0
- dirigent_examples/shelves/execute/README.md +16 -0
- dirigent_examples/shelves/execute/long-log.yaml +89 -0
- dirigent_examples/shelves/failure/README.md +20 -0
- dirigent_examples/shelves/failure/error-handler.yaml +89 -0
- dirigent_examples/shelves/failure/optional-step.yaml +82 -0
- dirigent_examples/shelves/failure/retries.yaml +75 -0
- dirigent_examples/shelves/failure/retry-budget.yaml +82 -0
- dirigent_examples/shelves/failure/step-timeout.yaml +96 -0
- dirigent_examples/shelves/git/README.md +32 -0
- dirigent_examples/shelves/git/git-checkout-build.yaml +125 -0
- dirigent_examples/shelves/git/git-checkout-compose.yaml +138 -0
- dirigent_examples/shelves/git/git-checkout-public.yaml +84 -0
- dirigent_examples/shelves/graph/README.md +22 -0
- dirigent_examples/shelves/graph/deep-chain.yaml +119 -0
- dirigent_examples/shelves/graph/fan-in.yaml +80 -0
- dirigent_examples/shelves/graph/fan-out.yaml +66 -0
- dirigent_examples/shelves/graph/linear.yaml +66 -0
- dirigent_examples/shelves/graph/parallel-branches.yaml +57 -0
- dirigent_examples/shelves/graph/parallel-sleep.yaml +55 -0
- dirigent_examples/shelves/graph/skip-diamond.yaml +105 -0
- dirigent_examples/shelves/graph/wide-fan.yaml +147 -0
- dirigent_examples/shelves/hello-world.yaml +30 -0
- dirigent_examples/shelves/open-data/README.md +67 -0
- dirigent_examples/shelves/open-data/feeds-composition.yaml +120 -0
- dirigent_examples/shelves/open-data/gdacs-disaster-updates.yaml +230 -0
- dirigent_examples/shelves/open-data/github-releases-relay.yaml +225 -0
- dirigent_examples/shelves/open-data/hdx-dataset-watch.yaml +211 -0
- dirigent_examples/shelves/open-data/kobo-submissions-to-csv.yaml +149 -0
- dirigent_examples/shelves/open-data/nominatim-geocode-facilities.yaml +169 -0
- dirigent_examples/shelves/open-data/odk-central-submissions.yaml +146 -0
- dirigent_examples/shelves/open-data/open-meteo-weekly-report.yaml +142 -0
- dirigent_examples/shelves/open-data/overpass-health-facilities.yaml +154 -0
- dirigent_examples/shelves/open-data/usgs-earthquakes-alert.yaml +208 -0
- dirigent_examples/shelves/open-data/who-gho-indicators-to-parquet.yaml +163 -0
- dirigent_examples/shelves/open-data/wikidata-country-reference.yaml +146 -0
- dirigent_examples/shelves/open-data/world-bank-population-trend.yaml +166 -0
- dirigent_examples/shelves/patterns/README.md +144 -0
- dirigent_examples/shelves/patterns/concurrency-queue.yaml +89 -0
- dirigent_examples/shelves/patterns/concurrency-replace.yaml +92 -0
- dirigent_examples/shelves/patterns/concurrency-skip.yaml +97 -0
- dirigent_examples/shelves/patterns/connections-referenced-vs-carried.yaml +140 -0
- dirigent_examples/shelves/patterns/deadline-on-a-sensor.yaml +106 -0
- dirigent_examples/shelves/patterns/fan-out-continue.yaml +88 -0
- dirigent_examples/shelves/patterns/fan-out-fail-fast.yaml +80 -0
- dirigent_examples/shelves/patterns/fan-out-from-params.yaml +84 -0
- dirigent_examples/shelves/patterns/fan-out-item-wise.yaml +111 -0
- dirigent_examples/shelves/patterns/fan-out-literal-list.yaml +81 -0
- dirigent_examples/shelves/patterns/fan-out-nested-objects.yaml +107 -0
- dirigent_examples/shelves/patterns/fan-out-then-join.yaml +86 -0
- dirigent_examples/shelves/patterns/log-levels.yaml +119 -0
- dirigent_examples/shelves/patterns/outputs-inline-vs-storage.yaml +140 -0
- dirigent_examples/shelves/patterns/params-every-type.yaml +259 -0
- dirigent_examples/shelves/patterns/params-validation-refuses.yaml +131 -0
- dirigent_examples/shelves/patterns/pipeline-run-child.yaml +96 -0
- dirigent_examples/shelves/patterns/pipeline-run-fire-and-forget.yaml +99 -0
- dirigent_examples/shelves/patterns/pipeline-run-strict.yaml +103 -0
- dirigent_examples/shelves/patterns/pipeline-run-wait.yaml +107 -0
- dirigent_examples/shelves/patterns/pipeline-run-with-params.yaml +124 -0
- dirigent_examples/shelves/patterns/poll-cadence.yaml +102 -0
- dirigent_examples/shelves/patterns/priority-layered.yaml +120 -0
- dirigent_examples/shelves/patterns/references-cheat-sheet.yaml +186 -0
- dirigent_examples/shelves/patterns/retry-budget-exhausted.yaml +92 -0
- dirigent_examples/shelves/patterns/retry-exponential-backoff.yaml +88 -0
- dirigent_examples/shelves/patterns/retry-only-transient.yaml +108 -0
- dirigent_examples/shelves/patterns/retry-with-jitter.yaml +102 -0
- dirigent_examples/shelves/patterns/rule-all-done.yaml +83 -0
- dirigent_examples/shelves/patterns/rule-all-success.yaml +79 -0
- dirigent_examples/shelves/patterns/rule-always.yaml +92 -0
- dirigent_examples/shelves/patterns/rule-one-failed.yaml +86 -0
- dirigent_examples/shelves/patterns/schedule-at-once.yaml +110 -0
- dirigent_examples/shelves/patterns/schedule-cron-timezone.yaml +121 -0
- dirigent_examples/shelves/patterns/schedule-interval.yaml +109 -0
- dirigent_examples/shelves/patterns/schedule-window-half-open.yaml +105 -0
- dirigent_examples/shelves/patterns/sensor-http-ready.yaml +121 -0
- dirigent_examples/shelves/patterns/sensor-storage-exists.yaml +134 -0
- dirigent_examples/shelves/patterns/step-names-and-keys.yaml +99 -0
- dirigent_examples/shelves/patterns/timeout-fails-the-step.yaml +94 -0
- dirigent_examples/shelves/patterns/timeout-skips-the-step.yaml +102 -0
- dirigent_examples/shelves/patterns/webhook-mapping-nested-payload.yaml +125 -0
- dirigent_examples/shelves/patterns/webhook-signed.yaml +144 -0
- dirigent_examples/shelves/preview/s3-parquet-to-ingestion.yaml +92 -0
- dirigent_examples/shelves/python/README.md +31 -0
- dirigent_examples/shelves/python/apply_and_run.py +52 -0
- dirigent_examples/shelves/python/ci_gate.py +76 -0
- dirigent_examples/shelves/python/connections.py +61 -0
- dirigent_examples/shelves/python/error_handling.py +84 -0
- dirigent_examples/shelves/python/follow_logs.py +39 -0
- dirigent_examples/shelves/python/list_and_filter.py +52 -0
- dirigent_examples/shelves/queues/README.md +59 -0
- dirigent_examples/shelves/queues/kafka-consume-then-transform.yaml +105 -0
- dirigent_examples/shelves/queues/kafka-produce-then-consume.yaml +124 -0
- dirigent_examples/shelves/queues/rabbitmq-consume-ack-on-success.yaml +102 -0
- dirigent_examples/shelves/queues/report-to-kafka.yaml +105 -0
- dirigent_examples/shelves/queues/report-to-rabbitmq.yaml +105 -0
- dirigent_examples/shelves/recipes/README.md +130 -0
- dirigent_examples/shelves/recipes/csv-header-rules.yaml +147 -0
- dirigent_examples/shelves/recipes/csv-to-ndjson.yaml +107 -0
- dirigent_examples/shelves/recipes/etl-csv-clean-validate-parquet.yaml +207 -0
- dirigent_examples/shelves/recipes/filter-by-predicate.yaml +116 -0
- dirigent_examples/shelves/recipes/filter-then-map-then-reduce.yaml +109 -0
- dirigent_examples/shelves/recipes/http-fetch-validate-post.yaml +142 -0
- dirigent_examples/shelves/recipes/http-follow-redirects.yaml +100 -0
- dirigent_examples/shelves/recipes/http-get-with-query.yaml +96 -0
- dirigent_examples/shelves/recipes/http-headers-and-auth-connection.yaml +110 -0
- dirigent_examples/shelves/recipes/http-post-file-from-storage.yaml +117 -0
- dirigent_examples/shelves/recipes/http-post-json-echo.yaml +105 -0
- dirigent_examples/shelves/recipes/http-post-report.yaml +183 -0
- dirigent_examples/shelves/recipes/http-save-body-to-storage.yaml +106 -0
- dirigent_examples/shelves/recipes/http-success-status-list.yaml +80 -0
- dirigent_examples/shelves/recipes/http-timeout-override.yaml +104 -0
- dirigent_examples/shelves/recipes/jq-dedupe-by-key.yaml +78 -0
- dirigent_examples/shelves/recipes/jq-defaults-and-nulls.yaml +91 -0
- dirigent_examples/shelves/recipes/jq-group-by-and-sum.yaml +74 -0
- dirigent_examples/shelves/recipes/jq-join-two-lists.yaml +77 -0
- dirigent_examples/shelves/recipes/jq-long-to-wide.yaml +76 -0
- dirigent_examples/shelves/recipes/jq-nested-to-flat.yaml +89 -0
- dirigent_examples/shelves/recipes/jq-pivot-wide-to-long.yaml +65 -0
- dirigent_examples/shelves/recipes/jq-running-totals.yaml +82 -0
- dirigent_examples/shelves/recipes/jq-string-cleaning.yaml +88 -0
- dirigent_examples/shelves/recipes/jq-top-n.yaml +85 -0
- dirigent_examples/shelves/recipes/jq-validate-in-jq-vs-schema.yaml +124 -0
- dirigent_examples/shelves/recipes/jq-window-dates.yaml +82 -0
- dirigent_examples/shelves/recipes/json-to-csv-flattening.yaml +141 -0
- dirigent_examples/shelves/recipes/large-output-to-storage.yaml +134 -0
- dirigent_examples/shelves/recipes/map-enrich-with-lookup.yaml +96 -0
- dirigent_examples/shelves/recipes/ndjson-to-parquet.yaml +130 -0
- dirigent_examples/shelves/recipes/pagination-by-fan-out.yaml +115 -0
- dirigent_examples/shelves/recipes/parquet-round-trip-types.yaml +163 -0
- dirigent_examples/shelves/recipes/reconcile-two-sources.yaml +159 -0
- dirigent_examples/shelves/recipes/report-built-in.yaml +72 -0
- dirigent_examples/shelves/recipes/report-daily-digest.yaml +186 -0
- dirigent_examples/shelves/recipes/report-to-file.yaml +131 -0
- dirigent_examples/shelves/recipes/report-to-webhook.yaml +136 -0
- dirigent_examples/shelves/recipes/schema-carried.yaml +112 -0
- dirigent_examples/shelves/recipes/schema-formats.yaml +107 -0
- dirigent_examples/shelves/recipes/schema-referenced.yaml +86 -0
- dirigent_examples/shelves/recipes/schema-refuses-then-rule.yaml +127 -0
- dirigent_examples/shelves/recipes/storage-copy-dated-archive.yaml +114 -0
- dirigent_examples/shelves/recipes/storage-exists-gate.yaml +127 -0
- dirigent_examples/shelves/recipes/storage-manifest-of-a-fan-out.yaml +104 -0
- dirigent_examples/shelves/recipes/storage-write-then-read.yaml +119 -0
- dirigent_examples/shelves/recipes/webhook-post-hmac.yaml +132 -0
- dirigent_examples/shelves/recipes/webhook-post-summary.yaml +142 -0
- dirigent_examples/shelves/s3/README.md +34 -0
- dirigent_examples/shelves/s3/report-to-s3.yaml +93 -0
- dirigent_examples/shelves/s3/s3-copy-and-verify.yaml +105 -0
- dirigent_examples/shelves/s3/s3-csv-report.yaml +87 -0
- dirigent_examples/shelves/s3/s3-parquet-report.yaml +77 -0
- dirigent_examples/shelves/s3/s3-round-trip.yaml +125 -0
- dirigent_examples/shelves/schemas/README.md +36 -0
- dirigent_examples/shelves/schemas/echo-reading.json +18 -0
- dirigent_examples/shelves/schemas/ou-record.json +13 -0
- dirigent_examples/shelves/schemas/station-reading.json +13 -0
- dirigent_examples/shelves/sensors/README.md +16 -0
- dirigent_examples/shelves/sensors/sensor-gate.yaml +65 -0
- dirigent_examples/shelves/sensors/time-window.yaml +61 -0
- dirigent_examples/shelves/sql/README.md +52 -0
- dirigent_examples/shelves/sql/duckdb-parquet-to-report.yaml +146 -0
- dirigent_examples/shelves/sql/sql-postgres-readonly.yaml +111 -0
- dirigent_examples/shelves/sql/sql-query-to-storage.yaml +85 -0
- dirigent_examples/shelves/sql/sql-sqlite-roundtrip.yaml +114 -0
- dirigent_examples/shelves/sql/warehouse.sql +42 -0
- dirigent_examples/shelves/transform/README.md +36 -0
- dirigent_examples/shelves/transform/csv-report.yaml +55 -0
- dirigent_examples/shelves/transform/jq-filter-and-map.yaml +70 -0
- dirigent_examples/shelves/transform/jq-group-and-aggregate.yaml +70 -0
- dirigent_examples/shelves/transform/jq-join-two-sources.yaml +98 -0
- dirigent_examples/shelves/transform/jq-reshape.yaml +91 -0
- dirigent_examples/shelves/transform/jq-stream-through-storage.yaml +112 -0
- dirigent_examples/shelves/transform/ndjson-round-trip.yaml +56 -0
- dirigent_examples/shelves/transform/parquet-round-trip.yaml +68 -0
- dirigent_examples/shelves/transform/std-convert-fan-out.yaml +142 -0
- dirigent_examples/shelves/transform/xml-feed-to-ndjson.yaml +116 -0
- dirigent_examples/shelves/transform/yaml-config-to-json.yaml +104 -0
- dirigent_examples/shelves/triggers/README.md +45 -0
- dirigent_examples/shelves/triggers/at-one-time.yaml +78 -0
- dirigent_examples/shelves/triggers/cron-nightly.yaml +79 -0
- dirigent_examples/shelves/triggers/cron-windowed.yaml +86 -0
- dirigent_examples/shelves/triggers/document-nightly.yaml +80 -0
- dirigent_examples/shelves/triggers/interval-rolling.yaml +88 -0
- dirigent_examples/shelves/triggers/managed-and-manual.yaml +109 -0
- dirigent_examples/shelves/triggers/webhook-trigger.yaml +75 -0
- dirigent_examples/shelves/validate/README.md +31 -0
- dirigent_examples/shelves/validate/expects-a-shape.yaml +56 -0
- dirigent_examples/shelves/validate/the-shape-is-wrong.yaml +46 -0
- dirigent_examples-0.15.0.dist-info/METADATA +21 -0
- dirigent_examples-0.15.0.dist-info/RECORD +216 -0
- dirigent_examples-0.15.0.dist-info/WHEEL +4 -0
- dirigent_examples-0.15.0.dist-info/entry_points.txt +3 -0
- dirigent_examples-0.15.0.dist-info/licenses/LICENSE +18 -0
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
# Every JSON Schema type a parameter can be, and the formats that actually assert.
|
|
2
|
+
#
|
|
3
|
+
# demo/params-showcase.yaml is about how parameters are PASSED -- flags, files, dotted keys,
|
|
4
|
+
# coercion. This one is about what they are DECLARED as: the six types, the keywords that
|
|
5
|
+
# constrain each of them, and the formats a dirigent instance genuinely checks rather than
|
|
6
|
+
# merely annotates.
|
|
7
|
+
#
|
|
8
|
+
# The schema is plain JSON Schema, draft 2020-12, and it is load-bearing in three places: it
|
|
9
|
+
# validates every run's parameters whoever started them, the UI renders the run dialog from
|
|
10
|
+
# it, and apply checks every ${params.x} in the document against it so a typo is caught at
|
|
11
|
+
# apply time rather than at 05:00.
|
|
12
|
+
#
|
|
13
|
+
# ABOUT format, AND THIS IS THE PART THAT SURPRISES PEOPLE. A format keyword asserts only
|
|
14
|
+
# where the validator is given a checker for it. Parameter validation is not given one, so
|
|
15
|
+
# every format below is an ANNOTATION: it documents intent and it is what the UI builds a date
|
|
16
|
+
# picker from, and -p day=2026-13-40 is accepted. Types and bounds and enums and patterns all
|
|
17
|
+
# assert; formats do not.
|
|
18
|
+
#
|
|
19
|
+
# Where they do assert is validate.schema, which is handed the instance's format checker --
|
|
20
|
+
# the standard formats from the library (date, date-time, time, duration, email, uri, uuid,
|
|
21
|
+
# ipv4, hostname, regex, json-pointer), the ones dirigent adds (ulid, uuid4, uuid7, md5, sha1,
|
|
22
|
+
# sha256, sha512, base64), and whatever a plugin pack contributes, which is how a pack's
|
|
23
|
+
# own format comes to be a format a pipeline can gate on. So a parameter that genuinely has to be a date
|
|
24
|
+
# gets a gate, and this file has one: a pattern is the other answer, and it asserts in the
|
|
25
|
+
# parameter schema itself.
|
|
26
|
+
#
|
|
27
|
+
# Every parameter below has a default, so the whole thing runs with no flags at all. That is
|
|
28
|
+
# a property worth keeping in a corpus: an example nobody can run without reading the header
|
|
29
|
+
# first is an example nobody runs.
|
|
30
|
+
#
|
|
31
|
+
# Hop by hop:
|
|
32
|
+
#
|
|
33
|
+
# echo one transform.jq step that pulls every parameter into one object, so the run's
|
|
34
|
+
# output is the resolved parameter set and the defaults are visible in it.
|
|
35
|
+
# gate validate.schema against the schema this document carries, which is where the
|
|
36
|
+
# formats actually bite. With the defaults it passes and hands the value straight on.
|
|
37
|
+
# shape proves the types survived: an integer is arithmetic, a boolean is a branch, and an
|
|
38
|
+
# array has a length. A parameter that arrived as text would fail here.
|
|
39
|
+
#
|
|
40
|
+
# EXPECT THIS RUN TO SUCCEED in about a second, with no network and no allowlist.
|
|
41
|
+
#
|
|
42
|
+
# To change it: -p day=2026-13-40 is ACCEPTED by the parameter schema and REFUSED by the gate,
|
|
43
|
+
# which is the whole point of having both. params-validation-refuses.yaml is the other half:
|
|
44
|
+
# what a request that fails the parameter schema itself is answered with.
|
|
45
|
+
#
|
|
46
|
+
# dg run --local examples/patterns/params-every-type.yaml
|
|
47
|
+
# dg run --local examples/patterns/params-every-type.yaml -p day=2026-06-01 -p attempts=5
|
|
48
|
+
# dg run --local examples/patterns/params-every-type.yaml -p day=2026-13-40 # fails at the gate
|
|
49
|
+
|
|
50
|
+
format: dirigent/v1
|
|
51
|
+
kind: pipeline
|
|
52
|
+
code: params-every-type
|
|
53
|
+
name: Every parameter type
|
|
54
|
+
description: |
|
|
55
|
+
The six JSON Schema types and the keywords that constrain each of them, with every
|
|
56
|
+
parameter carrying a default so the document runs bare.
|
|
57
|
+
|
|
58
|
+
A `format` in a parameter schema is an **annotation**: it documents intent and the UI
|
|
59
|
+
renders from it, and nothing checks it. Formats assert inside `validate.schema`, which is
|
|
60
|
+
handed the instance's checker -- so this document carries a schema and gates on it.
|
|
61
|
+
|
|
62
|
+
tags: [patterns, transform, validate, params]
|
|
63
|
+
|
|
64
|
+
# Carried in the document rather than named on an instance, so a --local run gates against
|
|
65
|
+
# exactly the same shape a server would. A server refuses a document that carries a schema
|
|
66
|
+
# this way for a pipeline it stores; a published example is where carrying one is right.
|
|
67
|
+
schemas:
|
|
68
|
+
run-inputs:
|
|
69
|
+
type: object
|
|
70
|
+
required: [day, correlation_id, upload_id, expected_digest]
|
|
71
|
+
properties:
|
|
72
|
+
# The same four format keywords as in params below, in the one place they assert.
|
|
73
|
+
day: {type: string, format: date}
|
|
74
|
+
correlation_id: {type: string, format: uuid4}
|
|
75
|
+
upload_id: {type: string, format: ulid}
|
|
76
|
+
expected_digest: {type: string, format: sha256}
|
|
77
|
+
|
|
78
|
+
requires:
|
|
79
|
+
blocks:
|
|
80
|
+
- transform.jq
|
|
81
|
+
- validate.schema
|
|
82
|
+
|
|
83
|
+
params:
|
|
84
|
+
type: object
|
|
85
|
+
# Nothing is required, because everything has a default. required is for the parameter a
|
|
86
|
+
# run has no sensible value for -- the day being backfilled, the tenant being loaded.
|
|
87
|
+
additionalProperties: false
|
|
88
|
+
properties:
|
|
89
|
+
# A string, constrained by length and by pattern. The pattern is anchored, because an
|
|
90
|
+
# unanchored one matches anywhere in the value and refuses almost nothing.
|
|
91
|
+
dataset:
|
|
92
|
+
type: string
|
|
93
|
+
description: Which dataset this run covers.
|
|
94
|
+
default: climate
|
|
95
|
+
minLength: 1
|
|
96
|
+
maxLength: 64
|
|
97
|
+
pattern: "^[a-z][a-z0-9-]*$"
|
|
98
|
+
|
|
99
|
+
# An enum, which is a closed set rather than a type. The UI renders it as a select.
|
|
100
|
+
environment:
|
|
101
|
+
type: string
|
|
102
|
+
description: Where this run writes.
|
|
103
|
+
default: staging
|
|
104
|
+
enum: [staging, production]
|
|
105
|
+
|
|
106
|
+
# An integer with bounds, so a fat-fingered 1000 is refused rather than retried a
|
|
107
|
+
# thousand times.
|
|
108
|
+
attempts:
|
|
109
|
+
type: integer
|
|
110
|
+
description: How many times a downstream loader should try.
|
|
111
|
+
default: 3
|
|
112
|
+
minimum: 1
|
|
113
|
+
maximum: 10
|
|
114
|
+
|
|
115
|
+
# A number, which is any JSON number rather than a whole one. exclusiveMinimum is the
|
|
116
|
+
# keyword for a value that may approach zero and never reach it.
|
|
117
|
+
threshold:
|
|
118
|
+
type: number
|
|
119
|
+
description: The confidence a prediction has to clear.
|
|
120
|
+
default: 0.75
|
|
121
|
+
exclusiveMinimum: 0.0
|
|
122
|
+
maximum: 1.0
|
|
123
|
+
|
|
124
|
+
# A boolean, which the UI renders as a switch. Named for what true means, so nobody has
|
|
125
|
+
# to work out what a false "no_dry_run" would do.
|
|
126
|
+
dry_run:
|
|
127
|
+
type: boolean
|
|
128
|
+
description: When true nothing downstream writes.
|
|
129
|
+
default: true
|
|
130
|
+
|
|
131
|
+
# An array. items constrains every element, and the two bounds are what stop a fan-out
|
|
132
|
+
# over this from being either empty or enormous.
|
|
133
|
+
regions:
|
|
134
|
+
type: array
|
|
135
|
+
description: The regions this run covers.
|
|
136
|
+
default: [east, west]
|
|
137
|
+
minItems: 1
|
|
138
|
+
maxItems: 16
|
|
139
|
+
uniqueItems: true
|
|
140
|
+
items:
|
|
141
|
+
type: string
|
|
142
|
+
minLength: 1
|
|
143
|
+
|
|
144
|
+
# An object, with its own properties and its own closed door. A nested leaf is addressed
|
|
145
|
+
# from the CLI with a dotted key: -p window.days=30.
|
|
146
|
+
window:
|
|
147
|
+
type: object
|
|
148
|
+
description: The rolling window a load covers.
|
|
149
|
+
default: {days: 7, align: midnight}
|
|
150
|
+
additionalProperties: false
|
|
151
|
+
required: [days]
|
|
152
|
+
properties:
|
|
153
|
+
days:
|
|
154
|
+
type: integer
|
|
155
|
+
minimum: 1
|
|
156
|
+
maximum: 365
|
|
157
|
+
align:
|
|
158
|
+
type: string
|
|
159
|
+
enum: [midnight, hour]
|
|
160
|
+
|
|
161
|
+
# Formats, one line each. None of these asserts here; the gate below is what asserts.
|
|
162
|
+
day:
|
|
163
|
+
type: string
|
|
164
|
+
description: A calendar date, which the UI renders as a date picker.
|
|
165
|
+
default: "2026-01-01"
|
|
166
|
+
format: date
|
|
167
|
+
|
|
168
|
+
since:
|
|
169
|
+
type: string
|
|
170
|
+
description: An instant, offset included.
|
|
171
|
+
default: "2026-01-01T00:00:00+00:00"
|
|
172
|
+
format: date-time
|
|
173
|
+
|
|
174
|
+
cutoff:
|
|
175
|
+
type: string
|
|
176
|
+
description: A local time of day, with no date and no zone.
|
|
177
|
+
default: "23:30:00"
|
|
178
|
+
format: time
|
|
179
|
+
|
|
180
|
+
catalog_url:
|
|
181
|
+
type: string
|
|
182
|
+
description: Where the catalog is read from.
|
|
183
|
+
default: https://postman-echo.com/get
|
|
184
|
+
format: uri
|
|
185
|
+
|
|
186
|
+
owner_email:
|
|
187
|
+
type: string
|
|
188
|
+
description: Who to write to about this run.
|
|
189
|
+
default: data-team@example.org
|
|
190
|
+
format: email
|
|
191
|
+
|
|
192
|
+
correlation_id:
|
|
193
|
+
type: string
|
|
194
|
+
description: A UUID version 4 specifically; the plain uuid format would take any version.
|
|
195
|
+
default: 6f0b6e34-6c0e-4c2f-9a1e-2f43a1a0d0b7
|
|
196
|
+
format: uuid4
|
|
197
|
+
|
|
198
|
+
upload_id:
|
|
199
|
+
type: string
|
|
200
|
+
description: A Crockford base32 ULID, one of the formats dirigent contributes.
|
|
201
|
+
default: 01J8Z3M4N5P6Q7R8S9TVWXYZ01
|
|
202
|
+
format: ulid
|
|
203
|
+
|
|
204
|
+
expected_digest:
|
|
205
|
+
type: string
|
|
206
|
+
description: A sha256 hex digest, checked for length and alphabet rather than parsed.
|
|
207
|
+
default: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
|
208
|
+
format: sha256
|
|
209
|
+
|
|
210
|
+
steps:
|
|
211
|
+
echo:
|
|
212
|
+
block: transform.jq
|
|
213
|
+
config:
|
|
214
|
+
input:
|
|
215
|
+
dataset: "${params.dataset}"
|
|
216
|
+
environment: "${params.environment}"
|
|
217
|
+
attempts: "${params.attempts}"
|
|
218
|
+
threshold: "${params.threshold}"
|
|
219
|
+
dry_run: "${params.dry_run}"
|
|
220
|
+
regions: "${params.regions}"
|
|
221
|
+
# A whole reference to an object resolves to the object, so the nested shape survives
|
|
222
|
+
# intact rather than being flattened into text.
|
|
223
|
+
window: "${params.window}"
|
|
224
|
+
day: "${params.day}"
|
|
225
|
+
since: "${params.since}"
|
|
226
|
+
cutoff: "${params.cutoff}"
|
|
227
|
+
catalog_url: "${params.catalog_url}"
|
|
228
|
+
owner_email: "${params.owner_email}"
|
|
229
|
+
correlation_id: "${params.correlation_id}"
|
|
230
|
+
upload_id: "${params.upload_id}"
|
|
231
|
+
expected_digest: "${params.expected_digest}"
|
|
232
|
+
program: |
|
|
233
|
+
.
|
|
234
|
+
|
|
235
|
+
gate:
|
|
236
|
+
block: validate.schema
|
|
237
|
+
depends_on: [echo]
|
|
238
|
+
# Where the formats bite. The gate is also a waypoint: everything below reads the gate's
|
|
239
|
+
# output rather than the echo's, so the document itself shows that nothing past this point
|
|
240
|
+
# saw an unchecked value.
|
|
241
|
+
config:
|
|
242
|
+
input: "${steps.echo.output.value}"
|
|
243
|
+
schema: run-inputs
|
|
244
|
+
|
|
245
|
+
shape:
|
|
246
|
+
block: transform.jq
|
|
247
|
+
depends_on: [gate]
|
|
248
|
+
# The proof that types survived the trip. Arithmetic on a string, or a branch on the text
|
|
249
|
+
# "false", would fail here rather than quietly doing the wrong thing.
|
|
250
|
+
config:
|
|
251
|
+
input: "${steps.gate.output.value}"
|
|
252
|
+
program: |
|
|
253
|
+
{
|
|
254
|
+
budget: (.attempts * .window.days),
|
|
255
|
+
confident: (.threshold > 0.5),
|
|
256
|
+
writes: (if .dry_run then "nothing" else .environment end),
|
|
257
|
+
regions: (.regions | length),
|
|
258
|
+
nested_leaf: .window.align
|
|
259
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# What a bad run request is answered with, and how early the answer comes.
|
|
2
|
+
#
|
|
3
|
+
# Parameter validation is not a step. It happens when the RUN IS CREATED: defaults are filled
|
|
4
|
+
# in, the schema is checked against the resolved object, and a request that does not satisfy
|
|
5
|
+
# it is refused before a run row exists. So a bad request produces no run at all -- nothing to
|
|
6
|
+
# find in dg runs list, nothing half-executed, no side effect anywhere. The error names the
|
|
7
|
+
# location and says what was wrong with it:
|
|
8
|
+
#
|
|
9
|
+
# parameter attempts is invalid: 99 is greater than the maximum of 10
|
|
10
|
+
# parameter tenant is invalid: '' should be non-empty
|
|
11
|
+
# parameter tenant is invalid: 'Nordic' does not match '^[a-z][a-z0-9-]*$'
|
|
12
|
+
#
|
|
13
|
+
# Each guard is worth having on purpose:
|
|
14
|
+
#
|
|
15
|
+
# minimum/maximum catches the fat-fingered order of magnitude.
|
|
16
|
+
# minLength/pattern catches the value of the right type and the wrong shape.
|
|
17
|
+
# required catches the parameter nobody can guess a default for.
|
|
18
|
+
# additionalProperties: false catches the TYPO, which is the one people forget. Without it,
|
|
19
|
+
# regoins=[east] is accepted in silence, regions keeps its default, and
|
|
20
|
+
# the run does the wrong work with a clean bill of health. The CLI
|
|
21
|
+
# catches an unknown -p by name before it sends anything, naming what
|
|
22
|
+
# the pipeline does declare; this keyword is what catches the same typo
|
|
23
|
+
# arriving from the API, from a schedule's pinned params, or from a
|
|
24
|
+
# webhook's mapped payload.
|
|
25
|
+
#
|
|
26
|
+
# WHAT IS NOT CHECKED HERE: format. A format keyword asserts only where the validator is given
|
|
27
|
+
# a checker for it, and parameter validation is not given one -- so day below is documentation
|
|
28
|
+
# and the UI's date picker, and -p day=2026-13-40 starts a run. A value that genuinely has to
|
|
29
|
+
# be a date is gated with validate.schema, which IS handed the instance's format checker;
|
|
30
|
+
# params-every-type.yaml carries that gate. A pattern is the other answer, and a pattern does
|
|
31
|
+
# assert here.
|
|
32
|
+
#
|
|
33
|
+
# Defaults are filled in BEFORE validation, so a parameter with a default is optional and one
|
|
34
|
+
# without a default and inside required is mandatory. There is no third state.
|
|
35
|
+
#
|
|
36
|
+
# Hop by hop:
|
|
37
|
+
#
|
|
38
|
+
# plan builds the load plan out of the validated parameters. If it runs at all, every
|
|
39
|
+
# value it reads has already satisfied the schema -- which is why no step in this
|
|
40
|
+
# file checks anything.
|
|
41
|
+
# receipt posts the plan on, standing in for the work.
|
|
42
|
+
#
|
|
43
|
+
# EXPECT THIS RUN TO SUCCEED with the defaults, in about a second. The demonstration is the
|
|
44
|
+
# runs that never start:
|
|
45
|
+
#
|
|
46
|
+
# dg run --local examples/patterns/params-validation-refuses.yaml
|
|
47
|
+
# dg run --local examples/patterns/params-validation-refuses.yaml -p tenant=Nordic # refused, pattern
|
|
48
|
+
# dg run --local examples/patterns/params-validation-refuses.yaml -p attempts=99 # refused
|
|
49
|
+
# dg run --local examples/patterns/params-validation-refuses.yaml -p regoins='["east"]' # refused
|
|
50
|
+
# dg run --local examples/patterns/params-validation-refuses.yaml -p tenant= # refused, minLength
|
|
51
|
+
#
|
|
52
|
+
# The same schema refuses the same values whoever is asking: the CLI, the API, a schedule's
|
|
53
|
+
# pinned params, and a webhook's mapped payload all land in one validate call. That is what
|
|
54
|
+
# makes a webhook safe to expose -- a caller cannot reach a parameter the mapping does not
|
|
55
|
+
# name, and cannot get a value past the schema even for one it does.
|
|
56
|
+
|
|
57
|
+
format: dirigent/v1
|
|
58
|
+
kind: pipeline
|
|
59
|
+
code: params-validation-refuses
|
|
60
|
+
name: A run request that is refused
|
|
61
|
+
description: |
|
|
62
|
+
Parameters are validated when the run is **created**, so a bad request produces no run at
|
|
63
|
+
all: nothing to find in `dg runs list`, and no side effect anywhere.
|
|
64
|
+
|
|
65
|
+
`format`, bounds, `required` and `additionalProperties: false` are four different guards.
|
|
66
|
+
The last one catches the typo, which is the one people forget.
|
|
67
|
+
|
|
68
|
+
tags: [patterns, http, transform, params]
|
|
69
|
+
|
|
70
|
+
requires:
|
|
71
|
+
blocks:
|
|
72
|
+
- transform.jq
|
|
73
|
+
- http.request
|
|
74
|
+
|
|
75
|
+
params:
|
|
76
|
+
type: object
|
|
77
|
+
# The parameter nobody can guess for you. Everything else has a default, so this is the
|
|
78
|
+
# only one a run must supply -- and there is a default here too, so the example runs bare.
|
|
79
|
+
required: [tenant]
|
|
80
|
+
# The typo guard. Turning this off is how a run does the wrong work and reports success.
|
|
81
|
+
additionalProperties: false
|
|
82
|
+
properties:
|
|
83
|
+
tenant:
|
|
84
|
+
type: string
|
|
85
|
+
description: Whose data this run loads.
|
|
86
|
+
default: nordic
|
|
87
|
+
minLength: 1
|
|
88
|
+
maxLength: 32
|
|
89
|
+
pattern: "^[a-z][a-z0-9-]*$"
|
|
90
|
+
day:
|
|
91
|
+
type: string
|
|
92
|
+
description: The day being loaded. The format here documents and renders; it does not check.
|
|
93
|
+
default: "2026-01-01"
|
|
94
|
+
format: date
|
|
95
|
+
attempts:
|
|
96
|
+
type: integer
|
|
97
|
+
description: How many times the loader tries; bounded so a slip cannot mean ninety-nine.
|
|
98
|
+
default: 3
|
|
99
|
+
minimum: 1
|
|
100
|
+
maximum: 10
|
|
101
|
+
regions:
|
|
102
|
+
type: array
|
|
103
|
+
description: The regions to load. Misspell the name and the run is refused, not defaulted.
|
|
104
|
+
default: [east, west]
|
|
105
|
+
minItems: 1
|
|
106
|
+
items:
|
|
107
|
+
type: string
|
|
108
|
+
|
|
109
|
+
steps:
|
|
110
|
+
plan:
|
|
111
|
+
block: transform.jq
|
|
112
|
+
# Nothing here re-checks anything. A step downstream of validation is entitled to assume
|
|
113
|
+
# the schema held, and a document that validates its own parameters again is a document
|
|
114
|
+
# whose schema was not doing its job.
|
|
115
|
+
config:
|
|
116
|
+
input:
|
|
117
|
+
tenant: "${params.tenant}"
|
|
118
|
+
day: "${params.day}"
|
|
119
|
+
attempts: "${params.attempts}"
|
|
120
|
+
regions: "${params.regions}"
|
|
121
|
+
program: |
|
|
122
|
+
{tenant, day, attempts, regions, work: ((.regions | length) * .attempts)}
|
|
123
|
+
|
|
124
|
+
receipt:
|
|
125
|
+
block: http.request
|
|
126
|
+
depends_on: [plan]
|
|
127
|
+
config:
|
|
128
|
+
url: https://postman-echo.com/post
|
|
129
|
+
method: POST
|
|
130
|
+
body:
|
|
131
|
+
plan: "${steps.plan.output.value}"
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# The pipeline the four composition examples call, and an ordinary pipeline in every way.
|
|
2
|
+
#
|
|
3
|
+
# Nothing here says it is a child. It has its own code, its own parameter schema, its own run
|
|
4
|
+
# history, and it can be run on its own, scheduled on its own, or called by four different
|
|
5
|
+
# parents. Being callable is not something a document opts into.
|
|
6
|
+
#
|
|
7
|
+
# ITS PARAMETER SCHEMA IS THE INTERFACE. A parent writes these parameters out explicitly
|
|
8
|
+
# rather than forwarding whatever it was run with, and they are validated against the schema
|
|
9
|
+
# below when the parent's step executes -- not when the parent is applied. So a parent that
|
|
10
|
+
# calls this with a region of 42 applies cleanly and fails at that step, which is why
|
|
11
|
+
# additionalProperties: false and the bounds below are worth having.
|
|
12
|
+
#
|
|
13
|
+
# The status parameter exists for one of the parents. pipeline-run-strict.yaml needs a child
|
|
14
|
+
# that settles completed_with_errors on purpose, and this is how: a tolerated failure, which
|
|
15
|
+
# is a failed step whose dependents carry on and whose run is not clean.
|
|
16
|
+
#
|
|
17
|
+
# Hop by hop:
|
|
18
|
+
#
|
|
19
|
+
# fetch a tolerated call. Asked for 200 it succeeds and the run is green; asked for 500
|
|
20
|
+
# it settles failed and is tolerated, so the run ends completed_with_errors with
|
|
21
|
+
# the branch below it still running.
|
|
22
|
+
# summarise reads the parameters back into the small record a parent reads as this pipeline's
|
|
23
|
+
# answer. It runs either way, because a tolerated failure reads as a success to
|
|
24
|
+
# whatever depends on it.
|
|
25
|
+
#
|
|
26
|
+
# EXPECT THIS RUN TO SUCCEED on its defaults, in about a second:
|
|
27
|
+
#
|
|
28
|
+
# dg run --local examples/patterns/pipeline-run-child.yaml -p region=east
|
|
29
|
+
# dg run --local examples/patterns/pipeline-run-child.yaml -p region=east -p status=500 # completed_with_errors
|
|
30
|
+
#
|
|
31
|
+
# Apply it before any of the parents; each of them names it in requires.pipelines, so a parent
|
|
32
|
+
# applied to an instance without it is refused up front with the code to apply first.
|
|
33
|
+
|
|
34
|
+
format: dirigent/v1
|
|
35
|
+
kind: pipeline
|
|
36
|
+
code: pipeline-run-child
|
|
37
|
+
name: The called pipeline
|
|
38
|
+
description: |
|
|
39
|
+
An ordinary pipeline that four composition examples call. Nothing about it declares that
|
|
40
|
+
it is a child; its parameter schema is the whole interface.
|
|
41
|
+
|
|
42
|
+
`-p status=500` makes it settle `completed_with_errors` on purpose, which is what
|
|
43
|
+
`pipeline-run-strict.yaml` needs something to do.
|
|
44
|
+
|
|
45
|
+
tags: [patterns, http, transform, composition]
|
|
46
|
+
|
|
47
|
+
requires:
|
|
48
|
+
blocks:
|
|
49
|
+
- http.request
|
|
50
|
+
- transform.jq
|
|
51
|
+
|
|
52
|
+
params:
|
|
53
|
+
type: object
|
|
54
|
+
required: [region]
|
|
55
|
+
# A closed door, so a parent that misspells a parameter fails at the step with the reason
|
|
56
|
+
# rather than running with a default nobody chose.
|
|
57
|
+
additionalProperties: false
|
|
58
|
+
properties:
|
|
59
|
+
region:
|
|
60
|
+
type: string
|
|
61
|
+
description: Which region this run loads.
|
|
62
|
+
minLength: 1
|
|
63
|
+
maxLength: 32
|
|
64
|
+
pattern: "^[a-z][a-z0-9-]*$"
|
|
65
|
+
day:
|
|
66
|
+
type: string
|
|
67
|
+
format: date
|
|
68
|
+
description: The day being loaded.
|
|
69
|
+
default: "2026-01-01"
|
|
70
|
+
status:
|
|
71
|
+
type: integer
|
|
72
|
+
description: What the tolerated call is answered with; 500 settles this run completed_with_errors.
|
|
73
|
+
default: 200
|
|
74
|
+
enum: [200, 500]
|
|
75
|
+
|
|
76
|
+
steps:
|
|
77
|
+
fetch:
|
|
78
|
+
block: http.request
|
|
79
|
+
# Tolerated: it settles failed, its dependents are shown a success, and the run reports
|
|
80
|
+
# completed_with_errors rather than failed. That third status is what strict is about.
|
|
81
|
+
continue_on_failure: true
|
|
82
|
+
config:
|
|
83
|
+
# A conditional written as data: the parameter picks the status, because the reference
|
|
84
|
+
# language has no branching construct and does not want one.
|
|
85
|
+
url: "https://postman-echo.com/status/${params.status}"
|
|
86
|
+
method: GET
|
|
87
|
+
|
|
88
|
+
summarise:
|
|
89
|
+
block: transform.jq
|
|
90
|
+
depends_on: [fetch]
|
|
91
|
+
config:
|
|
92
|
+
input:
|
|
93
|
+
region: "${params.region}"
|
|
94
|
+
day: "${params.day}"
|
|
95
|
+
program: |
|
|
96
|
+
{loaded: .region, day: .day}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# pipeline.run with wait: false -- start the child, hand back its run id, and move on.
|
|
2
|
+
#
|
|
3
|
+
# wait: false finishes the parent's step THE MOMENT THE CHILD RUN EXISTS. The step's output
|
|
4
|
+
# carries the child's code and its run id, and a status of "started" -- which is not a claim
|
|
5
|
+
# about how the child ended, because at that instant nothing knows.
|
|
6
|
+
#
|
|
7
|
+
# So this is a fork, not a call, and the consequences are worth being blunt about:
|
|
8
|
+
#
|
|
9
|
+
# * The child's outcome never reaches this run. A child that fails leaves the parent green.
|
|
10
|
+
# * Nothing downstream of the step may assume the child's work happened. An edge below a
|
|
11
|
+
# fire-and-forget step orders itself against the child's CREATION and nothing else.
|
|
12
|
+
# * The run id in the output is the whole handoff. It is what an operator pastes into
|
|
13
|
+
# dg runs show, and what a notification carries, so that the fork is findable later.
|
|
14
|
+
#
|
|
15
|
+
# Use it for the branch nobody is waiting on: a notification, a cache warm, a downstream job
|
|
16
|
+
# with its own alerting. Use wait: true whenever "and then" is part of the sentence.
|
|
17
|
+
#
|
|
18
|
+
# A third status is possible here and worth knowing about: skipped. If the child's own
|
|
19
|
+
# concurrency policy refuses the run -- concurrency: skip on the child, with one already
|
|
20
|
+
# going -- no child run is created, run_id is null, and the step reports skipped rather than
|
|
21
|
+
# failing. A parent that assumes a run id is always there will read a null.
|
|
22
|
+
#
|
|
23
|
+
# Hop by hop:
|
|
24
|
+
#
|
|
25
|
+
# waited the child, called the ordinary way, so this file has one of each to compare.
|
|
26
|
+
# forked the child again, not waited on. It finishes in milliseconds.
|
|
27
|
+
# handoff reads both outputs side by side: one status is the child's real outcome, the
|
|
28
|
+
# other is "started".
|
|
29
|
+
#
|
|
30
|
+
# EXPECT THIS RUN TO SUCCEED in about three seconds, with waited's status succeeded and
|
|
31
|
+
# forked's status started. In a --local run the throwaway instance goes away when the parent
|
|
32
|
+
# settles, so a forked child may not get to finish -- which is the honest shape of the thing:
|
|
33
|
+
# nobody is waiting for it.
|
|
34
|
+
#
|
|
35
|
+
# dg run --local examples/patterns/pipeline-run-fire-and-forget.yaml \
|
|
36
|
+
# --also-apply examples/patterns/pipeline-run-child.yaml
|
|
37
|
+
|
|
38
|
+
format: dirigent/v1
|
|
39
|
+
kind: pipeline
|
|
40
|
+
code: pipeline-run-fire-and-forget
|
|
41
|
+
name: A parent that does not wait
|
|
42
|
+
description: |
|
|
43
|
+
`wait: false` finishes the step as soon as the child run exists, reporting `started` and
|
|
44
|
+
the child's run id.
|
|
45
|
+
|
|
46
|
+
A fork, not a call: the child's outcome never reaches this run, and nothing downstream may
|
|
47
|
+
assume the child's work happened.
|
|
48
|
+
|
|
49
|
+
tags: [patterns, pipeline, transform, composition]
|
|
50
|
+
|
|
51
|
+
requires:
|
|
52
|
+
blocks:
|
|
53
|
+
- pipeline.run
|
|
54
|
+
- transform.jq
|
|
55
|
+
pipelines:
|
|
56
|
+
- pipeline-run-child
|
|
57
|
+
|
|
58
|
+
params:
|
|
59
|
+
type: object
|
|
60
|
+
properties:
|
|
61
|
+
day:
|
|
62
|
+
type: string
|
|
63
|
+
format: date
|
|
64
|
+
default: "2026-01-01"
|
|
65
|
+
|
|
66
|
+
steps:
|
|
67
|
+
waited:
|
|
68
|
+
block: pipeline.run
|
|
69
|
+
poll: 1s
|
|
70
|
+
config:
|
|
71
|
+
pipeline: pipeline-run-child
|
|
72
|
+
params:
|
|
73
|
+
region: east
|
|
74
|
+
day: "${params.day}"
|
|
75
|
+
|
|
76
|
+
forked:
|
|
77
|
+
block: pipeline.run
|
|
78
|
+
# No poll, because there is nothing to probe: the step is over as soon as the run row
|
|
79
|
+
# exists. A cadence here would be configuration that never fires.
|
|
80
|
+
config:
|
|
81
|
+
pipeline: pipeline-run-child
|
|
82
|
+
params:
|
|
83
|
+
region: west
|
|
84
|
+
day: "${params.day}"
|
|
85
|
+
# The one line this file is about.
|
|
86
|
+
wait: false
|
|
87
|
+
|
|
88
|
+
handoff:
|
|
89
|
+
block: transform.jq
|
|
90
|
+
depends_on: [waited, forked]
|
|
91
|
+
config:
|
|
92
|
+
input:
|
|
93
|
+
waited_status: "${steps.waited.output.status}"
|
|
94
|
+
forked_status: "${steps.forked.output.status}"
|
|
95
|
+
# The handoff. Without this in an output somewhere, a forked child is a run nobody
|
|
96
|
+
# can find on purpose.
|
|
97
|
+
forked_run: "${steps.forked.output.run_id}"
|
|
98
|
+
program: |
|
|
99
|
+
{waited_status, forked_status, forked_run, note: "started is not an outcome"}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# strict: what a child that ended completed_with_errors does to the step that called it.
|
|
2
|
+
#
|
|
3
|
+
# completed_with_errors is a third run status, not a shade of failed: it means every step that
|
|
4
|
+
# had to run ran, and at least one tolerated failure happened along the way. Whether that is
|
|
5
|
+
# acceptable is a judgement the CALLER makes, not the child, because the same child called
|
|
6
|
+
# from two places can be load-bearing in one and best-effort in the other. strict is where
|
|
7
|
+
# that judgement is written down.
|
|
8
|
+
#
|
|
9
|
+
# strict: false (the default) a child that ended completed_with_errors SUCCEEDS this step.
|
|
10
|
+
# strict: true it FAILS this step, exactly as a failed child would.
|
|
11
|
+
#
|
|
12
|
+
# A child that ended failed fails the step either way. strict only ever decides the middle
|
|
13
|
+
# case.
|
|
14
|
+
#
|
|
15
|
+
# Both calls below ask the child for the same deliberate tolerated failure, so the child ends
|
|
16
|
+
# completed_with_errors twice and the only difference is the flag.
|
|
17
|
+
#
|
|
18
|
+
# Hop by hop:
|
|
19
|
+
#
|
|
20
|
+
# lenient strict: false against a child that ends completed_with_errors. The step SUCCEEDS,
|
|
21
|
+
# and its output carries the child's real status -- so "succeeded" here is the
|
|
22
|
+
# step's outcome, not a claim about the child.
|
|
23
|
+
# strict strict: true against the same thing. The step FAILS.
|
|
24
|
+
# verdict all_done, so the run has an outcome to report whichever way the pair went. It
|
|
25
|
+
# reads only the lenient step's output: the strict one failed, so it has none.
|
|
26
|
+
#
|
|
27
|
+
# EXPECT THIS RUN TO FAIL, in about five seconds, with lenient succeeded, strict failed and
|
|
28
|
+
# verdict succeeded. The failure is the demonstration.
|
|
29
|
+
#
|
|
30
|
+
# To change it: -p status=200 makes the child end cleanly and both calls go green, which is
|
|
31
|
+
# the point -- strict costs nothing on a healthy child and is the difference between noticing
|
|
32
|
+
# and not noticing on an unhealthy one.
|
|
33
|
+
#
|
|
34
|
+
# dg run --local examples/patterns/pipeline-run-strict.yaml \
|
|
35
|
+
# --also-apply examples/patterns/pipeline-run-child.yaml # fails, by design
|
|
36
|
+
# dg run --local examples/patterns/pipeline-run-strict.yaml \
|
|
37
|
+
# --also-apply examples/patterns/pipeline-run-child.yaml -p status=200 # succeeds
|
|
38
|
+
|
|
39
|
+
format: dirigent/v1
|
|
40
|
+
kind: pipeline
|
|
41
|
+
code: pipeline-run-strict
|
|
42
|
+
name: Strict about a child's errors
|
|
43
|
+
description: |
|
|
44
|
+
`completed_with_errors` is a third status, and whether it is acceptable is the caller's
|
|
45
|
+
judgement: `strict: false` (the default) passes it, `strict: true` fails the step on it.
|
|
46
|
+
|
|
47
|
+
A child that ended `failed` fails the step either way. As written the strict call fails
|
|
48
|
+
and the run reports `failed`.
|
|
49
|
+
|
|
50
|
+
tags: [patterns, pipeline, transform, composition, failure]
|
|
51
|
+
|
|
52
|
+
requires:
|
|
53
|
+
blocks:
|
|
54
|
+
- pipeline.run
|
|
55
|
+
- transform.jq
|
|
56
|
+
pipelines:
|
|
57
|
+
- pipeline-run-child
|
|
58
|
+
|
|
59
|
+
params:
|
|
60
|
+
type: object
|
|
61
|
+
properties:
|
|
62
|
+
status:
|
|
63
|
+
type: integer
|
|
64
|
+
description: What the child's tolerated call is answered with; 500 makes it completed_with_errors.
|
|
65
|
+
default: 500
|
|
66
|
+
enum: [200, 500]
|
|
67
|
+
|
|
68
|
+
steps:
|
|
69
|
+
lenient:
|
|
70
|
+
block: pipeline.run
|
|
71
|
+
poll: 1s
|
|
72
|
+
config:
|
|
73
|
+
pipeline: pipeline-run-child
|
|
74
|
+
params:
|
|
75
|
+
region: east
|
|
76
|
+
status: "${params.status}"
|
|
77
|
+
# The default, written out so the pair reads as two positions rather than one setting.
|
|
78
|
+
strict: false
|
|
79
|
+
|
|
80
|
+
strict:
|
|
81
|
+
block: pipeline.run
|
|
82
|
+
poll: 1s
|
|
83
|
+
config:
|
|
84
|
+
pipeline: pipeline-run-child
|
|
85
|
+
params:
|
|
86
|
+
region: west
|
|
87
|
+
status: "${params.status}"
|
|
88
|
+
# The one line this file is about.
|
|
89
|
+
strict: true
|
|
90
|
+
|
|
91
|
+
verdict:
|
|
92
|
+
block: transform.jq
|
|
93
|
+
depends_on: [lenient, strict]
|
|
94
|
+
# all_done, because the interesting run is the one where the strict call failed, and
|
|
95
|
+
# something still has to say what the pair amounted to.
|
|
96
|
+
rule: all_done
|
|
97
|
+
config:
|
|
98
|
+
input:
|
|
99
|
+
# Only the lenient step is read. The strict one failed, so it has no stored output,
|
|
100
|
+
# and a reference to it would settle this attempt as rejected.
|
|
101
|
+
child_status: "${steps.lenient.output.status}"
|
|
102
|
+
program: |
|
|
103
|
+
{child_status, note: "the same child status, two different step outcomes"}
|