dirigent-examples 0.15.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dirigent_examples/__init__.py +22 -0
- dirigent_examples/py.typed +0 -0
- dirigent_examples/shelves/README.md +299 -0
- dirigent_examples/shelves/composition/README.md +18 -0
- dirigent_examples/shelves/composition/chained-instances.yaml +97 -0
- dirigent_examples/shelves/composition/composition-child.yaml +64 -0
- dirigent_examples/shelves/composition/composition-parent.yaml +89 -0
- dirigent_examples/shelves/connections.yaml +52 -0
- dirigent_examples/shelves/demo/README.md +19 -0
- dirigent_examples/shelves/demo/markdown-showcase.yaml +117 -0
- dirigent_examples/shelves/demo/params-showcase.yaml +92 -0
- dirigent_examples/shelves/demo/requires.yaml +65 -0
- dirigent_examples/shelves/demo/weekly-import-malawi.yaml +54 -0
- dirigent_examples/shelves/demo/weekly-import-nepal.yaml +75 -0
- dirigent_examples/shelves/docker/README.md +29 -0
- dirigent_examples/shelves/docker/docker-build-push.yaml +110 -0
- dirigent_examples/shelves/docker/docker-build-run.yaml +106 -0
- dirigent_examples/shelves/docker/docker-compose-database.yaml +124 -0
- dirigent_examples/shelves/docker/docker-compose-failing-up.yaml +83 -0
- dirigent_examples/shelves/docker/docker-compose-file.yaml +117 -0
- dirigent_examples/shelves/docker/docker-compose-profiles-env.yaml +133 -0
- dirigent_examples/shelves/docker/docker-compose-stack.yaml +70 -0
- dirigent_examples/shelves/docker/docker-hello.yaml +53 -0
- dirigent_examples/shelves/docker/docker-remote-daemon.yaml +92 -0
- dirigent_examples/shelves/docker/docker-run-failing-teardown.yaml +94 -0
- dirigent_examples/shelves/docker/docker-ticker.yaml +49 -0
- dirigent_examples/shelves/execute/README.md +16 -0
- dirigent_examples/shelves/execute/long-log.yaml +89 -0
- dirigent_examples/shelves/failure/README.md +20 -0
- dirigent_examples/shelves/failure/error-handler.yaml +89 -0
- dirigent_examples/shelves/failure/optional-step.yaml +82 -0
- dirigent_examples/shelves/failure/retries.yaml +75 -0
- dirigent_examples/shelves/failure/retry-budget.yaml +82 -0
- dirigent_examples/shelves/failure/step-timeout.yaml +96 -0
- dirigent_examples/shelves/git/README.md +32 -0
- dirigent_examples/shelves/git/git-checkout-build.yaml +125 -0
- dirigent_examples/shelves/git/git-checkout-compose.yaml +138 -0
- dirigent_examples/shelves/git/git-checkout-public.yaml +84 -0
- dirigent_examples/shelves/graph/README.md +22 -0
- dirigent_examples/shelves/graph/deep-chain.yaml +119 -0
- dirigent_examples/shelves/graph/fan-in.yaml +80 -0
- dirigent_examples/shelves/graph/fan-out.yaml +66 -0
- dirigent_examples/shelves/graph/linear.yaml +66 -0
- dirigent_examples/shelves/graph/parallel-branches.yaml +57 -0
- dirigent_examples/shelves/graph/parallel-sleep.yaml +55 -0
- dirigent_examples/shelves/graph/skip-diamond.yaml +105 -0
- dirigent_examples/shelves/graph/wide-fan.yaml +147 -0
- dirigent_examples/shelves/hello-world.yaml +30 -0
- dirigent_examples/shelves/open-data/README.md +67 -0
- dirigent_examples/shelves/open-data/feeds-composition.yaml +120 -0
- dirigent_examples/shelves/open-data/gdacs-disaster-updates.yaml +230 -0
- dirigent_examples/shelves/open-data/github-releases-relay.yaml +225 -0
- dirigent_examples/shelves/open-data/hdx-dataset-watch.yaml +211 -0
- dirigent_examples/shelves/open-data/kobo-submissions-to-csv.yaml +149 -0
- dirigent_examples/shelves/open-data/nominatim-geocode-facilities.yaml +169 -0
- dirigent_examples/shelves/open-data/odk-central-submissions.yaml +146 -0
- dirigent_examples/shelves/open-data/open-meteo-weekly-report.yaml +142 -0
- dirigent_examples/shelves/open-data/overpass-health-facilities.yaml +154 -0
- dirigent_examples/shelves/open-data/usgs-earthquakes-alert.yaml +208 -0
- dirigent_examples/shelves/open-data/who-gho-indicators-to-parquet.yaml +163 -0
- dirigent_examples/shelves/open-data/wikidata-country-reference.yaml +146 -0
- dirigent_examples/shelves/open-data/world-bank-population-trend.yaml +166 -0
- dirigent_examples/shelves/patterns/README.md +144 -0
- dirigent_examples/shelves/patterns/concurrency-queue.yaml +89 -0
- dirigent_examples/shelves/patterns/concurrency-replace.yaml +92 -0
- dirigent_examples/shelves/patterns/concurrency-skip.yaml +97 -0
- dirigent_examples/shelves/patterns/connections-referenced-vs-carried.yaml +140 -0
- dirigent_examples/shelves/patterns/deadline-on-a-sensor.yaml +106 -0
- dirigent_examples/shelves/patterns/fan-out-continue.yaml +88 -0
- dirigent_examples/shelves/patterns/fan-out-fail-fast.yaml +80 -0
- dirigent_examples/shelves/patterns/fan-out-from-params.yaml +84 -0
- dirigent_examples/shelves/patterns/fan-out-item-wise.yaml +111 -0
- dirigent_examples/shelves/patterns/fan-out-literal-list.yaml +81 -0
- dirigent_examples/shelves/patterns/fan-out-nested-objects.yaml +107 -0
- dirigent_examples/shelves/patterns/fan-out-then-join.yaml +86 -0
- dirigent_examples/shelves/patterns/log-levels.yaml +119 -0
- dirigent_examples/shelves/patterns/outputs-inline-vs-storage.yaml +140 -0
- dirigent_examples/shelves/patterns/params-every-type.yaml +259 -0
- dirigent_examples/shelves/patterns/params-validation-refuses.yaml +131 -0
- dirigent_examples/shelves/patterns/pipeline-run-child.yaml +96 -0
- dirigent_examples/shelves/patterns/pipeline-run-fire-and-forget.yaml +99 -0
- dirigent_examples/shelves/patterns/pipeline-run-strict.yaml +103 -0
- dirigent_examples/shelves/patterns/pipeline-run-wait.yaml +107 -0
- dirigent_examples/shelves/patterns/pipeline-run-with-params.yaml +124 -0
- dirigent_examples/shelves/patterns/poll-cadence.yaml +102 -0
- dirigent_examples/shelves/patterns/priority-layered.yaml +120 -0
- dirigent_examples/shelves/patterns/references-cheat-sheet.yaml +186 -0
- dirigent_examples/shelves/patterns/retry-budget-exhausted.yaml +92 -0
- dirigent_examples/shelves/patterns/retry-exponential-backoff.yaml +88 -0
- dirigent_examples/shelves/patterns/retry-only-transient.yaml +108 -0
- dirigent_examples/shelves/patterns/retry-with-jitter.yaml +102 -0
- dirigent_examples/shelves/patterns/rule-all-done.yaml +83 -0
- dirigent_examples/shelves/patterns/rule-all-success.yaml +79 -0
- dirigent_examples/shelves/patterns/rule-always.yaml +92 -0
- dirigent_examples/shelves/patterns/rule-one-failed.yaml +86 -0
- dirigent_examples/shelves/patterns/schedule-at-once.yaml +110 -0
- dirigent_examples/shelves/patterns/schedule-cron-timezone.yaml +121 -0
- dirigent_examples/shelves/patterns/schedule-interval.yaml +109 -0
- dirigent_examples/shelves/patterns/schedule-window-half-open.yaml +105 -0
- dirigent_examples/shelves/patterns/sensor-http-ready.yaml +121 -0
- dirigent_examples/shelves/patterns/sensor-storage-exists.yaml +134 -0
- dirigent_examples/shelves/patterns/step-names-and-keys.yaml +99 -0
- dirigent_examples/shelves/patterns/timeout-fails-the-step.yaml +94 -0
- dirigent_examples/shelves/patterns/timeout-skips-the-step.yaml +102 -0
- dirigent_examples/shelves/patterns/webhook-mapping-nested-payload.yaml +125 -0
- dirigent_examples/shelves/patterns/webhook-signed.yaml +144 -0
- dirigent_examples/shelves/preview/s3-parquet-to-ingestion.yaml +92 -0
- dirigent_examples/shelves/python/README.md +31 -0
- dirigent_examples/shelves/python/apply_and_run.py +52 -0
- dirigent_examples/shelves/python/ci_gate.py +76 -0
- dirigent_examples/shelves/python/connections.py +61 -0
- dirigent_examples/shelves/python/error_handling.py +84 -0
- dirigent_examples/shelves/python/follow_logs.py +39 -0
- dirigent_examples/shelves/python/list_and_filter.py +52 -0
- dirigent_examples/shelves/queues/README.md +59 -0
- dirigent_examples/shelves/queues/kafka-consume-then-transform.yaml +105 -0
- dirigent_examples/shelves/queues/kafka-produce-then-consume.yaml +124 -0
- dirigent_examples/shelves/queues/rabbitmq-consume-ack-on-success.yaml +102 -0
- dirigent_examples/shelves/queues/report-to-kafka.yaml +105 -0
- dirigent_examples/shelves/queues/report-to-rabbitmq.yaml +105 -0
- dirigent_examples/shelves/recipes/README.md +130 -0
- dirigent_examples/shelves/recipes/csv-header-rules.yaml +147 -0
- dirigent_examples/shelves/recipes/csv-to-ndjson.yaml +107 -0
- dirigent_examples/shelves/recipes/etl-csv-clean-validate-parquet.yaml +207 -0
- dirigent_examples/shelves/recipes/filter-by-predicate.yaml +116 -0
- dirigent_examples/shelves/recipes/filter-then-map-then-reduce.yaml +109 -0
- dirigent_examples/shelves/recipes/http-fetch-validate-post.yaml +142 -0
- dirigent_examples/shelves/recipes/http-follow-redirects.yaml +100 -0
- dirigent_examples/shelves/recipes/http-get-with-query.yaml +96 -0
- dirigent_examples/shelves/recipes/http-headers-and-auth-connection.yaml +110 -0
- dirigent_examples/shelves/recipes/http-post-file-from-storage.yaml +117 -0
- dirigent_examples/shelves/recipes/http-post-json-echo.yaml +105 -0
- dirigent_examples/shelves/recipes/http-post-report.yaml +183 -0
- dirigent_examples/shelves/recipes/http-save-body-to-storage.yaml +106 -0
- dirigent_examples/shelves/recipes/http-success-status-list.yaml +80 -0
- dirigent_examples/shelves/recipes/http-timeout-override.yaml +104 -0
- dirigent_examples/shelves/recipes/jq-dedupe-by-key.yaml +78 -0
- dirigent_examples/shelves/recipes/jq-defaults-and-nulls.yaml +91 -0
- dirigent_examples/shelves/recipes/jq-group-by-and-sum.yaml +74 -0
- dirigent_examples/shelves/recipes/jq-join-two-lists.yaml +77 -0
- dirigent_examples/shelves/recipes/jq-long-to-wide.yaml +76 -0
- dirigent_examples/shelves/recipes/jq-nested-to-flat.yaml +89 -0
- dirigent_examples/shelves/recipes/jq-pivot-wide-to-long.yaml +65 -0
- dirigent_examples/shelves/recipes/jq-running-totals.yaml +82 -0
- dirigent_examples/shelves/recipes/jq-string-cleaning.yaml +88 -0
- dirigent_examples/shelves/recipes/jq-top-n.yaml +85 -0
- dirigent_examples/shelves/recipes/jq-validate-in-jq-vs-schema.yaml +124 -0
- dirigent_examples/shelves/recipes/jq-window-dates.yaml +82 -0
- dirigent_examples/shelves/recipes/json-to-csv-flattening.yaml +141 -0
- dirigent_examples/shelves/recipes/large-output-to-storage.yaml +134 -0
- dirigent_examples/shelves/recipes/map-enrich-with-lookup.yaml +96 -0
- dirigent_examples/shelves/recipes/ndjson-to-parquet.yaml +130 -0
- dirigent_examples/shelves/recipes/pagination-by-fan-out.yaml +115 -0
- dirigent_examples/shelves/recipes/parquet-round-trip-types.yaml +163 -0
- dirigent_examples/shelves/recipes/reconcile-two-sources.yaml +159 -0
- dirigent_examples/shelves/recipes/report-built-in.yaml +72 -0
- dirigent_examples/shelves/recipes/report-daily-digest.yaml +186 -0
- dirigent_examples/shelves/recipes/report-to-file.yaml +131 -0
- dirigent_examples/shelves/recipes/report-to-webhook.yaml +136 -0
- dirigent_examples/shelves/recipes/schema-carried.yaml +112 -0
- dirigent_examples/shelves/recipes/schema-formats.yaml +107 -0
- dirigent_examples/shelves/recipes/schema-referenced.yaml +86 -0
- dirigent_examples/shelves/recipes/schema-refuses-then-rule.yaml +127 -0
- dirigent_examples/shelves/recipes/storage-copy-dated-archive.yaml +114 -0
- dirigent_examples/shelves/recipes/storage-exists-gate.yaml +127 -0
- dirigent_examples/shelves/recipes/storage-manifest-of-a-fan-out.yaml +104 -0
- dirigent_examples/shelves/recipes/storage-write-then-read.yaml +119 -0
- dirigent_examples/shelves/recipes/webhook-post-hmac.yaml +132 -0
- dirigent_examples/shelves/recipes/webhook-post-summary.yaml +142 -0
- dirigent_examples/shelves/s3/README.md +34 -0
- dirigent_examples/shelves/s3/report-to-s3.yaml +93 -0
- dirigent_examples/shelves/s3/s3-copy-and-verify.yaml +105 -0
- dirigent_examples/shelves/s3/s3-csv-report.yaml +87 -0
- dirigent_examples/shelves/s3/s3-parquet-report.yaml +77 -0
- dirigent_examples/shelves/s3/s3-round-trip.yaml +125 -0
- dirigent_examples/shelves/schemas/README.md +36 -0
- dirigent_examples/shelves/schemas/echo-reading.json +18 -0
- dirigent_examples/shelves/schemas/ou-record.json +13 -0
- dirigent_examples/shelves/schemas/station-reading.json +13 -0
- dirigent_examples/shelves/sensors/README.md +16 -0
- dirigent_examples/shelves/sensors/sensor-gate.yaml +65 -0
- dirigent_examples/shelves/sensors/time-window.yaml +61 -0
- dirigent_examples/shelves/sql/README.md +52 -0
- dirigent_examples/shelves/sql/duckdb-parquet-to-report.yaml +146 -0
- dirigent_examples/shelves/sql/sql-postgres-readonly.yaml +111 -0
- dirigent_examples/shelves/sql/sql-query-to-storage.yaml +85 -0
- dirigent_examples/shelves/sql/sql-sqlite-roundtrip.yaml +114 -0
- dirigent_examples/shelves/sql/warehouse.sql +42 -0
- dirigent_examples/shelves/transform/README.md +36 -0
- dirigent_examples/shelves/transform/csv-report.yaml +55 -0
- dirigent_examples/shelves/transform/jq-filter-and-map.yaml +70 -0
- dirigent_examples/shelves/transform/jq-group-and-aggregate.yaml +70 -0
- dirigent_examples/shelves/transform/jq-join-two-sources.yaml +98 -0
- dirigent_examples/shelves/transform/jq-reshape.yaml +91 -0
- dirigent_examples/shelves/transform/jq-stream-through-storage.yaml +112 -0
- dirigent_examples/shelves/transform/ndjson-round-trip.yaml +56 -0
- dirigent_examples/shelves/transform/parquet-round-trip.yaml +68 -0
- dirigent_examples/shelves/transform/std-convert-fan-out.yaml +142 -0
- dirigent_examples/shelves/transform/xml-feed-to-ndjson.yaml +116 -0
- dirigent_examples/shelves/transform/yaml-config-to-json.yaml +104 -0
- dirigent_examples/shelves/triggers/README.md +45 -0
- dirigent_examples/shelves/triggers/at-one-time.yaml +78 -0
- dirigent_examples/shelves/triggers/cron-nightly.yaml +79 -0
- dirigent_examples/shelves/triggers/cron-windowed.yaml +86 -0
- dirigent_examples/shelves/triggers/document-nightly.yaml +80 -0
- dirigent_examples/shelves/triggers/interval-rolling.yaml +88 -0
- dirigent_examples/shelves/triggers/managed-and-manual.yaml +109 -0
- dirigent_examples/shelves/triggers/webhook-trigger.yaml +75 -0
- dirigent_examples/shelves/validate/README.md +31 -0
- dirigent_examples/shelves/validate/expects-a-shape.yaml +56 -0
- dirigent_examples/shelves/validate/the-shape-is-wrong.yaml +46 -0
- dirigent_examples-0.15.0.dist-info/METADATA +21 -0
- dirigent_examples-0.15.0.dist-info/RECORD +216 -0
- dirigent_examples-0.15.0.dist-info/WHEEL +4 -0
- dirigent_examples-0.15.0.dist-info/entry_points.txt +3 -0
- dirigent_examples-0.15.0.dist-info/licenses/LICENSE +18 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# An XML feed read as records, one line each, and the mapping that makes it reversible.
|
|
2
|
+
#
|
|
3
|
+
# convert.std reads an element as an object: an attribute is a key prefixed with @, the text
|
|
4
|
+
# of an element with no children is the bare string, a child element is a key of its tag, and
|
|
5
|
+
# a tag that repeats is a list in document order. Nothing else fits in, so mixed content --
|
|
6
|
+
# text sitting beside child elements -- is refused naming the element rather than flattened.
|
|
7
|
+
#
|
|
8
|
+
# xml to ndjson is the direction that scales: the root's children are the records, read with
|
|
9
|
+
# iterparse and dropped as they are written, so a feed far larger than memory converts. It is
|
|
10
|
+
# one-way, because an ndjson line carries no root element name to write a document under;
|
|
11
|
+
# xml to json is the pair that round-trips, and this document runs both over the one feed
|
|
12
|
+
# object it stores first.
|
|
13
|
+
#
|
|
14
|
+
# A conversion names two uris and carries no value, so each direction here lands in the run's
|
|
15
|
+
# scratch space and a storage.read is what brings it back in: the ndjson as text a jq program
|
|
16
|
+
# splits, the json as the value it already is.
|
|
17
|
+
#
|
|
18
|
+
# Everything read out of an element is text, exactly as it is out of a csv: 4.5 is the three
|
|
19
|
+
# characters until a jq program says tonumber.
|
|
20
|
+
#
|
|
21
|
+
# dg run --local examples/transform/xml-feed-to-ndjson.yaml --keep
|
|
22
|
+
|
|
23
|
+
format: dirigent/v1
|
|
24
|
+
kind: pipeline
|
|
25
|
+
code: xml-feed-to-ndjson
|
|
26
|
+
name: An XML feed as records
|
|
27
|
+
description: |
|
|
28
|
+
Store an XML feed, convert it to one record per line, aggregate the records with jq, and
|
|
29
|
+
convert the same object to one whole value to show the mapping that round-trips.
|
|
30
|
+
|
|
31
|
+
The mapping, in four rules:
|
|
32
|
+
|
|
33
|
+
1. an attribute is a key prefixed with `@`
|
|
34
|
+
2. an element's text is the value where it has no children
|
|
35
|
+
3. a child element is a key of its tag
|
|
36
|
+
4. a tag that repeats is a list, in document order
|
|
37
|
+
|
|
38
|
+
tags: [transform, storage]
|
|
39
|
+
|
|
40
|
+
requires:
|
|
41
|
+
blocks:
|
|
42
|
+
- convert.std
|
|
43
|
+
- transform.jq
|
|
44
|
+
- storage.read
|
|
45
|
+
- storage.write
|
|
46
|
+
|
|
47
|
+
steps:
|
|
48
|
+
feed:
|
|
49
|
+
block: storage.write
|
|
50
|
+
config:
|
|
51
|
+
target: ${run.scratch}/feed.xml
|
|
52
|
+
# Written inline so the example needs no network. In a real pipeline this object is
|
|
53
|
+
# the feed an upstream http.request put in storage, and records starts there.
|
|
54
|
+
text: |
|
|
55
|
+
<?xml version="1.0" encoding="utf-8"?>
|
|
56
|
+
<feed source="stations">
|
|
57
|
+
<reading station="st-1"><region>east</region><celsius>4.5</celsius></reading>
|
|
58
|
+
<reading station="st-2"><region>east</region><celsius>-3.0</celsius></reading>
|
|
59
|
+
<reading station="st-3"><region>west</region><celsius>1.2</celsius></reading>
|
|
60
|
+
</feed>
|
|
61
|
+
content_type: application/xml
|
|
62
|
+
|
|
63
|
+
records:
|
|
64
|
+
block: convert.std
|
|
65
|
+
depends_on: [feed]
|
|
66
|
+
config:
|
|
67
|
+
source: ${steps.feed.output.uri}
|
|
68
|
+
target: ${run.scratch}/readings.ndjson
|
|
69
|
+
from: xml
|
|
70
|
+
# The records are the root's children, so each <reading> becomes one line. The root's
|
|
71
|
+
# own attribute, source="stations", is not on any of them: it belongs to the document,
|
|
72
|
+
# and the json direction below is where it survives.
|
|
73
|
+
to: ndjson
|
|
74
|
+
|
|
75
|
+
lines:
|
|
76
|
+
block: storage.read
|
|
77
|
+
depends_on: [records]
|
|
78
|
+
config:
|
|
79
|
+
source: ${steps.records.output.target}
|
|
80
|
+
# Nothing records what an .ndjson object is, and an object a read cannot name is
|
|
81
|
+
# refused rather than guessed at, so the step says it here.
|
|
82
|
+
content_type: application/x-ndjson
|
|
83
|
+
|
|
84
|
+
mean_by_region:
|
|
85
|
+
block: transform.jq
|
|
86
|
+
depends_on: [lines]
|
|
87
|
+
config:
|
|
88
|
+
# ndjson is one JSON value per line and is text rather than the JSON family, so the
|
|
89
|
+
# read handed over text and the program splits before it parses. tonumber for the one
|
|
90
|
+
# field it wants as a number.
|
|
91
|
+
input: ${steps.lines.output.text}
|
|
92
|
+
program: |
|
|
93
|
+
split("\n")
|
|
94
|
+
| map(select(length > 0) | fromjson)
|
|
95
|
+
| group_by(.region)
|
|
96
|
+
| [.[] | {region: .[0].region,
|
|
97
|
+
stations: [.[] | ."@station"],
|
|
98
|
+
mean_celsius: ([.[] | .celsius | tonumber] | add / length)}]
|
|
99
|
+
|
|
100
|
+
whole_document:
|
|
101
|
+
block: convert.std
|
|
102
|
+
depends_on: [feed]
|
|
103
|
+
config:
|
|
104
|
+
# The same feed the other way: one object, keyed by the root element's tag, carrying
|
|
105
|
+
# the document's own @source and the repeated readings as a list. This is the pair
|
|
106
|
+
# that round-trips -- json to xml writes this object back out as the feed.
|
|
107
|
+
source: ${steps.feed.output.uri}
|
|
108
|
+
target: ${run.scratch}/feed.json
|
|
109
|
+
from: xml
|
|
110
|
+
to: json
|
|
111
|
+
|
|
112
|
+
document:
|
|
113
|
+
block: storage.read
|
|
114
|
+
depends_on: [whole_document]
|
|
115
|
+
config:
|
|
116
|
+
source: ${steps.whole_document.output.target}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# A YAML document is one value, so a config file converts to the object it describes.
|
|
2
|
+
#
|
|
3
|
+
# This is the half of convert.std that is not the sequence of records: yaml and json trade
|
|
4
|
+
# one whole value in either direction, so a settings file becomes a JSON object a jq program
|
|
5
|
+
# can read, and a JSON value becomes a YAML document again. The stream spelling -- one `---`
|
|
6
|
+
# document per ndjson line -- is the other half, and examples/transform/ndjson-round-trip.yaml
|
|
7
|
+
# is where the sequence side lives.
|
|
8
|
+
#
|
|
9
|
+
# The config is an object in storage from the first hop, because a conversion reads a uri
|
|
10
|
+
# and writes a uri and never holds a value. storage.read is what brings the parsed config
|
|
11
|
+
# into the run for jq, and storage.write is what puts the answer back where the last
|
|
12
|
+
# conversion can pick it up.
|
|
13
|
+
#
|
|
14
|
+
# The one thing to know before pointing this at a real file: YAML says things JSON cannot.
|
|
15
|
+
# An unquoted 2026-03-01 is a date, not a string, and the codec refuses it naming its path
|
|
16
|
+
# rather than guessing what text it meant. That is why the retention date below is quoted.
|
|
17
|
+
#
|
|
18
|
+
# dg run --local examples/transform/yaml-config-to-json.yaml --keep
|
|
19
|
+
|
|
20
|
+
format: dirigent/v1
|
|
21
|
+
kind: pipeline
|
|
22
|
+
code: yaml-config-to-json
|
|
23
|
+
name: A config as an object
|
|
24
|
+
description: |
|
|
25
|
+
Read a stored YAML config as the JSON object it describes, take what a run needs out of
|
|
26
|
+
it, and write the answer back out as YAML.
|
|
27
|
+
|
|
28
|
+
`convert.std` reads `yaml` two ways. This document uses the first:
|
|
29
|
+
|
|
30
|
+
1. one **document** is one value, traded with `json`
|
|
31
|
+
2. one **stream** of documents is one line each, traded with `ndjson`
|
|
32
|
+
|
|
33
|
+
tags: [transform, storage]
|
|
34
|
+
|
|
35
|
+
requires:
|
|
36
|
+
blocks:
|
|
37
|
+
- convert.std
|
|
38
|
+
- transform.jq
|
|
39
|
+
- storage.read
|
|
40
|
+
- storage.write
|
|
41
|
+
|
|
42
|
+
steps:
|
|
43
|
+
config:
|
|
44
|
+
block: storage.write
|
|
45
|
+
config:
|
|
46
|
+
target: ${run.scratch}/settings.yaml
|
|
47
|
+
# Written inline so the example needs no network. In a real pipeline this object is
|
|
48
|
+
# the config a git.checkout or an http.request put in storage, and parse starts there.
|
|
49
|
+
text: |
|
|
50
|
+
name: daily-load
|
|
51
|
+
schedule: 0 3 * * *
|
|
52
|
+
# Quoted, because an unquoted date is a YAML date and JSON has no date: the codec
|
|
53
|
+
# refuses it at $.retention.before rather than deciding what text it meant.
|
|
54
|
+
retention:
|
|
55
|
+
before: "2026-03-01"
|
|
56
|
+
targets:
|
|
57
|
+
- code: warehouse
|
|
58
|
+
enabled: true
|
|
59
|
+
- code: archive
|
|
60
|
+
enabled: false
|
|
61
|
+
content_type: application/yaml
|
|
62
|
+
|
|
63
|
+
parse:
|
|
64
|
+
block: convert.std
|
|
65
|
+
depends_on: [config]
|
|
66
|
+
config:
|
|
67
|
+
source: ${steps.config.output.uri}
|
|
68
|
+
target: ${run.scratch}/settings.json
|
|
69
|
+
from: yaml
|
|
70
|
+
to: json
|
|
71
|
+
|
|
72
|
+
settings:
|
|
73
|
+
block: storage.read
|
|
74
|
+
depends_on: [parse]
|
|
75
|
+
config:
|
|
76
|
+
source: ${steps.parse.output.target}
|
|
77
|
+
|
|
78
|
+
enabled:
|
|
79
|
+
block: transform.jq
|
|
80
|
+
depends_on: [settings]
|
|
81
|
+
config:
|
|
82
|
+
# What the read handed over is the config as a value: keys are keys, and true is a
|
|
83
|
+
# boolean rather than the word.
|
|
84
|
+
input: ${steps.settings.output.value}
|
|
85
|
+
program: |
|
|
86
|
+
{name, schedule, before: .retention.before,
|
|
87
|
+
targets: [.targets[] | select(.enabled) | .code]}
|
|
88
|
+
|
|
89
|
+
store:
|
|
90
|
+
block: storage.write
|
|
91
|
+
depends_on: [enabled]
|
|
92
|
+
config:
|
|
93
|
+
target: ${run.scratch}/enabled.json
|
|
94
|
+
value: ${steps.enabled.output.value}
|
|
95
|
+
|
|
96
|
+
as_yaml:
|
|
97
|
+
block: convert.std
|
|
98
|
+
depends_on: [store]
|
|
99
|
+
config:
|
|
100
|
+
# Back the way it came: one JSON value, one YAML document.
|
|
101
|
+
source: ${steps.store.output.uri}
|
|
102
|
+
target: ${run.scratch}/enabled.yaml
|
|
103
|
+
from: json
|
|
104
|
+
to: yaml
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Trigger examples
|
|
2
|
+
|
|
3
|
+
What starts a run on its own: a clock, or an inbound webhook. A trigger is declared in the
|
|
4
|
+
document and materialised by `dg apply`; the scheduler fires the clocks and the server
|
|
5
|
+
answers the hooks, on the instance the document was applied to. Operational state -- paused,
|
|
6
|
+
last fired, the firing and delivery history -- stays in the instance, so none of it appears
|
|
7
|
+
in these files. [docs/design.md](../../docs/design.md#the-misfire-policy) is the page behind
|
|
8
|
+
the clocks.
|
|
9
|
+
|
|
10
|
+
Each file is prefixed with the clock kind it teaches, the way `transform/`'s files are
|
|
11
|
+
prefixed with the engine kind: `cron-` for a calendar moment, `interval-` for a rolling
|
|
12
|
+
cadence, `at-` for a single firing. Every one of them also runs ad hoc, with nothing on the
|
|
13
|
+
allowlist and no network:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
dg run --local examples/triggers/cron-nightly.yaml -p day=2026-01-01
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
One of them reads the window its firing covers, which no document declares and only a run
|
|
20
|
+
carries, so running it ad hoc means saying which interval it is for:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
dg run --local examples/triggers/cron-windowed.yaml --window 2026-06-01..2026-06-02
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Pipelines
|
|
27
|
+
|
|
28
|
+
| File | What it teaches |
|
|
29
|
+
| --- | --- |
|
|
30
|
+
| [cron-nightly.yaml](cron-nightly.yaml) | A calendar moment: five in the morning in the schedule's own zone, across daylight saving, with parameters pinned over the pipeline's defaults. |
|
|
31
|
+
| [cron-windowed.yaml](cron-windowed.yaml) | The interval a firing covers rather than the moment it happened at: `${run.window.start}` and `${run.window.end}`, derived from the cadence and read like a parameter. |
|
|
32
|
+
| [interval-rolling.yaml](interval-rolling.yaml) | A rolling cadence: when an interval beats a cron, and the misfire grace that turns a weekend outage into one firing rather than sixty. |
|
|
33
|
+
| [at-one-time.yaml](at-one-time.yaml) | The one-time firing: the backfill and the launch, and how a moment written without an offset is read in the schedule's own zone. |
|
|
34
|
+
| [managed-and-manual.yaml](managed-and-manual.yaml) | Against a real instance: what an apply materialises, what it removes, and the hand-made schedule it leaves alone. |
|
|
35
|
+
| [document-nightly.yaml](document-nightly.yaml) | A `kind: triggers` document: clocks for a pipeline defined elsewhere, owning its own rows and refusing a pipeline no instance holds. |
|
|
36
|
+
| [webhook-trigger.yaml](webhook-trigger.yaml) | The inbound trigger: a token minted at apply, a strict payload-to-parameter mapping, and everything else in the POST ignored. |
|
|
37
|
+
|
|
38
|
+
A pipeline carries as many schedules as it needs, each with its own zone and its own
|
|
39
|
+
parameters: nightly against production and hourly against staging is two schedules on one
|
|
40
|
+
pipeline, never two pipelines.
|
|
41
|
+
|
|
42
|
+
Every file here but one is a pipeline document that carries its own clocks, which is the
|
|
43
|
+
primary form: one file is one deployable unit, and its digest versions the triggers with the
|
|
44
|
+
steps. `document-nightly.yaml` is the other form, for a pipeline somebody else's file
|
|
45
|
+
defines -- it names `managed-and-manual`, so read the two together.
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# The third clock: one firing, at one instant, and then never again.
|
|
2
|
+
#
|
|
3
|
+
# The schedule fires only on an instance this document has been applied to. The same
|
|
4
|
+
# document runs ad hoc anywhere, with nothing on the allowlist and no network:
|
|
5
|
+
#
|
|
6
|
+
# dg run --local examples/triggers/at-one-time.yaml -p from=2025-01-01 -p to=2025-12-31
|
|
7
|
+
# dg apply examples/triggers/at-one-time.yaml # against an instance
|
|
8
|
+
# dg schedule list at-one-time
|
|
9
|
+
#
|
|
10
|
+
# This is the backfill and the launch: a job that has to happen once, at a moment agreed
|
|
11
|
+
# with somebody, and that nobody should have to be awake for. Declaring it in the document
|
|
12
|
+
# makes the moment reviewable in a pull request rather than typed into a terminal at night.
|
|
13
|
+
|
|
14
|
+
format: dirigent/v1
|
|
15
|
+
kind: pipeline
|
|
16
|
+
code: at-one-time
|
|
17
|
+
name: A single agreed instant
|
|
18
|
+
description: A one-time schedule firing a bounded backfill at a single agreed instant.
|
|
19
|
+
|
|
20
|
+
tags: [triggers, transform, schedule]
|
|
21
|
+
|
|
22
|
+
requires:
|
|
23
|
+
blocks:
|
|
24
|
+
- transform.jq
|
|
25
|
+
|
|
26
|
+
params:
|
|
27
|
+
type: object
|
|
28
|
+
properties:
|
|
29
|
+
from:
|
|
30
|
+
type: string
|
|
31
|
+
format: date
|
|
32
|
+
description: First day of the backfill window, inclusive.
|
|
33
|
+
default: "2026-01-01"
|
|
34
|
+
to:
|
|
35
|
+
type: string
|
|
36
|
+
format: date
|
|
37
|
+
description: Last day of the backfill window, inclusive.
|
|
38
|
+
default: "2026-01-07"
|
|
39
|
+
environment:
|
|
40
|
+
type: string
|
|
41
|
+
enum: [staging, production]
|
|
42
|
+
default: staging
|
|
43
|
+
|
|
44
|
+
steps:
|
|
45
|
+
window:
|
|
46
|
+
block: transform.jq
|
|
47
|
+
config:
|
|
48
|
+
input:
|
|
49
|
+
from: "${params.from}"
|
|
50
|
+
to: "${params.to}"
|
|
51
|
+
environment: "${params.environment}"
|
|
52
|
+
program: |
|
|
53
|
+
{environment, from, to, months: [range(1; 13)]}
|
|
54
|
+
|
|
55
|
+
backfill:
|
|
56
|
+
block: transform.jq
|
|
57
|
+
depends_on: [window]
|
|
58
|
+
config:
|
|
59
|
+
input: "${steps.window.output.value}"
|
|
60
|
+
program: |
|
|
61
|
+
{environment, backfilled: "\(.from)..\(.to)", batches: (.months | length)}
|
|
62
|
+
|
|
63
|
+
triggers:
|
|
64
|
+
schedules:
|
|
65
|
+
- code: launch-backfill
|
|
66
|
+
# A moment written without an offset is anchored in the zone this schedule declares,
|
|
67
|
+
# not in UTC and not in the host's local time: this is half past four in the morning
|
|
68
|
+
# in Oslo. Write the offset yourself -- 2027-01-06T03:30:00+00:00 -- and it is taken
|
|
69
|
+
# as given and the zone below is not consulted.
|
|
70
|
+
at: 2027-01-06 04:30:00
|
|
71
|
+
timezone: Europe/Oslo
|
|
72
|
+
# Once the instant has gone by there is no next firing, so the schedule pauses itself
|
|
73
|
+
# rather than being deleted: its parameters and its firing history stay readable with
|
|
74
|
+
# dg schedule firings at-one-time launch-backfill.
|
|
75
|
+
params:
|
|
76
|
+
from: "2025-01-01"
|
|
77
|
+
to: "2025-12-31"
|
|
78
|
+
environment: production
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# A calendar moment: five in the morning, Oslo time, every day.
|
|
2
|
+
#
|
|
3
|
+
# The schedule fires only on an instance this document has been applied to. The same
|
|
4
|
+
# document runs ad hoc anywhere, with nothing on the allowlist and no network:
|
|
5
|
+
#
|
|
6
|
+
# dg run --local examples/triggers/cron-nightly.yaml -p day=2026-01-01
|
|
7
|
+
# dg apply examples/triggers/cron-nightly.yaml # against an instance
|
|
8
|
+
# dg schedule list cron-nightly
|
|
9
|
+
#
|
|
10
|
+
# The pipeline is deliberately trivial -- one jq program shaping a small report -- because
|
|
11
|
+
# what this file teaches is the clock above it, not the work below it.
|
|
12
|
+
|
|
13
|
+
format: dirigent/v1
|
|
14
|
+
kind: pipeline
|
|
15
|
+
code: cron-nightly
|
|
16
|
+
name: Nightly load
|
|
17
|
+
description: |
|
|
18
|
+
One pipeline on a nightly cron schedule, in its own timezone, with pinned parameters.
|
|
19
|
+
|
|
20
|
+
What this file teaches is the **clock**, not the work below it. The zone belongs
|
|
21
|
+
to the schedule rather than to whichever host runs the scheduler, so `0 5 * * *`
|
|
22
|
+
in `Europe/Oslo` stays five in the morning across a daylight-saving boundary.
|
|
23
|
+
|
|
24
|
+
tags: [triggers, transform, schedule]
|
|
25
|
+
|
|
26
|
+
requires:
|
|
27
|
+
blocks:
|
|
28
|
+
- transform.jq
|
|
29
|
+
|
|
30
|
+
params:
|
|
31
|
+
type: object
|
|
32
|
+
properties:
|
|
33
|
+
day:
|
|
34
|
+
type: string
|
|
35
|
+
format: date
|
|
36
|
+
description: The day being loaded, which the schedule pins for every firing.
|
|
37
|
+
default: "2026-01-01"
|
|
38
|
+
environment:
|
|
39
|
+
type: string
|
|
40
|
+
enum: [staging, production]
|
|
41
|
+
default: staging
|
|
42
|
+
|
|
43
|
+
steps:
|
|
44
|
+
plan:
|
|
45
|
+
block: transform.jq
|
|
46
|
+
config:
|
|
47
|
+
input:
|
|
48
|
+
day: "${params.day}"
|
|
49
|
+
environment: "${params.environment}"
|
|
50
|
+
program: |
|
|
51
|
+
{day, environment, datasets: ["cases", "climate"]}
|
|
52
|
+
|
|
53
|
+
report:
|
|
54
|
+
block: transform.jq
|
|
55
|
+
depends_on: [plan]
|
|
56
|
+
config:
|
|
57
|
+
input: "${steps.plan.output.value}"
|
|
58
|
+
program: |
|
|
59
|
+
{loaded: .day, environment: .environment, datasets: (.datasets | length)}
|
|
60
|
+
|
|
61
|
+
triggers:
|
|
62
|
+
schedules:
|
|
63
|
+
- code: nightly
|
|
64
|
+
# A trigger carries the same pair: a code that addresses it, and an optional
|
|
65
|
+
# name and description that only ever read.
|
|
66
|
+
name: Nightly, Oslo time
|
|
67
|
+
description: Fires the day's load once the upstream extract has settled.
|
|
68
|
+
cron: "0 5 * * *"
|
|
69
|
+
# The zone belongs to the schedule, not to the host the scheduler happens to run on.
|
|
70
|
+
# This is five in the morning in Oslo all year, so it stays five in the morning across
|
|
71
|
+
# a daylight-saving boundary rather than sliding to four or six.
|
|
72
|
+
timezone: Europe/Oslo
|
|
73
|
+
# Parameters pinned here override the pipeline's defaults for this schedule's firings
|
|
74
|
+
# only: environment defaults to staging above, and every nightly firing is production.
|
|
75
|
+
# A second schedule on the same pipeline pins its own, which is why nightly against
|
|
76
|
+
# production and hourly against staging is two schedules and not two pipelines.
|
|
77
|
+
params:
|
|
78
|
+
day: "2026-01-01"
|
|
79
|
+
environment: production
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# The interval a firing covers, rather than the moment it happened at.
|
|
2
|
+
#
|
|
3
|
+
# A schedule answers "when does this start". A window answers "what does this run cover",
|
|
4
|
+
# which is the question a data pipeline actually has: the nightly load at 05:00 is meant to
|
|
5
|
+
# read yesterday, not this instant. Every schedule-fired run carries the interval that just
|
|
6
|
+
# closed -- ${run.window.start} up to but not including ${run.window.end} -- and a step reads
|
|
7
|
+
# it the way it reads a parameter.
|
|
8
|
+
#
|
|
9
|
+
# THE WINDOW IS A PROPERTY OF THE RUN, NOT OF THIS DOCUMENT. Nothing below declares it: the
|
|
10
|
+
# cadence above computes it. So an ad hoc run has to be given one, and a run that carries no
|
|
11
|
+
# window refuses the reference rather than resolving it to an empty string and quietly asking
|
|
12
|
+
# the upstream system for everything it has:
|
|
13
|
+
#
|
|
14
|
+
# dg run --local examples/triggers/cron-windowed.yaml --window 2026-06-01..2026-06-02
|
|
15
|
+
# dg apply examples/triggers/cron-windowed.yaml # against an instance
|
|
16
|
+
# dg backfill cron-windowed --schedule nightly \
|
|
17
|
+
# --from 2026-06-01T05:00:00Z --to 2026-06-08T05:00:00Z --dry-run
|
|
18
|
+
#
|
|
19
|
+
# The window derived from a cron cadence is wall-clock, not a fixed 24 hours: in Europe/Oslo
|
|
20
|
+
# the spring-forward night is 23 hours long and the fall-back night is 25, and the pair of
|
|
21
|
+
# instants below says so honestly rather than rounding it away.
|
|
22
|
+
|
|
23
|
+
format: dirigent/v1
|
|
24
|
+
kind: pipeline
|
|
25
|
+
code: cron-windowed
|
|
26
|
+
name: Windowed daily load
|
|
27
|
+
description: |
|
|
28
|
+
A nightly schedule whose runs read the **interval that just closed** rather than
|
|
29
|
+
the instant they started at.
|
|
30
|
+
|
|
31
|
+
`${run.window.start}` and `${run.window.end}` are ISO 8601 instants, half-open:
|
|
32
|
+
the start is included and the end is not, so two consecutive firings tile the
|
|
33
|
+
timeline without overlapping or leaving a gap.
|
|
34
|
+
|
|
35
|
+
tags: [triggers, transform, schedule]
|
|
36
|
+
|
|
37
|
+
requires:
|
|
38
|
+
blocks:
|
|
39
|
+
- transform.jq
|
|
40
|
+
|
|
41
|
+
params:
|
|
42
|
+
type: object
|
|
43
|
+
properties:
|
|
44
|
+
dataset:
|
|
45
|
+
type: string
|
|
46
|
+
description: Which dataset the load reads, which the schedule pins for every firing.
|
|
47
|
+
default: cases
|
|
48
|
+
|
|
49
|
+
steps:
|
|
50
|
+
# The query a real extractor would be handed. Half-open is the whole point of writing both
|
|
51
|
+
# ends: `>= from` and `< to` never double-counts a row that lands exactly on a boundary.
|
|
52
|
+
query:
|
|
53
|
+
block: transform.jq
|
|
54
|
+
config:
|
|
55
|
+
input:
|
|
56
|
+
dataset: "${params.dataset}"
|
|
57
|
+
from: "${run.window.start}"
|
|
58
|
+
to: "${run.window.end}"
|
|
59
|
+
program: |
|
|
60
|
+
{
|
|
61
|
+
dataset,
|
|
62
|
+
from,
|
|
63
|
+
to,
|
|
64
|
+
predicate: "observed_at >= \(.from) and observed_at < \(.to)"
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
receipt:
|
|
68
|
+
block: transform.jq
|
|
69
|
+
depends_on: [query]
|
|
70
|
+
config:
|
|
71
|
+
input: "${steps.query.output.value}"
|
|
72
|
+
program: |
|
|
73
|
+
{loaded: .dataset, covering: "\(.from)..\(.to)"}
|
|
74
|
+
|
|
75
|
+
triggers:
|
|
76
|
+
schedules:
|
|
77
|
+
- code: nightly
|
|
78
|
+
name: Nightly, Oslo time
|
|
79
|
+
description: Reads the day that closed at five this morning.
|
|
80
|
+
cron: "0 5 * * *"
|
|
81
|
+
# The zone belongs to the schedule, so the window it derives is a day of Oslo's clock
|
|
82
|
+
# and not a fixed number of hours. This is what makes the daylight-saving nights come
|
|
83
|
+
# out right without anything in the steps below knowing that daylight saving exists.
|
|
84
|
+
timezone: Europe/Oslo
|
|
85
|
+
params:
|
|
86
|
+
dataset: cases
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# Clocks for a pipeline defined somewhere else: the ops team's clock file.
|
|
2
|
+
#
|
|
3
|
+
# A kind: triggers document declares nothing but schedules and webhooks, and names the
|
|
4
|
+
# pipeline they fire. The pipeline itself lives in managed-and-manual.yaml, owned by
|
|
5
|
+
# whoever writes pipelines; this file is owned by whoever decides when they run. Two files,
|
|
6
|
+
# two review cycles, one instance.
|
|
7
|
+
#
|
|
8
|
+
# Applying it does exactly what a pipeline document's triggers: section does, with one
|
|
9
|
+
# difference: the rows it creates are owned by THIS document rather than by the pipeline's.
|
|
10
|
+
# So an apply here creates, redeclares and retires its own schedules and its own webhook,
|
|
11
|
+
# and never touches the two schedules and the webhook managed-and-manual.yaml declares, nor
|
|
12
|
+
# anything an operator made by hand.
|
|
13
|
+
#
|
|
14
|
+
# It refuses three things, and says which at apply:
|
|
15
|
+
#
|
|
16
|
+
# - a pipeline: that no instance holds, or that is deactivated. Apply the pipeline first;
|
|
17
|
+
# a directory apply does that for you, because pipelines go before triggers documents.
|
|
18
|
+
# - a schedule or webhook code the pipeline already carries under a different owner, and
|
|
19
|
+
# names the owner in the refusal.
|
|
20
|
+
# - pinned params: the pipeline's current parameter schema would reject, checked here at
|
|
21
|
+
# apply and again at every firing.
|
|
22
|
+
#
|
|
23
|
+
# Try it against a running instance, in this order:
|
|
24
|
+
#
|
|
25
|
+
# dg apply examples/triggers/managed-and-manual.yaml # the pipeline, first
|
|
26
|
+
# dg apply examples/triggers/document-nightly.yaml # its clocks, second
|
|
27
|
+
# dg schedule list managed-and-manual # five now: two inline, three here
|
|
28
|
+
# dg trigger-document show document-nightly # what this file owns
|
|
29
|
+
# dg trigger-document delete document-nightly # its three go; the inline two stay
|
|
30
|
+
#
|
|
31
|
+
# There is nothing to run: this document has no steps. `dg run --local` on it refuses and
|
|
32
|
+
# says to run the pipeline it names.
|
|
33
|
+
|
|
34
|
+
format: dirigent/v1
|
|
35
|
+
kind: triggers
|
|
36
|
+
code: document-nightly
|
|
37
|
+
name: Nightly batch clocks
|
|
38
|
+
description: Schedules and a webhook for a pipeline another document defines.
|
|
39
|
+
|
|
40
|
+
pipeline: managed-and-manual
|
|
41
|
+
|
|
42
|
+
triggers:
|
|
43
|
+
schedules:
|
|
44
|
+
# Four in the morning Oslo time, an hour ahead of the pipeline's own nightly schedule:
|
|
45
|
+
# this is the operations copy that loads yesterday against production, and it is a
|
|
46
|
+
# separate code so both can exist without either owner editing the other's file.
|
|
47
|
+
- code: ops-nightly
|
|
48
|
+
cron: "0 4 * * *"
|
|
49
|
+
timezone: Europe/Oslo
|
|
50
|
+
params:
|
|
51
|
+
day: "2026-01-01"
|
|
52
|
+
environment: production
|
|
53
|
+
|
|
54
|
+
# Monday mornings only, for the weekly reconciliation the same pipeline serves. A cron
|
|
55
|
+
# expression says this in one line; an interval could not, because a week of hours
|
|
56
|
+
# drifts across daylight saving and Monday does not.
|
|
57
|
+
- code: ops-weekly
|
|
58
|
+
cron: "0 6 * * 1"
|
|
59
|
+
timezone: Europe/Oslo
|
|
60
|
+
params:
|
|
61
|
+
day: "2026-01-01"
|
|
62
|
+
environment: production
|
|
63
|
+
|
|
64
|
+
# A rolling smoke test against staging, on an interval rather than a calendar: nothing
|
|
65
|
+
# about it needs to land at a particular wall-clock time, and an interval keeps firing
|
|
66
|
+
# at a fixed spacing however the clocks move.
|
|
67
|
+
- code: ops-staging-smoke
|
|
68
|
+
interval: 6h
|
|
69
|
+
params:
|
|
70
|
+
day: "2026-01-01"
|
|
71
|
+
environment: staging
|
|
72
|
+
|
|
73
|
+
webhooks:
|
|
74
|
+
# The operations team's own endpoint, beside the pipeline's. Its token is minted by the
|
|
75
|
+
# instance and shown once, and deleting this document deletes the endpoint with it --
|
|
76
|
+
# which is the point of putting it here rather than in the pipeline's own file.
|
|
77
|
+
- code: ops-manual-kick
|
|
78
|
+
params_from_payload:
|
|
79
|
+
day: "$.run.date"
|
|
80
|
+
environment: "$.run.environment"
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# A rolling cadence: every hour from whenever it was declared, not at any particular time.
|
|
2
|
+
#
|
|
3
|
+
# The schedule fires only on an instance this document has been applied to. The same
|
|
4
|
+
# document runs ad hoc anywhere, with nothing on the allowlist and no network:
|
|
5
|
+
#
|
|
6
|
+
# dg run --local examples/triggers/interval-rolling.yaml
|
|
7
|
+
# dg apply examples/triggers/interval-rolling.yaml # against an instance
|
|
8
|
+
# dg schedule list interval-rolling
|
|
9
|
+
#
|
|
10
|
+
# An interval beats a cron when what you want is a cadence -- freshness, a poll, a rolling
|
|
11
|
+
# window -- and nobody cares which minute of the hour it lands on. A cron beats an interval
|
|
12
|
+
# when the firing is a calendar moment someone downstream expects: the nightly load, the
|
|
13
|
+
# Monday report, the close of the month. Write "every 1h" as an interval and "at 05:00 Oslo"
|
|
14
|
+
# as a cron, and neither has to fake the other.
|
|
15
|
+
|
|
16
|
+
format: dirigent/v1
|
|
17
|
+
kind: pipeline
|
|
18
|
+
code: interval-rolling
|
|
19
|
+
name: A rolling interval
|
|
20
|
+
description: A pipeline on a rolling hourly interval, refreshing a window rather than hitting a calendar moment.
|
|
21
|
+
|
|
22
|
+
# Two firings of a rolling refresh are not worth having at once: if one is still going when
|
|
23
|
+
# the next hour comes round, skip that firing rather than stack a second run on it.
|
|
24
|
+
|
|
25
|
+
tags: [triggers, transform, schedule]
|
|
26
|
+
|
|
27
|
+
concurrency: skip
|
|
28
|
+
|
|
29
|
+
requires:
|
|
30
|
+
blocks:
|
|
31
|
+
- transform.jq
|
|
32
|
+
|
|
33
|
+
params:
|
|
34
|
+
type: object
|
|
35
|
+
properties:
|
|
36
|
+
window_hours:
|
|
37
|
+
type: integer
|
|
38
|
+
default: 24
|
|
39
|
+
minimum: 1
|
|
40
|
+
maximum: 168
|
|
41
|
+
environment:
|
|
42
|
+
type: string
|
|
43
|
+
enum: [staging, production]
|
|
44
|
+
default: staging
|
|
45
|
+
|
|
46
|
+
steps:
|
|
47
|
+
window:
|
|
48
|
+
block: transform.jq
|
|
49
|
+
config:
|
|
50
|
+
input:
|
|
51
|
+
hours: "${params.window_hours}"
|
|
52
|
+
environment: "${params.environment}"
|
|
53
|
+
program: |
|
|
54
|
+
{environment, hours, buckets: [range(0; .hours; 6)]}
|
|
55
|
+
|
|
56
|
+
refresh:
|
|
57
|
+
block: transform.jq
|
|
58
|
+
depends_on: [window]
|
|
59
|
+
config:
|
|
60
|
+
input: "${steps.window.output.value}"
|
|
61
|
+
program: |
|
|
62
|
+
{environment, refreshed_hours: .hours, batches: (.buckets | length)}
|
|
63
|
+
|
|
64
|
+
triggers:
|
|
65
|
+
schedules:
|
|
66
|
+
- code: hourly-refresh
|
|
67
|
+
interval: 1h
|
|
68
|
+
# A duration is the same length in every zone, so an interval never consults the
|
|
69
|
+
# calendar and the zone here is read by nothing. It is the cron and at clocks that
|
|
70
|
+
# need one.
|
|
71
|
+
timezone: UTC
|
|
72
|
+
params:
|
|
73
|
+
window_hours: 24
|
|
74
|
+
environment: staging
|
|
75
|
+
|
|
76
|
+
# The misfire policy is what keeps an hourly schedule from becoming a catchup storm.
|
|
77
|
+
# A firing late by less than the grace (scheduler_misfire_grace, five minutes by
|
|
78
|
+
# default) is ordinary: the clock advances from the slot it owed, so the cadence stays
|
|
79
|
+
# anchored where it was rather than drifting by the length of the tick or by how long a
|
|
80
|
+
# run took. A firing late by more than the grace is a misfire: it fires exactly once and
|
|
81
|
+
# the next firing is computed from now, abandoning every slot that was missed. A
|
|
82
|
+
# scheduler down over a weekend therefore wakes up and runs this once, not sixty times.
|
|
83
|
+
- code: six-hourly-production
|
|
84
|
+
interval: 6h
|
|
85
|
+
timezone: UTC
|
|
86
|
+
params:
|
|
87
|
+
window_hours: 168
|
|
88
|
+
environment: production
|