dirigent-examples 0.18.4__tar.gz → 0.20.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/PKG-INFO +3 -3
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/pyproject.toml +3 -3
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/pyproject.toml.orig +3 -3
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/README.md +31 -12
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/composition/chained-instances.yaml +13 -12
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/composition/composition-child.yaml +7 -6
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/connections.yaml +13 -10
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/demo/params-showcase.yaml +6 -9
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/demo/requires.yaml +12 -6
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/failure/error-handler.yaml +20 -15
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/failure/retries.yaml +22 -18
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/graph/README.md +2 -1
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/graph/fan-in.yaml +25 -19
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/graph/fan-out.yaml +3 -5
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/graph/linear.yaml +19 -9
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/graph/parallel-branches.yaml +13 -13
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/open-data/gdacs-disaster-updates.yaml +2 -2
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/open-data/github-releases-relay.yaml +2 -2
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/open-data/usgs-earthquakes-alert.yaml +3 -3
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/README.md +23 -6
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/connections-referenced-vs-carried.yaml +30 -20
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/deadline-on-a-sensor.yaml +5 -7
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/fan-out-continue.yaml +21 -17
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/fan-out-fail-fast.yaml +24 -22
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/fan-out-from-params.yaml +11 -13
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/fan-out-nested-objects.yaml +31 -30
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/log-levels.yaml +20 -22
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/outputs-inline-vs-storage.yaml +10 -11
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/params-every-type.yaml +1 -1
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/params-validation-refuses.yaml +5 -7
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/pipeline-run-child.yaml +19 -18
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/pipeline-run-strict.yaml +9 -9
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/retry-budget-exhausted.yaml +23 -19
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/retry-exponential-backoff.yaml +23 -22
- dirigent_examples-0.20.0/src/dirigent_examples/shelves/patterns/retry-only-transient.yaml +129 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/retry-with-jitter.yaml +20 -18
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/rule-all-done.yaml +18 -17
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/rule-all-success.yaml +23 -22
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/rule-always.yaml +23 -26
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/rule-one-failed.yaml +22 -23
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/sensor-http-ready.yaml +26 -11
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/sensor-storage-exists.yaml +18 -11
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/timeout-fails-the-step.yaml +28 -33
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/timeout-skips-the-step.yaml +47 -27
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/webhook-signed.yaml +20 -10
- dirigent_examples-0.20.0/src/dirigent_examples/shelves/playground/README.md +55 -0
- dirigent_examples-0.20.0/src/dirigent_examples/shelves/playground/a-gate-catching-drift.yaml +98 -0
- dirigent_examples-0.20.0/src/dirigent_examples/shelves/playground/a-page-at-a-time.yaml +90 -0
- dirigent_examples-0.20.0/src/dirigent_examples/shelves/playground/a-stream-to-read.yaml +102 -0
- dirigent_examples-0.20.0/src/dirigent_examples/shelves/playground/generated-rows.yaml +80 -0
- dirigent_examples-0.20.0/src/dirigent_examples/shelves/playground/in-the-middle-of-a-flow.yaml +73 -0
- dirigent_examples-0.20.0/src/dirigent_examples/shelves/playground/retry-that-really-retries.yaml +80 -0
- dirigent_examples-0.20.0/src/dirigent_examples/shelves/playground/waiting-for-a-batch.yaml +92 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/README.md +15 -4
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/http-fetch-validate-post.yaml +30 -15
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/http-follow-redirects.yaml +39 -13
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/http-get-with-query.yaml +23 -9
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/http-headers-and-auth-connection.yaml +39 -26
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/http-post-file-from-storage.yaml +24 -8
- dirigent_examples-0.18.4/src/dirigent_examples/shelves/recipes/http-post-json-echo.yaml → dirigent_examples-0.20.0/src/dirigent_examples/shelves/recipes/http-post-json-body.yaml +28 -14
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/http-post-report.yaml +36 -22
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/http-save-body-to-storage.yaml +22 -8
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/http-success-status-list.yaml +21 -3
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/http-timeout-override.yaml +26 -15
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/pagination-by-fan-out.yaml +18 -4
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/report-daily-digest.yaml +25 -9
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/report-to-webhook.yaml +16 -6
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/schema-formats.yaml +1 -1
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/storage-keep-past-the-run.yaml +8 -4
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/webhook-post-hmac.yaml +31 -19
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/webhook-post-summary.yaml +21 -9
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/s3/s3-round-trip.yaml +5 -5
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/schemas/README.md +1 -1
- dirigent_examples-0.20.0/src/dirigent_examples/shelves/schemas/playground-reading.json +25 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/showcase/README.md +5 -5
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/showcase/morning-briefing.yaml +15 -16
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/showcase/nightly-regional-load.yaml +69 -57
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/showcase/one-region-refuses.yaml +40 -36
- dirigent_examples-0.18.4/src/dirigent_examples/shelves/patterns/retry-only-transient.yaml +0 -108
- dirigent_examples-0.18.4/src/dirigent_examples/shelves/schemas/echo-reading.json +0 -18
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/LICENSE +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/__init__.py +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/py.typed +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/composition/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/composition/composition-parent.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/demo/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/demo/markdown-showcase.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/demo/weekly-import-malawi.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/demo/weekly-import-nepal.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/docker/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/docker/docker-build-push.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/docker/docker-build-run.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/docker/docker-compose-database.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/docker/docker-compose-failing-up.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/docker/docker-compose-file.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/docker/docker-compose-profiles-env.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/docker/docker-compose-stack.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/docker/docker-hello.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/docker/docker-remote-daemon.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/docker/docker-run-failing-teardown.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/docker/docker-ticker.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/execute/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/execute/long-log.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/failure/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/failure/optional-step.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/failure/retry-budget.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/failure/step-timeout.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/git/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/git/git-checkout-build.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/git/git-checkout-compose.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/git/git-checkout-public.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/graph/deep-chain.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/graph/parallel-sleep.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/graph/skip-diamond.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/graph/wide-fan.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/hello-world.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/open-data/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/open-data/feeds-composition.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/open-data/hdx-dataset-watch.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/open-data/kobo-submissions-to-csv.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/open-data/nominatim-geocode-facilities.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/open-data/odk-central-submissions.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/open-data/open-meteo-weekly-report.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/open-data/overpass-health-facilities.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/open-data/who-gho-indicators-to-parquet.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/open-data/wikidata-country-reference.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/open-data/world-bank-population-trend.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/concurrency-queue.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/concurrency-replace.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/concurrency-skip.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/fan-out-item-wise.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/fan-out-literal-list.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/fan-out-then-join.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/importance-critical.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/pipeline-run-fire-and-forget.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/pipeline-run-wait.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/pipeline-run-with-params.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/poll-cadence.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/priority-layered.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/references-cheat-sheet.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/schedule-at-once.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/schedule-cron-timezone.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/schedule-interval.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/schedule-window-half-open.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/step-names-and-keys.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/patterns/webhook-mapping-nested-payload.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/preview/s3-parquet-to-ingestion.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/python/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/python/apply_and_run.py +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/python/ci_gate.py +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/python/connections.py +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/python/error_handling.py +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/python/follow_logs.py +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/python/list_and_filter.py +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/queues/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/queues/kafka-consume-then-transform.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/queues/kafka-produce-then-consume.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/queues/rabbitmq-consume-ack-on-success.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/queues/report-to-kafka.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/queues/report-to-rabbitmq.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/csv-header-rules.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/csv-to-ndjson.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/etl-csv-clean-validate-parquet.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/filter-by-predicate.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/filter-then-map-then-reduce.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/jq-dedupe-by-key.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/jq-defaults-and-nulls.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/jq-group-by-and-sum.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/jq-join-two-lists.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/jq-long-to-wide.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/jq-nested-to-flat.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/jq-pivot-wide-to-long.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/jq-running-totals.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/jq-string-cleaning.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/jq-top-n.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/jq-validate-in-jq-vs-schema.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/jq-window-dates.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/json-to-csv-flattening.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/large-output-to-storage.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/map-enrich-with-lookup.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/ndjson-to-parquet.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/parquet-round-trip-types.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/reconcile-two-sources.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/report-built-in.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/report-to-file.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/schema-carried.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/schema-referenced.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/schema-refuses-then-rule.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/storage-copy-dated-archive.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/storage-exists-gate.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/storage-manifest-of-a-fan-out.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/recipes/storage-write-then-read.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/s3/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/s3/report-to-s3.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/s3/s3-copy-and-verify.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/s3/s3-csv-report.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/s3/s3-parquet-report.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/schemas/ou-record.json +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/schemas/station-reading.json +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/sensors/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/sensors/sensor-gate.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/sensors/time-window.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/sql/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/sql/duckdb-parquet-to-report.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/sql/sql-postgres-readonly.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/sql/sql-query-to-storage.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/sql/sql-sqlite-roundtrip.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/sql/warehouse.sql +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/transform/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/transform/csv-report.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/transform/jq-filter-and-map.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/transform/jq-group-and-aggregate.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/transform/jq-join-two-sources.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/transform/jq-reshape.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/transform/jq-stream-through-storage.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/transform/ndjson-round-trip.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/transform/parquet-round-trip.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/transform/std-convert-fan-out.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/transform/xml-feed-to-ndjson.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/transform/yaml-config-to-json.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/triggers/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/triggers/at-one-time.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/triggers/cron-nightly.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/triggers/cron-windowed.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/triggers/document-nightly.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/triggers/interval-rolling.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/triggers/managed-and-manual.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/triggers/webhook-trigger.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/validate/README.md +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/validate/expects-a-shape.yaml +0 -0
- {dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/validate/the-shape-is-wrong.yaml +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dirigent-examples
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.20.0
|
|
4
4
|
Summary: The dirigent example corpus: the shelves of runnable documents, installed.
|
|
5
5
|
License-Expression: LicenseRef-Proprietary
|
|
6
6
|
License-File: LICENSE
|
|
7
7
|
Classifier: Programming Language :: Python :: 3
|
|
8
8
|
Classifier: Programming Language :: Python :: 3.13
|
|
9
|
-
Requires-Dist: dirigent-client==0.
|
|
10
|
-
Requires-Dist: dirigent-plugin==0.
|
|
9
|
+
Requires-Dist: dirigent-client==0.20.0
|
|
10
|
+
Requires-Dist: dirigent-plugin==0.20.0
|
|
11
11
|
Requires-Python: >=3.13
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dirigent-examples"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.20.0"
|
|
4
4
|
description = "The dirigent example corpus: the shelves of runnable documents, installed."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.13"
|
|
@@ -11,8 +11,8 @@ classifiers = [
|
|
|
11
11
|
"Programming Language :: Python :: 3.13",
|
|
12
12
|
]
|
|
13
13
|
dependencies = [
|
|
14
|
-
"dirigent-client==0.
|
|
15
|
-
"dirigent-plugin==0.
|
|
14
|
+
"dirigent-client==0.20.0",
|
|
15
|
+
"dirigent-plugin==0.20.0",
|
|
16
16
|
]
|
|
17
17
|
|
|
18
18
|
[project.entry-points."dirigent.plugins.v1"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dirigent-examples"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.20.0"
|
|
4
4
|
description = "The dirigent example corpus: the shelves of runnable documents, installed."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.13"
|
|
@@ -12,8 +12,8 @@ classifiers = [
|
|
|
12
12
|
]
|
|
13
13
|
dependencies = [
|
|
14
14
|
# The python/ shelf is SDK scripts a reader runs, so the corpus carries the client.
|
|
15
|
-
"dirigent-client==0.
|
|
16
|
-
"dirigent-plugin==0.
|
|
15
|
+
"dirigent-client==0.20.0",
|
|
16
|
+
"dirigent-plugin==0.20.0",
|
|
17
17
|
]
|
|
18
18
|
|
|
19
19
|
[project.entry-points."dirigent.plugins.v1"]
|
{dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/README.md
RENAMED
|
@@ -14,6 +14,7 @@ The documents live on topic shelves, each with its own README:
|
|
|
14
14
|
| --- | --- |
|
|
15
15
|
| [`graph/`](graph) | The shapes a DAG takes: lines, branches, fans, joins, and skips. |
|
|
16
16
|
| [`patterns/`](patterns) | One engine behaviour per file: rules, retries, the two clocks, item policies, triggers, concurrency, and the reference language. |
|
|
17
|
+
| [`playground/`](playground) | Documents that need something to happen: generated rows, a step that fails twice, a shape that drifts, a page to walk. |
|
|
17
18
|
| [`failure/`](failure) | Retries, budgets, timeouts, tolerated failures, and the cleanup edge. |
|
|
18
19
|
| [`transform/`](transform) | The reshaping verbs: jq programs and the `convert.std` codec, no allowlist anywhere. |
|
|
19
20
|
| [`recipes/`](recipes) | One question per file: how to group, join, pivot, clean, convert, check, store, and call. |
|
|
@@ -62,14 +63,17 @@ is the list of what to create first.
|
|
|
62
63
|
**The tiering rule.** `examples/` is what runs: every document there is runnable, though some
|
|
63
64
|
need infrastructure whose setup their headers document -- the [`docker/`](docker) shelf needs a
|
|
64
65
|
Docker socket or a daemon a connection names, [`queues/`](queues) needs the brokers
|
|
65
|
-
`infra/compose.queues.yaml` starts, `sql-postgres-readonly.yaml` needs a PostgreSQL,
|
|
66
|
-
`s3-round-trip.yaml` needs an S3-compatible endpoint and a connection to it
|
|
66
|
+
`infra/compose.queues.yaml` starts, `sql-postgres-readonly.yaml` needs a PostgreSQL,
|
|
67
|
+
`s3-round-trip.yaml` needs an S3-compatible endpoint and a connection to it, and the documents
|
|
68
|
+
that make a real HTTP call need a `dg dev` to make it to.
|
|
67
69
|
`preview/` is for what cannot run yet because the *code* does not exist: documents whose block
|
|
68
70
|
ids belong to adapter packs nobody has written. A missing credential is not a preview; a
|
|
69
71
|
missing package is.
|
|
70
72
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
+
Almost nothing here reaches the network. Where a document needs something to *happen* --
|
|
74
|
+
rows to fan out over, a step that fails twice, a shape that drifts, a payload over the
|
|
75
|
+
storage threshold -- it uses `playground.generate`, a node that generates it and reaches
|
|
76
|
+
nothing at all. So you can run any of these for real without standing anything up:
|
|
73
77
|
|
|
74
78
|
```bash
|
|
75
79
|
dg run --local examples/hello-world.yaml
|
|
@@ -110,9 +114,20 @@ dg apply examples/graph/linear.yaml
|
|
|
110
114
|
dg run linear -p day=2026-01-01 --watch
|
|
111
115
|
```
|
|
112
116
|
|
|
117
|
+
The documents that are *about* HTTP are the exception, and they call the playground routes a
|
|
118
|
+
dirigent instance serves under `/api/v1/playground`, unauthenticated: the whole
|
|
119
|
+
[`recipes/` HTTP section](recipes/README.md), the connection and webhook files on
|
|
120
|
+
[`patterns/`](patterns), [`demo/requires.yaml`](demo/requires.yaml) and
|
|
121
|
+
[`composition/chained-instances.yaml`](composition/chained-instances.yaml). A local run mounts
|
|
122
|
+
no API of its own, so start an instance first and they reach that:
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
dg dev &
|
|
126
|
+
dg run --local examples/recipes/http-get-with-query.yaml
|
|
127
|
+
```
|
|
128
|
+
|
|
113
129
|
The CI lane that walks these files never touches the network: it validates the documents
|
|
114
|
-
and round-trips them, and the execution tests elsewhere use a mock transport.
|
|
115
|
-
is for humans running the examples for real.
|
|
130
|
+
and round-trips them, and the execution tests elsewhere use a mock transport.
|
|
116
131
|
|
|
117
132
|
## Engine patterns
|
|
118
133
|
|
|
@@ -143,7 +158,9 @@ they name credentials rather than carrying them, so the credential has to exist
|
|
|
143
158
|
document is applied. On a server, create it once:
|
|
144
159
|
|
|
145
160
|
```bash
|
|
146
|
-
dg connection create http
|
|
161
|
+
dg connection create http playground \
|
|
162
|
+
--set base_url=http://127.0.0.1:3333/api/v1/playground \
|
|
163
|
+
--set basic_username=playground --set basic_password=playground
|
|
147
164
|
```
|
|
148
165
|
|
|
149
166
|
A local run has no server to hold it, so it is handed the connection instead:
|
|
@@ -152,9 +169,9 @@ A local run has no server to hold it, so it is handed the connection instead:
|
|
|
152
169
|
dg run --local examples/demo/requires.yaml --connections examples/connections.yaml
|
|
153
170
|
```
|
|
154
171
|
|
|
155
|
-
[connections.yaml](connections.yaml) is that file. The basic-auth credentials in it are
|
|
156
|
-
|
|
157
|
-
rest and redacted in every API response.
|
|
172
|
+
[connections.yaml](connections.yaml) is that file. The basic-auth credentials in it are the
|
|
173
|
+
playground's own documented pair, which guards nothing. `basic_password` is a `SecretStr`, so
|
|
174
|
+
it is encrypted at rest and redacted in every API response.
|
|
158
175
|
|
|
159
176
|
## Schemas
|
|
160
177
|
|
|
@@ -181,7 +198,9 @@ than while learning the format. Each file answers one question a person doing da
|
|
|
181
198
|
actually asks -- how do I group and sum, how do I join two lists, which types survive a
|
|
182
199
|
parquet round trip, where do credentials live -- in a complete pipeline whose header comment
|
|
183
200
|
is the lesson. Nothing on the shelf needs infrastructure, a credential, or an allowlist
|
|
184
|
-
entry: the data is inline, generated by a jq program, or
|
|
201
|
+
entry: the data is inline, generated by a jq program, or generated by `playground.generate`.
|
|
202
|
+
The recipes under HTTP and webhooks call the instance's own playground, so those want a
|
|
203
|
+
`dg dev` running.
|
|
185
204
|
|
|
186
205
|
```bash
|
|
187
206
|
dg run --local examples/recipes/jq-group-by-and-sum.yaml
|
|
@@ -286,7 +305,7 @@ what a failure looks like, so the seeded one is deliberately mixed:
|
|
|
286
305
|
| Documents refused | 18 | Nine in [docker/](docker) and two in [git/](git) name a compose or a build block the seed does not allowlist, some of them also a connection it does not create; four name a connection that does not exist, one of them (`warehouse-nobody-created`) built by the seed on purpose; one names a schema no instance here holds; and two require a pipeline or a target applied after them |
|
|
287
306
|
| Schedules | 17, all paused | `--paused` is what stops seventeen clocks starting to fire at somebody who has not looked at them |
|
|
288
307
|
| Runs | 3 succeeded, 1 with errors, 2 failed | `hello-world`, `transform/jq-reshape.yaml` and `triggers/cron-windowed.yaml` settle green; `optional-step.yaml` settles `completed_with_errors`, which is a third status rather than a shade of failed; `error-handler.yaml` fails by design, and `s3-round-trip.yaml` cannot reach an object store nobody started |
|
|
289
|
-
| Connections | 10 healthy, 4 red | The ten that answer are `
|
|
308
|
+
| Connections | 10 healthy, 4 red | The ten that answer are `playground` and the instance's own playground routes the documents carry for their own standalone runs; `artifacts` points at `127.0.0.1:9000`, `work-db` at a database nobody started, and `orders-topic` and `shop-queue` at the two brokers in `infra/compose.queues.yaml`, where nothing is listening unless you started what their headers document |
|
|
290
309
|
|
|
291
310
|
The refusals are reported by code with the reason the instance gave, and the seeding carries
|
|
292
311
|
on past each one -- a refusal is a thing to look at, not an error to fix.
|
|
@@ -18,9 +18,10 @@
|
|
|
18
18
|
# The secret lives on the connection, never in this file: a document names a credential, it
|
|
19
19
|
# does not carry one.
|
|
20
20
|
#
|
|
21
|
-
# The receiver here is
|
|
22
|
-
# against
|
|
23
|
-
# header included:
|
|
21
|
+
# The receiver here is this instance's own playground, which reflects whatever it is posted,
|
|
22
|
+
# so this runs against one instance rather than two and the reflected body shows what went on
|
|
23
|
+
# the wire, signature header included:
|
|
24
|
+
# dg dev &
|
|
24
25
|
# dg run --local examples/composition/chained-instances.yaml \
|
|
25
26
|
# --connections examples/connections.yaml -p day=2026-01-01
|
|
26
27
|
#
|
|
@@ -40,7 +41,7 @@ requires:
|
|
|
40
41
|
- http.request
|
|
41
42
|
- webhook.post
|
|
42
43
|
connections:
|
|
43
|
-
-
|
|
44
|
+
- playground
|
|
44
45
|
|
|
45
46
|
params:
|
|
46
47
|
type: object
|
|
@@ -56,8 +57,8 @@ steps:
|
|
|
56
57
|
produce:
|
|
57
58
|
block: http.request
|
|
58
59
|
config:
|
|
59
|
-
connection:
|
|
60
|
-
path: /
|
|
60
|
+
connection: playground
|
|
61
|
+
path: /request
|
|
61
62
|
method: POST
|
|
62
63
|
body:
|
|
63
64
|
day: "${params.day}"
|
|
@@ -70,11 +71,11 @@ steps:
|
|
|
70
71
|
max_attempts: 3
|
|
71
72
|
backoff: 5s
|
|
72
73
|
config:
|
|
73
|
-
connection:
|
|
74
|
-
path: /
|
|
74
|
+
connection: playground
|
|
75
|
+
path: /request
|
|
75
76
|
# The receiving instance verifies the signature against the same secret, so a handoff
|
|
76
77
|
# cannot be forged by anything that merely knows the URL.
|
|
77
|
-
sign_with:
|
|
78
|
+
sign_with: playground
|
|
78
79
|
# A receiver that queues the work answers 202, which is a success here even though the
|
|
79
80
|
# work has not happened yet: the handoff is what this step is responsible for.
|
|
80
81
|
success_status: [200, 201, 202]
|
|
@@ -83,14 +84,14 @@ steps:
|
|
|
83
84
|
body:
|
|
84
85
|
day: "${params.day}"
|
|
85
86
|
upstream_run: "${run.id}"
|
|
86
|
-
rows: "${steps.produce.output.body.
|
|
87
|
+
rows: "${steps.produce.output.body.request.body.rows}"
|
|
87
88
|
|
|
88
89
|
confirm:
|
|
89
90
|
block: http.request
|
|
90
91
|
depends_on: [hand_off]
|
|
91
92
|
config:
|
|
92
|
-
connection:
|
|
93
|
-
path: /
|
|
93
|
+
connection: playground
|
|
94
|
+
path: /request
|
|
94
95
|
method: POST
|
|
95
96
|
body:
|
|
96
97
|
handed_off_with_status: "${steps.hand_off.output.status}"
|
|
@@ -20,7 +20,7 @@ code: composition-child
|
|
|
20
20
|
name: The called pipeline
|
|
21
21
|
description: Load one region's data for one day; callable on its own or by a parent pipeline.
|
|
22
22
|
|
|
23
|
-
tags: [composition,
|
|
23
|
+
tags: [composition, playground, storage]
|
|
24
24
|
|
|
25
25
|
params:
|
|
26
26
|
type: object
|
|
@@ -37,13 +37,14 @@ params:
|
|
|
37
37
|
|
|
38
38
|
steps:
|
|
39
39
|
fetch:
|
|
40
|
-
block:
|
|
40
|
+
block: playground.generate
|
|
41
41
|
retry:
|
|
42
42
|
max_attempts: 3
|
|
43
43
|
config:
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
44
|
+
# Stands in for the region's source read. Seeded from the region's own name, so two
|
|
45
|
+
# children of the same parent load genuinely different rows.
|
|
46
|
+
rows: 10
|
|
47
|
+
input:
|
|
47
48
|
region: "${params.region}"
|
|
48
49
|
day: "${params.day}"
|
|
49
50
|
|
|
@@ -54,7 +55,7 @@ steps:
|
|
|
54
55
|
# The child writes under its own run's scratch, so two children of the same parent
|
|
55
56
|
# never write to the same place, whatever region they were given.
|
|
56
57
|
target: "${run.scratch}/regions/${params.region}.json"
|
|
57
|
-
value: "${steps.fetch.output.
|
|
58
|
+
value: "${steps.fetch.output.records}"
|
|
58
59
|
|
|
59
60
|
archive:
|
|
60
61
|
block: storage.copy
|
{dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/connections.yaml
RENAMED
|
@@ -1,35 +1,38 @@
|
|
|
1
1
|
# Connections for `dg run --local`, which has no instance to hold credentials.
|
|
2
2
|
#
|
|
3
3
|
# On a server these live encrypted in the database and are created once:
|
|
4
|
-
# dg connection create http
|
|
4
|
+
# dg connection create http playground \
|
|
5
|
+
# --set base_url=http://127.0.0.1:3333/api/v1/playground
|
|
5
6
|
#
|
|
6
7
|
# A local run has no server, so it is given them here instead, and they are sealed with
|
|
7
8
|
# an ephemeral key that is thrown away with the throwaway database:
|
|
8
|
-
# dg run --local examples/
|
|
9
|
+
# dg run --local examples/demo/requires.yaml --connections examples/connections.yaml
|
|
9
10
|
#
|
|
10
|
-
# The
|
|
11
|
-
#
|
|
12
|
-
#
|
|
11
|
+
# The playground is served by a dirigent instance under /api/v1/playground, unauthenticated,
|
|
12
|
+
# so a document that needs a real HTTP hop calls its own instance and nothing else. Start one
|
|
13
|
+
# with `dg dev` before running anything that names this connection. The basic-auth pair below
|
|
14
|
+
# is the playground's own documented credential: it guards nothing, and basic_password is a
|
|
15
|
+
# SecretStr, so it is encrypted at rest and redacted in every API response.
|
|
13
16
|
#
|
|
14
17
|
# hmac_secret is the shared secret webhook.post signs an outbound body with, and the one a
|
|
15
18
|
# receiving instance declares its inbound webhook with. It is a SecretStr too.
|
|
16
19
|
|
|
17
20
|
connections:
|
|
18
|
-
|
|
21
|
+
playground:
|
|
19
22
|
kind: http
|
|
20
23
|
config:
|
|
21
24
|
# A document names the connection and a path; the base URL lives here, so moving an
|
|
22
25
|
# instance from staging to production edits the connection and no pipeline.
|
|
23
|
-
base_url:
|
|
24
|
-
basic_username:
|
|
26
|
+
base_url: http://127.0.0.1:3333/api/v1/playground
|
|
27
|
+
basic_username: playground
|
|
25
28
|
# Sealed on the way in and never read back out: an applied connection returns which
|
|
26
29
|
# fields are set, never what they are set to.
|
|
27
|
-
basic_password:
|
|
30
|
+
basic_password: playground
|
|
28
31
|
hmac_secret: a shared secret between two dirigent instances
|
|
29
32
|
timeout: 30s
|
|
30
33
|
# What dg connection check calls. A connection with no health path can only be tested
|
|
31
34
|
# by a real run.
|
|
32
|
-
health_path: /
|
|
35
|
+
health_path: /request
|
|
33
36
|
|
|
34
37
|
# The Kafka cluster infra/compose.queues.yaml brings up: one Redpanda node, unprotected,
|
|
35
38
|
# on the port it advertises itself as. A cluster that authenticates writes security:
|
|
@@ -35,7 +35,7 @@ code: params-showcase
|
|
|
35
35
|
name: Every parameter shape
|
|
36
36
|
description: Everything a run form is built from, in one parameter schema.
|
|
37
37
|
|
|
38
|
-
tags: [demo,
|
|
38
|
+
tags: [demo, playground]
|
|
39
39
|
|
|
40
40
|
params:
|
|
41
41
|
type: object
|
|
@@ -72,21 +72,18 @@ params:
|
|
|
72
72
|
|
|
73
73
|
steps:
|
|
74
74
|
process:
|
|
75
|
-
block:
|
|
75
|
+
block: playground.generate
|
|
76
76
|
# The fan-out is over a parameter, so how wide the run is is decided when it starts
|
|
77
77
|
# rather than when the document is written.
|
|
78
78
|
for_each: "${params.regions}"
|
|
79
79
|
items: continue
|
|
80
80
|
config:
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
81
|
+
# Every resolved parameter, carried into the step's own output, so the run shows what
|
|
82
|
+
# the schema above actually produced. A whole reference keeps its type: batch_size
|
|
83
|
+
# arrives as a number and dry_run as a boolean.
|
|
84
|
+
input:
|
|
84
85
|
day: "${params.day}"
|
|
85
86
|
dataset: "${params.dataset}"
|
|
86
87
|
region: "${item}"
|
|
87
88
|
batch_size: "${params.batch_size}"
|
|
88
89
|
dry_run: "${params.dry_run}"
|
|
89
|
-
body:
|
|
90
|
-
day: "${params.day}"
|
|
91
|
-
dataset: "${params.dataset}"
|
|
92
|
-
region: "${item}"
|
|
@@ -9,10 +9,16 @@
|
|
|
9
9
|
#
|
|
10
10
|
# The connection carries the credentials, and the document carries only its name. Create it
|
|
11
11
|
# once on a server:
|
|
12
|
-
# dg connection create http
|
|
12
|
+
# dg connection create http playground \
|
|
13
|
+
# --set base_url=http://127.0.0.1:3333/api/v1/playground \
|
|
14
|
+
# --set basic_username=playground --set basic_password=playground
|
|
13
15
|
#
|
|
14
16
|
# Or hand it to a local run, which has no server to hold it:
|
|
17
|
+
# dg dev &
|
|
15
18
|
# dg run --local examples/demo/requires.yaml --connections examples/connections.yaml
|
|
19
|
+
#
|
|
20
|
+
# The playground is served by a dirigent instance under /api/v1/playground, so the calls below
|
|
21
|
+
# reach the instance `dg dev` started and nothing on the public internet.
|
|
16
22
|
|
|
17
23
|
format: dirigent/v1
|
|
18
24
|
kind: pipeline
|
|
@@ -28,7 +34,7 @@ requires:
|
|
|
28
34
|
- storage.write
|
|
29
35
|
- storage.copy
|
|
30
36
|
connections:
|
|
31
|
-
-
|
|
37
|
+
- playground
|
|
32
38
|
|
|
33
39
|
steps:
|
|
34
40
|
authenticate:
|
|
@@ -36,16 +42,16 @@ steps:
|
|
|
36
42
|
config:
|
|
37
43
|
# The connection holds the base URL and the credential, so the document names a path
|
|
38
44
|
# and never a secret. What is stored is which connection, not what is in it.
|
|
39
|
-
connection:
|
|
40
|
-
path: /
|
|
45
|
+
connection: playground
|
|
46
|
+
path: /auth
|
|
41
47
|
method: GET
|
|
42
48
|
|
|
43
49
|
publish:
|
|
44
50
|
block: http.request
|
|
45
51
|
depends_on: [authenticate]
|
|
46
52
|
config:
|
|
47
|
-
connection:
|
|
48
|
-
path: /
|
|
53
|
+
connection: playground
|
|
54
|
+
path: /request
|
|
49
55
|
method: POST
|
|
50
56
|
body:
|
|
51
57
|
authenticated: "${steps.authenticate.output.status}"
|
|
@@ -25,13 +25,15 @@
|
|
|
25
25
|
# than one bad step.
|
|
26
26
|
#
|
|
27
27
|
# As written the load fails, so the notify branch runs and the success branch is skipped.
|
|
28
|
+
# The load is a playground.generate step with `fail_until` set, which fails the attempts
|
|
29
|
+
# inside that window on purpose, so the failing path is real and reaches nothing to fail at.
|
|
28
30
|
#
|
|
29
31
|
# This one uses shell.run, which executes code on the worker and is refused unless the
|
|
30
32
|
# instance allowlists it: export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["shell.run"]', or pass
|
|
31
33
|
# --enable-unsafe shell.run to a local run.
|
|
32
34
|
#
|
|
33
35
|
# dg run --local examples/failure/error-handler.yaml
|
|
34
|
-
# dg run --local examples/failure/error-handler.yaml -p
|
|
36
|
+
# dg run --local examples/failure/error-handler.yaml -p fail_until=0 # the other branch
|
|
35
37
|
|
|
36
38
|
format: dirigent/v1
|
|
37
39
|
kind: pipeline
|
|
@@ -39,36 +41,39 @@ code: error-handler
|
|
|
39
41
|
name: Handle a failing step
|
|
40
42
|
description: Load a batch, alert on failure, and always release the lock afterwards.
|
|
41
43
|
|
|
42
|
-
tags: [failure, execute,
|
|
44
|
+
tags: [failure, execute, playground, graph]
|
|
43
45
|
|
|
44
46
|
params:
|
|
45
47
|
type: object
|
|
46
48
|
properties:
|
|
47
|
-
|
|
49
|
+
fail_until:
|
|
48
50
|
type: integer
|
|
49
|
-
description:
|
|
50
|
-
default:
|
|
51
|
+
description: How many of the load's attempts fail; 0 makes it succeed on the first.
|
|
52
|
+
default: 9
|
|
53
|
+
minimum: 0
|
|
54
|
+
maximum: 9
|
|
51
55
|
|
|
52
56
|
steps:
|
|
53
57
|
load_batch:
|
|
54
|
-
block:
|
|
58
|
+
block: playground.generate
|
|
55
59
|
retry:
|
|
56
60
|
max_attempts: 2
|
|
57
61
|
backoff: 1s
|
|
58
62
|
config:
|
|
59
|
-
|
|
60
|
-
|
|
63
|
+
# Both attempts are inside the failing window, so the budget is spent and the step
|
|
64
|
+
# settles failed. The failure is classified transient, which is what a budget buys.
|
|
65
|
+
fail_until: "${params.fail_until}"
|
|
66
|
+
rows: 3
|
|
67
|
+
seed: 4
|
|
61
68
|
|
|
62
69
|
notify_failure:
|
|
63
|
-
block:
|
|
70
|
+
block: playground.generate
|
|
64
71
|
depends_on: [load_batch]
|
|
65
72
|
# one_failed is what makes this a handler: it runs only on the path where the load
|
|
66
73
|
# failed, and skips silently on the path where it did not.
|
|
67
74
|
rule: one_failed
|
|
68
75
|
config:
|
|
69
|
-
|
|
70
|
-
method: POST
|
|
71
|
-
body:
|
|
76
|
+
input:
|
|
72
77
|
text: the nightly load failed
|
|
73
78
|
|
|
74
79
|
publish_success:
|
|
@@ -79,11 +84,11 @@ steps:
|
|
|
79
84
|
argv: [echo, "load succeeded"]
|
|
80
85
|
|
|
81
86
|
release_lock:
|
|
82
|
-
block:
|
|
87
|
+
block: playground.generate
|
|
83
88
|
depends_on: [notify_failure, publish_success]
|
|
84
89
|
# Exactly one of the two branches above runs and the other skips, so all_done is what
|
|
85
90
|
# releases the lock either way. all_success would wait for a step that never runs.
|
|
86
91
|
rule: all_done
|
|
87
92
|
config:
|
|
88
|
-
|
|
89
|
-
|
|
93
|
+
input:
|
|
94
|
+
released: nightly-load
|
|
@@ -10,36 +10,40 @@
|
|
|
10
10
|
# A tolerated failure reads as succeeded to its dependents, so the branch carries on, while
|
|
11
11
|
# the run itself reports completed_with_errors rather than succeeded.
|
|
12
12
|
#
|
|
13
|
-
# As written, publish
|
|
14
|
-
#
|
|
15
|
-
#
|
|
13
|
+
# As written, publish fails three attempts over six seconds and then gives up, which is what
|
|
14
|
+
# makes the retry visible. `fail_until` is how the playground node fails on cue: it reads the
|
|
15
|
+
# attempt number the engine hands it and fails while that number is inside the window. Set it
|
|
16
|
+
# to 0 to watch the publish succeed -- the optional cache still times out, so the run still
|
|
17
|
+
# ends completed_with_errors:
|
|
16
18
|
#
|
|
17
19
|
# This one uses shell.run, which executes code on the worker and is refused unless the
|
|
18
20
|
# instance allowlists it: export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["shell.run"]', or pass
|
|
19
21
|
# --enable-unsafe shell.run to a local run.
|
|
20
22
|
#
|
|
21
|
-
# dg run --local examples/failure/retries.yaml
|
|
22
|
-
# dg run --local examples/failure/retries.yaml -p
|
|
23
|
+
# dg run --local examples/failure/retries.yaml # completed_with_errors
|
|
24
|
+
# dg run --local examples/failure/retries.yaml -p fail_until=0 # publish succeeds; completed_with_errors
|
|
23
25
|
|
|
24
26
|
format: dirigent/v1
|
|
25
27
|
kind: pipeline
|
|
26
28
|
code: retries
|
|
27
29
|
name: Retry behavior
|
|
28
|
-
description: Publish
|
|
30
|
+
description: Publish through a step that refuses, then warm an optional cache that is slow.
|
|
29
31
|
|
|
30
|
-
tags: [failure, execute,
|
|
32
|
+
tags: [failure, execute, playground, retry]
|
|
31
33
|
|
|
32
34
|
params:
|
|
33
35
|
type: object
|
|
34
36
|
properties:
|
|
35
|
-
|
|
37
|
+
fail_until:
|
|
36
38
|
type: integer
|
|
37
|
-
description:
|
|
38
|
-
default:
|
|
39
|
+
description: How many of the publish's attempts fail; 0 makes the first one land.
|
|
40
|
+
default: 9
|
|
41
|
+
minimum: 0
|
|
42
|
+
maximum: 9
|
|
39
43
|
|
|
40
44
|
steps:
|
|
41
45
|
publish:
|
|
42
|
-
block:
|
|
46
|
+
block: playground.generate
|
|
43
47
|
continue_on_failure: true
|
|
44
48
|
retry:
|
|
45
49
|
# 2s, then 4s, then give up: multiplier doubles the wait and max_backoff caps it, so a
|
|
@@ -51,22 +55,22 @@ steps:
|
|
|
51
55
|
multiplier: 2.0
|
|
52
56
|
jitter: 0.2
|
|
53
57
|
config:
|
|
54
|
-
|
|
55
|
-
|
|
58
|
+
fail_until: "${params.fail_until}"
|
|
59
|
+
rows: 12
|
|
60
|
+
seed: 8
|
|
56
61
|
|
|
57
62
|
warm_cache:
|
|
58
|
-
block:
|
|
63
|
+
block: playground.generate
|
|
59
64
|
depends_on: [publish]
|
|
60
65
|
continue_on_failure: true
|
|
61
|
-
# The
|
|
62
|
-
#
|
|
66
|
+
# The step takes five seconds and the timeout is one, so this fails on time rather than
|
|
67
|
+
# on the work: an optional step is not allowed to hold the run open.
|
|
63
68
|
timeout: 1s
|
|
64
69
|
retry:
|
|
65
70
|
max_attempts: 2
|
|
66
71
|
backoff: 1s
|
|
67
72
|
config:
|
|
68
|
-
|
|
69
|
-
method: GET
|
|
73
|
+
delay: 5s
|
|
70
74
|
|
|
71
75
|
finish:
|
|
72
76
|
block: shell.run
|
{dirigent_examples-0.18.4 → dirigent_examples-0.20.0}/src/dirigent_examples/shelves/graph/README.md
RENAMED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
The shapes a DAG takes: a straight line, branches that split and join, a fan over a list,
|
|
4
4
|
and a chain deep enough to watch the engine walk it. Every one runs with `dg run --local`,
|
|
5
|
-
and
|
|
5
|
+
and none of them calls out at all: where a step has to do something it is
|
|
6
|
+
`playground.generate`, a node that reaches nothing.
|
|
6
7
|
|
|
7
8
|
```bash
|
|
8
9
|
dg run --local examples/graph/linear.yaml -p day=2026-01-01 --enable-unsafe shell.run
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# absent rather than null, so the list can be shorter than the input, and index 0 is
|
|
12
12
|
# the first item that worked rather than the first one asked for.
|
|
13
13
|
#
|
|
14
|
-
# The list can be indexed: ${steps.fetch.output.0.
|
|
14
|
+
# The list can be indexed: ${steps.fetch.output.0.records} is the first item's rows. Bear rule 3
|
|
15
15
|
# in mind before using it -- with one failure upstream, index 0 is a different dataset than
|
|
16
16
|
# it was yesterday. Reading the whole list is the honest form.
|
|
17
17
|
#
|
|
@@ -26,25 +26,25 @@
|
|
|
26
26
|
# with rule: all_done is refused -- there is no stored output to read, and the attempt
|
|
27
27
|
# settles as rejected rather than silently receiving an empty list.
|
|
28
28
|
#
|
|
29
|
-
# One block, used twice, and it
|
|
29
|
+
# One block, used twice, and it reaches nothing at all, so this one runs offline:
|
|
30
30
|
#
|
|
31
31
|
# dg run --local examples/graph/fan-in.yaml
|
|
32
32
|
# dg run --local examples/graph/fan-in.yaml -p datasets='["temperature","humidity"]'
|
|
33
33
|
#
|
|
34
|
-
# The latency point: four slow
|
|
34
|
+
# The latency point: four slow reads cost four parked rows and nothing else. The engine
|
|
35
35
|
# is not holding four workers for the duration; it is holding four rows in the database.
|
|
36
36
|
|
|
37
37
|
format: dirigent/v1
|
|
38
38
|
kind: pipeline
|
|
39
39
|
code: fan-in
|
|
40
40
|
name: Branches that join
|
|
41
|
-
description:
|
|
41
|
+
description: Read several datasets in parallel, then hand the whole batch on in one step.
|
|
42
42
|
|
|
43
|
-
tags: [graph,
|
|
43
|
+
tags: [graph, playground]
|
|
44
44
|
|
|
45
45
|
requires:
|
|
46
46
|
blocks:
|
|
47
|
-
-
|
|
47
|
+
- playground.generate
|
|
48
48
|
|
|
49
49
|
params:
|
|
50
50
|
type: object
|
|
@@ -57,24 +57,30 @@ params:
|
|
|
57
57
|
|
|
58
58
|
steps:
|
|
59
59
|
fetch:
|
|
60
|
-
block:
|
|
60
|
+
block: playground.generate
|
|
61
61
|
for_each: "${params.datasets}"
|
|
62
62
|
items: continue
|
|
63
63
|
config:
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
64
|
+
# The element is the input, so each item's output carries its own dataset. A fan-in
|
|
65
|
+
# reads outputs and never elements, so anything an element knows has to be in there.
|
|
66
|
+
input: "${item}"
|
|
67
|
+
seed: 5
|
|
68
|
+
fields:
|
|
69
|
+
station: license_plate
|
|
70
|
+
celsius:
|
|
71
|
+
provider: pyfloat
|
|
72
|
+
args:
|
|
73
|
+
min_value: -10
|
|
74
|
+
max_value: 30
|
|
75
|
+
right_digits: 1
|
|
68
76
|
|
|
69
77
|
report:
|
|
70
|
-
block:
|
|
78
|
+
block: playground.generate
|
|
71
79
|
depends_on: [fetch]
|
|
72
80
|
config:
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
# elements were created, so one reference collects the whole fan and an index
|
|
78
|
-
# reaches one element of it.
|
|
81
|
+
# A fan-out step's output is the list of its elements' outputs, in the order the
|
|
82
|
+
# elements were created, so one reference collects the whole fan and an index
|
|
83
|
+
# reaches one element of it.
|
|
84
|
+
input:
|
|
79
85
|
collected: "${steps.fetch.output}"
|
|
80
|
-
|
|
86
|
+
first_dataset: "${steps.fetch.output.0.records.0.value}"
|