dirigent-examples 0.20.0__tar.gz → 0.22.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/PKG-INFO +3 -3
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/pyproject.toml +3 -3
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/pyproject.toml.orig +3 -3
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/README.md +1 -1
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/failure/README.md +1 -0
- dirigent_examples-0.22.0/src/dirigent_examples/shelves/failure/fan-out-too-wide.yaml +94 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/fan-out.yaml +3 -3
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/who-gho-indicators-to-parquet.yaml +3 -4
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/README.md +5 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/fan-out-item-wise.yaml +2 -2
- dirigent_examples-0.22.0/src/dirigent_examples/shelves/patterns/fan-out-late-after-continue.yaml +77 -0
- dirigent_examples-0.22.0/src/dirigent_examples/shelves/patterns/fan-out-late-item-wise.yaml +94 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/fan-out-literal-list.yaml +4 -6
- dirigent_examples-0.22.0/src/dirigent_examples/shelves/patterns/fan-out-over-a-nested-path.yaml +83 -0
- dirigent_examples-0.22.0/src/dirigent_examples/shelves/patterns/fan-out-over-an-empty-listing.yaml +92 -0
- dirigent_examples-0.22.0/src/dirigent_examples/shelves/patterns/fan-out-over-an-output.yaml +104 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/pipeline-run-with-params.yaml +3 -3
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/python/error_handling.py +1 -1
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/queues/README.md +3 -1
- dirigent_examples-0.22.0/src/dirigent_examples/shelves/queues/kafka-watch-then-transform.yaml +88 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/pagination-by-fan-out.yaml +6 -8
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/storage-manifest-of-a-fan-out.yaml +3 -4
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/jq-reshape.yaml +2 -2
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/std-convert-fan-out.yaml +2 -2
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/README.md +7 -5
- dirigent_examples-0.22.0/src/dirigent_examples/shelves/triggers/watch-a-sensor.yaml +85 -0
- dirigent_examples-0.22.0/src/dirigent_examples/shelves/triggers/watch-fan-out-per-message.yaml +86 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/LICENSE +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/__init__.py +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/py.typed +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/composition/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/composition/chained-instances.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/composition/composition-child.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/composition/composition-parent.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/connections.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/demo/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/demo/markdown-showcase.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/demo/params-showcase.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/demo/requires.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/demo/weekly-import-malawi.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/demo/weekly-import-nepal.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-build-push.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-build-run.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-compose-database.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-compose-failing-up.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-compose-file.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-compose-profiles-env.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-compose-stack.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-hello.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-remote-daemon.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-run-failing-teardown.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-ticker.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/execute/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/execute/long-log.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/failure/error-handler.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/failure/optional-step.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/failure/retries.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/failure/retry-budget.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/failure/step-timeout.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/git/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/git/git-checkout-build.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/git/git-checkout-compose.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/git/git-checkout-public.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/deep-chain.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/fan-in.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/linear.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/parallel-branches.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/parallel-sleep.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/skip-diamond.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/wide-fan.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/hello-world.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/feeds-composition.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/gdacs-disaster-updates.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/github-releases-relay.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/hdx-dataset-watch.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/kobo-submissions-to-csv.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/nominatim-geocode-facilities.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/odk-central-submissions.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/open-meteo-weekly-report.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/overpass-health-facilities.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/usgs-earthquakes-alert.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/wikidata-country-reference.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/world-bank-population-trend.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/concurrency-queue.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/concurrency-replace.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/concurrency-skip.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/connections-referenced-vs-carried.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/deadline-on-a-sensor.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/fan-out-continue.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/fan-out-fail-fast.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/fan-out-from-params.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/fan-out-nested-objects.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/fan-out-then-join.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/importance-critical.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/log-levels.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/outputs-inline-vs-storage.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/params-every-type.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/params-validation-refuses.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/pipeline-run-child.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/pipeline-run-fire-and-forget.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/pipeline-run-strict.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/pipeline-run-wait.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/poll-cadence.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/priority-layered.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/references-cheat-sheet.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/retry-budget-exhausted.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/retry-exponential-backoff.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/retry-only-transient.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/retry-with-jitter.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/rule-all-done.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/rule-all-success.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/rule-always.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/rule-one-failed.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/schedule-at-once.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/schedule-cron-timezone.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/schedule-interval.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/schedule-window-half-open.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/sensor-http-ready.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/sensor-storage-exists.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/step-names-and-keys.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/timeout-fails-the-step.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/timeout-skips-the-step.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/webhook-mapping-nested-payload.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/webhook-signed.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/a-gate-catching-drift.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/a-page-at-a-time.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/a-stream-to-read.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/generated-rows.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/in-the-middle-of-a-flow.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/retry-that-really-retries.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/waiting-for-a-batch.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/preview/s3-parquet-to-ingestion.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/python/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/python/apply_and_run.py +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/python/ci_gate.py +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/python/connections.py +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/python/follow_logs.py +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/python/list_and_filter.py +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/queues/kafka-consume-then-transform.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/queues/kafka-produce-then-consume.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/queues/rabbitmq-consume-ack-on-success.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/queues/report-to-kafka.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/queues/report-to-rabbitmq.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/csv-header-rules.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/csv-to-ndjson.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/etl-csv-clean-validate-parquet.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/filter-by-predicate.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/filter-then-map-then-reduce.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-fetch-validate-post.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-follow-redirects.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-get-with-query.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-headers-and-auth-connection.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-post-file-from-storage.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-post-json-body.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-post-report.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-save-body-to-storage.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-success-status-list.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-timeout-override.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-dedupe-by-key.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-defaults-and-nulls.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-group-by-and-sum.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-join-two-lists.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-long-to-wide.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-nested-to-flat.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-pivot-wide-to-long.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-running-totals.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-string-cleaning.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-top-n.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-validate-in-jq-vs-schema.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-window-dates.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/json-to-csv-flattening.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/large-output-to-storage.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/map-enrich-with-lookup.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/ndjson-to-parquet.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/parquet-round-trip-types.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/reconcile-two-sources.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/report-built-in.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/report-daily-digest.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/report-to-file.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/report-to-webhook.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/schema-carried.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/schema-formats.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/schema-referenced.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/schema-refuses-then-rule.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/storage-copy-dated-archive.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/storage-exists-gate.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/storage-keep-past-the-run.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/storage-write-then-read.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/webhook-post-hmac.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/webhook-post-summary.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/s3/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/s3/report-to-s3.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/s3/s3-copy-and-verify.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/s3/s3-csv-report.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/s3/s3-parquet-report.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/s3/s3-round-trip.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/schemas/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/schemas/ou-record.json +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/schemas/playground-reading.json +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/schemas/station-reading.json +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sensors/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sensors/sensor-gate.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sensors/time-window.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/showcase/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/showcase/morning-briefing.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/showcase/nightly-regional-load.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/showcase/one-region-refuses.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sql/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sql/duckdb-parquet-to-report.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sql/sql-postgres-readonly.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sql/sql-query-to-storage.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sql/sql-sqlite-roundtrip.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sql/warehouse.sql +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/csv-report.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/jq-filter-and-map.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/jq-group-and-aggregate.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/jq-join-two-sources.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/jq-stream-through-storage.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/ndjson-round-trip.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/parquet-round-trip.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/xml-feed-to-ndjson.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/yaml-config-to-json.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/at-one-time.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/cron-nightly.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/cron-windowed.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/document-nightly.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/interval-rolling.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/managed-and-manual.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/webhook-trigger.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/validate/README.md +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/validate/expects-a-shape.yaml +0 -0
- {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/validate/the-shape-is-wrong.yaml +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dirigent-examples
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.22.0
|
|
4
4
|
Summary: The dirigent example corpus: the shelves of runnable documents, installed.
|
|
5
5
|
License-Expression: LicenseRef-Proprietary
|
|
6
6
|
License-File: LICENSE
|
|
7
7
|
Classifier: Programming Language :: Python :: 3
|
|
8
8
|
Classifier: Programming Language :: Python :: 3.13
|
|
9
|
-
Requires-Dist: dirigent-client==0.
|
|
10
|
-
Requires-Dist: dirigent-plugin==0.
|
|
9
|
+
Requires-Dist: dirigent-client==0.22.0
|
|
10
|
+
Requires-Dist: dirigent-plugin==0.22.0
|
|
11
11
|
Requires-Python: >=3.13
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dirigent-examples"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.22.0"
|
|
4
4
|
description = "The dirigent example corpus: the shelves of runnable documents, installed."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.13"
|
|
@@ -11,8 +11,8 @@ classifiers = [
|
|
|
11
11
|
"Programming Language :: Python :: 3.13",
|
|
12
12
|
]
|
|
13
13
|
dependencies = [
|
|
14
|
-
"dirigent-client==0.
|
|
15
|
-
"dirigent-plugin==0.
|
|
14
|
+
"dirigent-client==0.22.0",
|
|
15
|
+
"dirigent-plugin==0.22.0",
|
|
16
16
|
]
|
|
17
17
|
|
|
18
18
|
[project.entry-points."dirigent.plugins.v1"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dirigent-examples"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.22.0"
|
|
4
4
|
description = "The dirigent example corpus: the shelves of runnable documents, installed."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.13"
|
|
@@ -12,8 +12,8 @@ classifiers = [
|
|
|
12
12
|
]
|
|
13
13
|
dependencies = [
|
|
14
14
|
# The python/ shelf is SDK scripts a reader runs, so the corpus carries the client.
|
|
15
|
-
"dirigent-client==0.
|
|
16
|
-
"dirigent-plugin==0.
|
|
15
|
+
"dirigent-client==0.22.0",
|
|
16
|
+
"dirigent-plugin==0.22.0",
|
|
17
17
|
]
|
|
18
18
|
|
|
19
19
|
[project.entry-points."dirigent.plugins.v1"]
|
{dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/README.md
RENAMED
|
@@ -18,7 +18,7 @@ The documents live on topic shelves, each with its own README:
|
|
|
18
18
|
| [`failure/`](failure) | Retries, budgets, timeouts, tolerated failures, and the cleanup edge. |
|
|
19
19
|
| [`transform/`](transform) | The reshaping verbs: jq programs and the `convert.std` codec, no allowlist anywhere. |
|
|
20
20
|
| [`recipes/`](recipes) | One question per file: how to group, join, pivot, clean, convert, check, store, and call. |
|
|
21
|
-
| [`triggers/`](triggers) | What starts a run on its own: the three clocks,
|
|
21
|
+
| [`triggers/`](triggers) | What starts a run on its own: the three clocks, the inbound webhook, and a watch on a sensor. |
|
|
22
22
|
| [`sensors/`](sensors) | Steps that wait for the world: a drop landing, a clock window opening. |
|
|
23
23
|
| [`queues/`](queues) | A run started by a message: a Kafka topic and a RabbitMQ queue, waited on as sensors. |
|
|
24
24
|
| [`execute/`](execute) | Code on the worker: a shell step behind the allowlist. |
|
|
@@ -18,3 +18,4 @@ dg run --local examples/failure/retries.yaml
|
|
|
18
18
|
| [step-timeout.yaml](step-timeout.yaml) | A timeout ends the attempt, and the policy decides whether another one starts. |
|
|
19
19
|
| [optional-step.yaml](optional-step.yaml) | `completed_with_errors`: a step that may fail without failing the run, and the third status that says so. |
|
|
20
20
|
| [error-handler.yaml](error-handler.yaml) | The cleanup edge: alert on failure, and always release the lock, however the load went. |
|
|
21
|
+
| [fan-out-too-wide.yaml](fan-out-too-wide.yaml) | `fan_out_max_items`: a listing wider than one grid may be fails the fan-out as rejected, before any item is written. |
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# A listing wider than the instance lets one grid be, refused before any item runs.
|
|
2
|
+
#
|
|
3
|
+
# Every fan-out grid is bounded by the instance setting fan_out_max_items, 10000 unless an
|
|
4
|
+
# operator says otherwise. A grid fixed when the run is created is checked then, and a run
|
|
5
|
+
# over the bound is refused outright. A grid read from a step's output can only be checked
|
|
6
|
+
# when that output exists, so it is checked there: the step that would have fanned out fails
|
|
7
|
+
# as rejected with run.fan_out_too_wide, naming how many items it would have been and the
|
|
8
|
+
# bound, and nothing is written for any of them. rejected is never retried, because the same
|
|
9
|
+
# list would be refused the same way.
|
|
10
|
+
#
|
|
11
|
+
# The bound is there because a grid is rows: every item is a run item and an attempt, written
|
|
12
|
+
# in one transaction, and a listing that answers with a million entries by mistake should cost
|
|
13
|
+
# one refused step rather than a million rows.
|
|
14
|
+
#
|
|
15
|
+
# Hop by hop:
|
|
16
|
+
#
|
|
17
|
+
# pages eleven pages of a listing, one item per page, each a thousand generated rows.
|
|
18
|
+
# flatten the join: one list of every row, 11000 of them.
|
|
19
|
+
# each_row a fan-out over that list. 11000 is over the 10000 the instance allows, so the
|
|
20
|
+
# step fails as rejected and nothing runs.
|
|
21
|
+
# notify rule: one_failed, the error branch, which reads what went wrong.
|
|
22
|
+
#
|
|
23
|
+
# EXPECT THIS RUN TO END failed, in a few seconds, with `each_row` failed as
|
|
24
|
+
# run.fan_out_too_wide and `notify` run.
|
|
25
|
+
#
|
|
26
|
+
# To change it: -p pages='[1,2,3,4,5]' lists 5000 rows, under the bound, and the fan-out
|
|
27
|
+
# runs all 5000 of them, which takes a while.
|
|
28
|
+
# DIRIGENT_FAN_OUT_MAX_ITEMS=100 lowers the bound for one instance instead, and even one page
|
|
29
|
+
# is refused.
|
|
30
|
+
#
|
|
31
|
+
# dg run --local examples/failure/fan-out-too-wide.yaml
|
|
32
|
+
# DIRIGENT_FAN_OUT_MAX_ITEMS=100 dg run --local examples/failure/fan-out-too-wide.yaml -p pages='[1]'
|
|
33
|
+
|
|
34
|
+
format: dirigent/v1
|
|
35
|
+
kind: pipeline
|
|
36
|
+
code: fan-out-too-wide
|
|
37
|
+
name: A fan-out too wide to write
|
|
38
|
+
description: |
|
|
39
|
+
A `for_each` over a step's output that lists more than `fan_out_max_items` fails as
|
|
40
|
+
rejected with `run.fan_out_too_wide`, before a single item is written.
|
|
41
|
+
|
|
42
|
+
The error branch behind it still runs, and the run ends failed.
|
|
43
|
+
|
|
44
|
+
tags: [failure, transform, fan-out, rules]
|
|
45
|
+
|
|
46
|
+
requires:
|
|
47
|
+
blocks:
|
|
48
|
+
- playground.generate
|
|
49
|
+
- transform.jq
|
|
50
|
+
|
|
51
|
+
params:
|
|
52
|
+
type: object
|
|
53
|
+
properties:
|
|
54
|
+
pages:
|
|
55
|
+
type: array
|
|
56
|
+
default: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
|
|
57
|
+
minItems: 1
|
|
58
|
+
maxItems: 20
|
|
59
|
+
items:
|
|
60
|
+
type: integer
|
|
61
|
+
description: The thousand-row pages the listing has; eleven is over the default bound.
|
|
62
|
+
|
|
63
|
+
steps:
|
|
64
|
+
pages:
|
|
65
|
+
block: playground.generate
|
|
66
|
+
for_each: ${params.pages}
|
|
67
|
+
config:
|
|
68
|
+
rows: 1000
|
|
69
|
+
seed: 1
|
|
70
|
+
fields:
|
|
71
|
+
code: uuid4
|
|
72
|
+
|
|
73
|
+
flatten:
|
|
74
|
+
block: transform.jq
|
|
75
|
+
depends_on: [pages]
|
|
76
|
+
config:
|
|
77
|
+
input: ${steps.pages.output}
|
|
78
|
+
program: '[.[].records[]]'
|
|
79
|
+
|
|
80
|
+
each_row:
|
|
81
|
+
block: transform.jq
|
|
82
|
+
depends_on: [flatten]
|
|
83
|
+
for_each: ${steps.flatten.output.value}
|
|
84
|
+
config:
|
|
85
|
+
input: ${item}
|
|
86
|
+
program: '.code'
|
|
87
|
+
|
|
88
|
+
notify:
|
|
89
|
+
block: transform.jq
|
|
90
|
+
depends_on: [each_row]
|
|
91
|
+
rule: one_failed
|
|
92
|
+
config:
|
|
93
|
+
input: {failed: each_row}
|
|
94
|
+
program: '"\(.failed) was refused: the listing was wider than the instance allows"'
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# Fan-out: one step mapped over a list, with one run item per element.
|
|
2
2
|
#
|
|
3
|
-
# for_each is expanded when the run is created
|
|
4
|
-
#
|
|
5
|
-
#
|
|
3
|
+
# This for_each reads params, so it is expanded when the run is created and the item grid
|
|
4
|
+
# exists from the moment the run is visible. One that reads another step's output is written
|
|
5
|
+
# when that step has settled instead; patterns/fan-out-over-an-output.yaml shows it.
|
|
6
6
|
#
|
|
7
7
|
# items: continue is the item policy: a region that fails does not stop the others, its
|
|
8
8
|
# failure is recorded against its own run item, and the run reports completed_with_errors
|
|
@@ -6,10 +6,9 @@
|
|
|
6
6
|
# in Dim1, and the number in NumericValue. The envelope is OData's; the rows inside it are what
|
|
7
7
|
# anyone actually wants.
|
|
8
8
|
#
|
|
9
|
-
# What this shows, beyond the fetch: how a fan-out is read back. `for_each`
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
# item absent from it. So there is no way for item three of one step to ask item three of another
|
|
9
|
+
# What this shows, beyond the fetch: how a fan-out is read back. `for_each` here reads params,
|
|
10
|
+
# so the grid is fixed when the run is created. And a fan-out step stores one output: the list
|
|
11
|
+
# of its items' outputs, in item order, with a failed item absent from it. So there is no way for item three of one step to ask item three of another
|
|
13
12
|
# what it produced; what a downstream step gets is the whole list at once. That is why the
|
|
14
13
|
# flattening below is a single step over every envelope, and why the country is read out of
|
|
15
14
|
# SpatialDim: the join key lives in the data, where a fan-in can see it.
|
|
@@ -65,6 +65,11 @@ One step definition, many run items.
|
|
|
65
65
|
| [fan-out-then-join.yaml](fan-out-then-join.yaml) | The join, which is just a step with no `for_each` of its own | `succeeded` |
|
|
66
66
|
| [fan-out-nested-objects.yaml](fan-out-nested-objects.yaml) | Elements that are objects, and `${item.limits.max_ms}` | `succeeded` |
|
|
67
67
|
| [fan-out-item-wise.yaml](fan-out-item-wise.yaml) | A second fan-out over the same grid, and the item it pairs with | `succeeded` |
|
|
68
|
+
| [fan-out-over-an-output.yaml](fan-out-over-an-output.yaml) | List, then fan out over what was listed: a grid written when its source succeeds | `succeeded` |
|
|
69
|
+
| [fan-out-over-an-empty-listing.yaml](fan-out-over-an-empty-listing.yaml) | A listing that found nothing: the fan-out over it is skipped, and so is what needed it | `succeeded` |
|
|
70
|
+
| [fan-out-over-a-nested-path.yaml](fan-out-over-a-nested-path.yaml) | A list four fields inside an output, reached with the dotted path | `succeeded` |
|
|
71
|
+
| [fan-out-late-item-wise.yaml](fan-out-late-item-wise.yaml) | Adopting a listed grid, and `${steps.<name>.item.output}` over it | `succeeded` |
|
|
72
|
+
| [fan-out-late-after-continue.yaml](fan-out-late-after-continue.yaml) | A late grid over a fan-out under `items: continue`: only the survivors | `completed_with_errors` |
|
|
68
73
|
|
|
69
74
|
## Parameters
|
|
70
75
|
|
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
#
|
|
13
13
|
# Three things the pairing buys, and one it costs:
|
|
14
14
|
#
|
|
15
|
-
# 1.
|
|
16
|
-
#
|
|
15
|
+
# 1. The width is the adopted grid's. Here that grid comes from params, so it was fixed when
|
|
16
|
+
# the run was created; adopting a grid read from a step's output expands when that one does.
|
|
17
17
|
# 2. The names must be a direct dependency: the adopted step goes in depends_on. Pairing
|
|
18
18
|
# across a step that is not waited for would read an item that has not run.
|
|
19
19
|
# 3. Adoption chains. A third step may map over the second's items and still reach the
|
dirigent_examples-0.22.0/src/dirigent_examples/shelves/patterns/fan-out-late-after-continue.yaml
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# A fan-out whose failures were tolerated, feeding a fan-out over what it did produce.
|
|
2
|
+
#
|
|
3
|
+
# A fan-out step's output is the list of its items' outputs, and only the items that
|
|
4
|
+
# succeeded are in it. So a late grid read from a fan-out under items: continue is exactly as
|
|
5
|
+
# wide as the part of the batch that worked: the failed item is not a hole in the list, it is
|
|
6
|
+
# absent from it, and the step after it never makes an item for it.
|
|
7
|
+
#
|
|
8
|
+
# The source has to be allowed to finish for this to happen. Under the default fail_fast one
|
|
9
|
+
# failed item fails the whole step, and a fan-out behind it under all_success is skipped,
|
|
10
|
+
# with nothing expanded. Under continue the source succeeds while one item did, and the run
|
|
11
|
+
# ends completed_with_errors to say that something was tolerated.
|
|
12
|
+
#
|
|
13
|
+
# Hop by hop:
|
|
14
|
+
#
|
|
15
|
+
# probe one item per station. Each element says how many attempts its station fails;
|
|
16
|
+
# lighthouse fails one, and the default policy gives a step a single attempt, so
|
|
17
|
+
# that item fails and the other two succeed.
|
|
18
|
+
# publish a fan-out over probe's output. Two stations succeeded, so it has two items;
|
|
19
|
+
# the one that failed is not in the list, so there is no item for it at all.
|
|
20
|
+
# tally the join: how many were published, which is two.
|
|
21
|
+
#
|
|
22
|
+
# EXPECT THIS RUN TO END completed_with_errors, with one failed item under `probe`, two items
|
|
23
|
+
# under `publish`, and a tally of two.
|
|
24
|
+
#
|
|
25
|
+
# To change it: set lighthouse's fail to 0 and every station gets through, `publish` has
|
|
26
|
+
# three items, and the run succeeds.
|
|
27
|
+
#
|
|
28
|
+
# dg run --local examples/patterns/fan-out-late-after-continue.yaml
|
|
29
|
+
|
|
30
|
+
format: dirigent/v1
|
|
31
|
+
kind: pipeline
|
|
32
|
+
code: fan-out-late-after-continue
|
|
33
|
+
name: A late grid over what survived
|
|
34
|
+
description: |
|
|
35
|
+
A late grid read from a fan-out under `items: continue` has one item per item that
|
|
36
|
+
succeeded; a failed item is absent from the list, so nothing is made for it.
|
|
37
|
+
|
|
38
|
+
The run ends completed_with_errors, which says a failure was tolerated.
|
|
39
|
+
|
|
40
|
+
tags: [patterns, transform, fan-out, failure, references]
|
|
41
|
+
|
|
42
|
+
requires:
|
|
43
|
+
blocks:
|
|
44
|
+
- playground.generate
|
|
45
|
+
- transform.jq
|
|
46
|
+
|
|
47
|
+
steps:
|
|
48
|
+
probe:
|
|
49
|
+
block: playground.generate
|
|
50
|
+
for_each:
|
|
51
|
+
- {station: harbour, fail: 0}
|
|
52
|
+
- {station: lighthouse, fail: 1}
|
|
53
|
+
- {station: pier, fail: 0}
|
|
54
|
+
# One refusal does not stop the other stations, and the step still succeeds.
|
|
55
|
+
items: continue
|
|
56
|
+
config:
|
|
57
|
+
input: {station: "${item.station}"}
|
|
58
|
+
# playground.generate fails this many attempts before it succeeds, and rejected is a
|
|
59
|
+
# class no retry is spent on.
|
|
60
|
+
fail_until: ${item.fail}
|
|
61
|
+
fail_as: rejected
|
|
62
|
+
|
|
63
|
+
publish:
|
|
64
|
+
block: transform.jq
|
|
65
|
+
depends_on: [probe]
|
|
66
|
+
# probe's output is only its succeeded items' outputs, so this grid is the survivors.
|
|
67
|
+
for_each: ${steps.probe.output}
|
|
68
|
+
config:
|
|
69
|
+
input: ${item.records}
|
|
70
|
+
program: '.[0].station'
|
|
71
|
+
|
|
72
|
+
tally:
|
|
73
|
+
block: transform.jq
|
|
74
|
+
depends_on: [publish]
|
|
75
|
+
config:
|
|
76
|
+
input: ${steps.publish.output}
|
|
77
|
+
program: '{published: length, stations: [.[].value]}'
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# Two fan-outs over one listed grid: item three of the second reads item three of the first.
|
|
2
|
+
#
|
|
3
|
+
# fan-out-item-wise.yaml pairs two fan-outs over a grid that was fixed when the run was
|
|
4
|
+
# created. This is the same pairing over a grid nobody knew the width of until a step had
|
|
5
|
+
# listed it. `measure` fans out over what `stations` listed, and `label` adopts that grid with
|
|
6
|
+
# for_each: ${steps.measure.items}, so each of its items reads its own match with
|
|
7
|
+
# ${steps.measure.item.output} rather than the whole batch.
|
|
8
|
+
#
|
|
9
|
+
# An adopter of a late grid is late too. The run is created with both `measure` and `label`
|
|
10
|
+
# waiting as one pending attempt each. `measure` expands when `stations` succeeds; `label`
|
|
11
|
+
# expands when `measure` has settled, into the same number of items at the same indices, so
|
|
12
|
+
# the pairing holds by position exactly as it does over a fixed grid.
|
|
13
|
+
#
|
|
14
|
+
# Hop by hop:
|
|
15
|
+
#
|
|
16
|
+
# stations the listing: three seeded stations.
|
|
17
|
+
# measure one item per listed station, each adding a generated reading to its record.
|
|
18
|
+
# label adopts measure's grid. ${item} is the same station measure had at that index,
|
|
19
|
+
# and ${steps.measure.item.output.records} is that item's own output -- one
|
|
20
|
+
# record, the station with its reading -- not the list of all three.
|
|
21
|
+
# board the join: one line per station, in listing order.
|
|
22
|
+
#
|
|
23
|
+
# EXPECT THIS RUN TO SUCCEED at once, with three items under `measure`, three under `label`
|
|
24
|
+
# at the same indices, and a board of three lines.
|
|
25
|
+
#
|
|
26
|
+
# To change it: -p stations=6 widens both grids together; they always match.
|
|
27
|
+
#
|
|
28
|
+
# dg run --local examples/patterns/fan-out-late-item-wise.yaml
|
|
29
|
+
# dg run --local examples/patterns/fan-out-late-item-wise.yaml -p stations=6
|
|
30
|
+
|
|
31
|
+
format: dirigent/v1
|
|
32
|
+
kind: pipeline
|
|
33
|
+
code: fan-out-late-item-wise
|
|
34
|
+
name: One listed item to one item
|
|
35
|
+
description: |
|
|
36
|
+
A step adopting a grid that was expanded from an output, `for_each: ${steps.<name>.items}`,
|
|
37
|
+
expands with the same indices once that grid has settled.
|
|
38
|
+
|
|
39
|
+
Each of its items reads its match with `${steps.<name>.item.output}`.
|
|
40
|
+
|
|
41
|
+
tags: [patterns, transform, fan-out, references]
|
|
42
|
+
|
|
43
|
+
requires:
|
|
44
|
+
blocks:
|
|
45
|
+
- playground.generate
|
|
46
|
+
- transform.jq
|
|
47
|
+
|
|
48
|
+
params:
|
|
49
|
+
type: object
|
|
50
|
+
properties:
|
|
51
|
+
stations:
|
|
52
|
+
type: integer
|
|
53
|
+
default: 3
|
|
54
|
+
minimum: 1
|
|
55
|
+
maximum: 50
|
|
56
|
+
description: How many stations the listing finds, which is how wide both grids become.
|
|
57
|
+
|
|
58
|
+
steps:
|
|
59
|
+
stations:
|
|
60
|
+
block: playground.generate
|
|
61
|
+
config:
|
|
62
|
+
rows: ${params.stations}
|
|
63
|
+
seed: 7
|
|
64
|
+
fields:
|
|
65
|
+
station: city
|
|
66
|
+
|
|
67
|
+
measure:
|
|
68
|
+
block: playground.generate
|
|
69
|
+
depends_on: [stations]
|
|
70
|
+
for_each: ${steps.stations.output.records}
|
|
71
|
+
config:
|
|
72
|
+
input: ${item}
|
|
73
|
+
fields:
|
|
74
|
+
celsius:
|
|
75
|
+
provider: pyint
|
|
76
|
+
args: {min_value: -20, max_value: 30}
|
|
77
|
+
|
|
78
|
+
label:
|
|
79
|
+
block: transform.jq
|
|
80
|
+
# The adopted step goes in depends_on: this grid is measure's, and it exists once measure
|
|
81
|
+
# has settled.
|
|
82
|
+
depends_on: [measure]
|
|
83
|
+
for_each: ${steps.measure.items}
|
|
84
|
+
config:
|
|
85
|
+
# The matching item's own output: one station, with its reading.
|
|
86
|
+
input: ${steps.measure.item.output.records}
|
|
87
|
+
program: '.[0] | "\(.station): \(.celsius) C"'
|
|
88
|
+
|
|
89
|
+
board:
|
|
90
|
+
block: transform.jq
|
|
91
|
+
depends_on: [label]
|
|
92
|
+
config:
|
|
93
|
+
input: ${steps.label.output}
|
|
94
|
+
program: '[.[].value]'
|
|
@@ -7,12 +7,10 @@
|
|
|
7
7
|
# ordinary run.
|
|
8
8
|
#
|
|
9
9
|
# What a fan-out actually is: one step definition, N run items, one attempt each, all of them
|
|
10
|
-
# claimable at once.
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
#
|
|
14
|
-
# before anything executes. A fan-out over a list fetched at run time is not expressible, and
|
|
15
|
-
# that is deliberate rather than missing.
|
|
10
|
+
# claimable at once. A literal list is expanded when the run is CREATED, so the item grid exists
|
|
11
|
+
# from the moment the run is visible. A list fetched at run time is the other shape: a for_each
|
|
12
|
+
# over ${steps.<step>.output.<field>} is written when that step has settled, and
|
|
13
|
+
# fan-out-over-an-output.yaml is that on its own.
|
|
16
14
|
#
|
|
17
15
|
# Hop by hop:
|
|
18
16
|
#
|
dirigent_examples-0.22.0/src/dirigent_examples/shelves/patterns/fan-out-over-a-nested-path.yaml
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# A fan-out over a list that sits deep inside another step's output.
|
|
2
|
+
#
|
|
3
|
+
# A real answer is rarely a bare list. It is an envelope -- a catalogue, a page of results, a
|
|
4
|
+
# network with regions inside it -- and the list worth fanning out over is a few fields down.
|
|
5
|
+
# `for_each` walks there with the same dotted path every reference uses: the output, then
|
|
6
|
+
# each field in turn, until it reaches the list. Nothing has to reshape the envelope first.
|
|
7
|
+
#
|
|
8
|
+
# The path is read when `stations` has succeeded, so it is checked then too. A field the
|
|
9
|
+
# output does not have fails `per_station` as rejected with reference.no_field, naming what
|
|
10
|
+
# was there instead; a field that holds something other than a list fails it with
|
|
11
|
+
# run.fan_out_not_a_list. Either way nothing is written and nothing runs.
|
|
12
|
+
#
|
|
13
|
+
# Hop by hop:
|
|
14
|
+
#
|
|
15
|
+
# catalogue a fixed envelope standing in for an API's answer: a network, its regions,
|
|
16
|
+
# and the stations of each region.
|
|
17
|
+
# per_station one item per station of the northern region, read from four fields down.
|
|
18
|
+
# ${item} is one station object, so ${item.code} is its code.
|
|
19
|
+
# collected the join: every station code, in the order the catalogue listed them.
|
|
20
|
+
#
|
|
21
|
+
# EXPECT THIS RUN TO SUCCEED at once, with three items under `per_station` and a join naming
|
|
22
|
+
# st-101, st-102 and st-103.
|
|
23
|
+
#
|
|
24
|
+
# To change it: write `south` for `north` in the path and the fan-out is the southern
|
|
25
|
+
# region's two stations; write `east`, which the catalogue does not have, and `per_station`
|
|
26
|
+
# fails as rejected. A reference cannot hold another reference, so the path is written out.
|
|
27
|
+
#
|
|
28
|
+
# dg run --local examples/patterns/fan-out-over-a-nested-path.yaml
|
|
29
|
+
|
|
30
|
+
format: dirigent/v1
|
|
31
|
+
kind: pipeline
|
|
32
|
+
code: fan-out-over-a-nested-path
|
|
33
|
+
name: Fan out over a nested path
|
|
34
|
+
description: |
|
|
35
|
+
`for_each: ${steps.<name>.output.<field>.<field>...}` reaches a list several fields inside
|
|
36
|
+
an upstream step's output, so an envelope needs no reshaping before it is fanned out over.
|
|
37
|
+
|
|
38
|
+
A missing field fails the fan-out as rejected, with nothing run.
|
|
39
|
+
|
|
40
|
+
tags: [patterns, transform, fan-out, references]
|
|
41
|
+
|
|
42
|
+
requires:
|
|
43
|
+
blocks:
|
|
44
|
+
- value.const
|
|
45
|
+
- playground.generate
|
|
46
|
+
- transform.jq
|
|
47
|
+
|
|
48
|
+
steps:
|
|
49
|
+
catalogue:
|
|
50
|
+
block: value.const
|
|
51
|
+
config:
|
|
52
|
+
value:
|
|
53
|
+
network:
|
|
54
|
+
name: Coastal network
|
|
55
|
+
regions:
|
|
56
|
+
north:
|
|
57
|
+
stations:
|
|
58
|
+
- {code: st-101, name: Harbour}
|
|
59
|
+
- {code: st-102, name: Lighthouse}
|
|
60
|
+
- {code: st-103, name: Pier}
|
|
61
|
+
south:
|
|
62
|
+
stations:
|
|
63
|
+
- {code: st-201, name: Dunes}
|
|
64
|
+
- {code: st-202, name: Estuary}
|
|
65
|
+
|
|
66
|
+
per_station:
|
|
67
|
+
block: playground.generate
|
|
68
|
+
depends_on: [catalogue]
|
|
69
|
+
# value.const answers under `value`, and the list is four fields inside that.
|
|
70
|
+
for_each: ${steps.catalogue.output.value.network.regions.north.stations}
|
|
71
|
+
config:
|
|
72
|
+
input: ${item}
|
|
73
|
+
fields:
|
|
74
|
+
celsius:
|
|
75
|
+
provider: pyint
|
|
76
|
+
args: {min_value: -5, max_value: 25}
|
|
77
|
+
|
|
78
|
+
collected:
|
|
79
|
+
block: transform.jq
|
|
80
|
+
depends_on: [per_station]
|
|
81
|
+
config:
|
|
82
|
+
input: ${steps.per_station.output}
|
|
83
|
+
program: '[.[].records[0].code]'
|
dirigent_examples-0.22.0/src/dirigent_examples/shelves/patterns/fan-out-over-an-empty-listing.yaml
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# A listing that came back empty, and the fan-out over it that therefore never runs.
|
|
2
|
+
#
|
|
3
|
+
# fan-out-over-an-output.yaml fans out over whatever a step listed. This is the same shape on
|
|
4
|
+
# the day the listing finds nothing: `stations` succeeds with no records, so `read` has no
|
|
5
|
+
# items to write. A late grid follows the rule every grid follows -- an empty list skips the
|
|
6
|
+
# step -- and a skip is a real terminal outcome, so everything behind it under the default
|
|
7
|
+
# all_success edge is skipped too.
|
|
8
|
+
#
|
|
9
|
+
# Nothing here failed. An empty listing is a quiet day, not an error, and the run ends
|
|
10
|
+
# succeeded. `read` is skipped with a reason: its one attempt carries run.fan_out_empty and
|
|
11
|
+
# says which step left the list empty.
|
|
12
|
+
#
|
|
13
|
+
# Hop by hop:
|
|
14
|
+
#
|
|
15
|
+
# stations the listing, asked for zero rows, so its `records` is [].
|
|
16
|
+
# read one item per listed station, and there are none: skipped, with nothing written.
|
|
17
|
+
# summary the join behind it, skipped because `read` was.
|
|
18
|
+
# report rule: all_done, so it runs whatever happened above it; it reads the listing,
|
|
19
|
+
# which is the one thing that did happen, and says how many stations there were.
|
|
20
|
+
#
|
|
21
|
+
# EXPECT THIS RUN TO SUCCEED at once, with `read` and `summary` skipped and `report` saying
|
|
22
|
+
# zero stations.
|
|
23
|
+
#
|
|
24
|
+
# To change it: -p stations=3 gives the listing something to find, and every step runs.
|
|
25
|
+
#
|
|
26
|
+
# dg run --local examples/patterns/fan-out-over-an-empty-listing.yaml
|
|
27
|
+
# dg run --local examples/patterns/fan-out-over-an-empty-listing.yaml -p stations=3
|
|
28
|
+
|
|
29
|
+
format: dirigent/v1
|
|
30
|
+
kind: pipeline
|
|
31
|
+
code: fan-out-over-an-empty-listing
|
|
32
|
+
name: Fan out over an empty listing
|
|
33
|
+
description: |
|
|
34
|
+
A `for_each` over a step's output that turns out to be an empty list skips the step, and
|
|
35
|
+
the skip carries down every `all_success` edge behind it.
|
|
36
|
+
|
|
37
|
+
The run still succeeds: an empty listing is a day with nothing to do.
|
|
38
|
+
|
|
39
|
+
tags: [patterns, transform, fan-out, references, rules]
|
|
40
|
+
|
|
41
|
+
requires:
|
|
42
|
+
blocks:
|
|
43
|
+
- playground.generate
|
|
44
|
+
- transform.jq
|
|
45
|
+
|
|
46
|
+
params:
|
|
47
|
+
type: object
|
|
48
|
+
properties:
|
|
49
|
+
stations:
|
|
50
|
+
type: integer
|
|
51
|
+
default: 0
|
|
52
|
+
minimum: 0
|
|
53
|
+
maximum: 50
|
|
54
|
+
description: How many stations the listing finds; zero is the case this file is about.
|
|
55
|
+
|
|
56
|
+
steps:
|
|
57
|
+
stations:
|
|
58
|
+
block: playground.generate
|
|
59
|
+
config:
|
|
60
|
+
rows: ${params.stations}
|
|
61
|
+
seed: 7
|
|
62
|
+
fields:
|
|
63
|
+
station: city
|
|
64
|
+
|
|
65
|
+
read:
|
|
66
|
+
block: playground.generate
|
|
67
|
+
depends_on: [stations]
|
|
68
|
+
# An empty list here writes no items, and a fan-out with none is skipped.
|
|
69
|
+
for_each: ${steps.stations.output.records}
|
|
70
|
+
config:
|
|
71
|
+
input: ${item}
|
|
72
|
+
fields:
|
|
73
|
+
celsius:
|
|
74
|
+
provider: pyint
|
|
75
|
+
args: {min_value: -20, max_value: 30}
|
|
76
|
+
|
|
77
|
+
summary:
|
|
78
|
+
block: transform.jq
|
|
79
|
+
depends_on: [read]
|
|
80
|
+
config:
|
|
81
|
+
input: ${steps.read.output}
|
|
82
|
+
program: 'length'
|
|
83
|
+
|
|
84
|
+
report:
|
|
85
|
+
block: transform.jq
|
|
86
|
+
depends_on: [summary]
|
|
87
|
+
# all_done runs once `summary` has settled, skipped included, which is what lets the run
|
|
88
|
+
# say what the empty day looked like.
|
|
89
|
+
rule: all_done
|
|
90
|
+
config:
|
|
91
|
+
input: ${steps.stations.output}
|
|
92
|
+
program: '{stations: .rows}'
|