dirigent-examples 0.20.0__tar.gz → 0.22.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/PKG-INFO +3 -3
  2. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/pyproject.toml +3 -3
  3. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/pyproject.toml.orig +3 -3
  4. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/README.md +1 -1
  5. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/failure/README.md +1 -0
  6. dirigent_examples-0.22.0/src/dirigent_examples/shelves/failure/fan-out-too-wide.yaml +94 -0
  7. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/fan-out.yaml +3 -3
  8. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/who-gho-indicators-to-parquet.yaml +3 -4
  9. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/README.md +5 -0
  10. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/fan-out-item-wise.yaml +2 -2
  11. dirigent_examples-0.22.0/src/dirigent_examples/shelves/patterns/fan-out-late-after-continue.yaml +77 -0
  12. dirigent_examples-0.22.0/src/dirigent_examples/shelves/patterns/fan-out-late-item-wise.yaml +94 -0
  13. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/fan-out-literal-list.yaml +4 -6
  14. dirigent_examples-0.22.0/src/dirigent_examples/shelves/patterns/fan-out-over-a-nested-path.yaml +83 -0
  15. dirigent_examples-0.22.0/src/dirigent_examples/shelves/patterns/fan-out-over-an-empty-listing.yaml +92 -0
  16. dirigent_examples-0.22.0/src/dirigent_examples/shelves/patterns/fan-out-over-an-output.yaml +104 -0
  17. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/pipeline-run-with-params.yaml +3 -3
  18. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/python/error_handling.py +1 -1
  19. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/queues/README.md +3 -1
  20. dirigent_examples-0.22.0/src/dirigent_examples/shelves/queues/kafka-watch-then-transform.yaml +88 -0
  21. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/pagination-by-fan-out.yaml +6 -8
  22. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/storage-manifest-of-a-fan-out.yaml +3 -4
  23. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/jq-reshape.yaml +2 -2
  24. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/std-convert-fan-out.yaml +2 -2
  25. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/README.md +7 -5
  26. dirigent_examples-0.22.0/src/dirigent_examples/shelves/triggers/watch-a-sensor.yaml +85 -0
  27. dirigent_examples-0.22.0/src/dirigent_examples/shelves/triggers/watch-fan-out-per-message.yaml +86 -0
  28. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/LICENSE +0 -0
  29. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/README.md +0 -0
  30. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/__init__.py +0 -0
  31. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/py.typed +0 -0
  32. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/composition/README.md +0 -0
  33. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/composition/chained-instances.yaml +0 -0
  34. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/composition/composition-child.yaml +0 -0
  35. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/composition/composition-parent.yaml +0 -0
  36. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/connections.yaml +0 -0
  37. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/demo/README.md +0 -0
  38. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/demo/markdown-showcase.yaml +0 -0
  39. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/demo/params-showcase.yaml +0 -0
  40. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/demo/requires.yaml +0 -0
  41. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/demo/weekly-import-malawi.yaml +0 -0
  42. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/demo/weekly-import-nepal.yaml +0 -0
  43. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/README.md +0 -0
  44. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-build-push.yaml +0 -0
  45. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-build-run.yaml +0 -0
  46. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-compose-database.yaml +0 -0
  47. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-compose-failing-up.yaml +0 -0
  48. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-compose-file.yaml +0 -0
  49. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-compose-profiles-env.yaml +0 -0
  50. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-compose-stack.yaml +0 -0
  51. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-hello.yaml +0 -0
  52. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-remote-daemon.yaml +0 -0
  53. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-run-failing-teardown.yaml +0 -0
  54. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/docker/docker-ticker.yaml +0 -0
  55. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/execute/README.md +0 -0
  56. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/execute/long-log.yaml +0 -0
  57. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/failure/error-handler.yaml +0 -0
  58. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/failure/optional-step.yaml +0 -0
  59. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/failure/retries.yaml +0 -0
  60. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/failure/retry-budget.yaml +0 -0
  61. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/failure/step-timeout.yaml +0 -0
  62. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/git/README.md +0 -0
  63. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/git/git-checkout-build.yaml +0 -0
  64. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/git/git-checkout-compose.yaml +0 -0
  65. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/git/git-checkout-public.yaml +0 -0
  66. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/README.md +0 -0
  67. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/deep-chain.yaml +0 -0
  68. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/fan-in.yaml +0 -0
  69. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/linear.yaml +0 -0
  70. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/parallel-branches.yaml +0 -0
  71. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/parallel-sleep.yaml +0 -0
  72. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/skip-diamond.yaml +0 -0
  73. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/graph/wide-fan.yaml +0 -0
  74. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/hello-world.yaml +0 -0
  75. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/README.md +0 -0
  76. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/feeds-composition.yaml +0 -0
  77. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/gdacs-disaster-updates.yaml +0 -0
  78. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/github-releases-relay.yaml +0 -0
  79. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/hdx-dataset-watch.yaml +0 -0
  80. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/kobo-submissions-to-csv.yaml +0 -0
  81. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/nominatim-geocode-facilities.yaml +0 -0
  82. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/odk-central-submissions.yaml +0 -0
  83. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/open-meteo-weekly-report.yaml +0 -0
  84. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/overpass-health-facilities.yaml +0 -0
  85. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/usgs-earthquakes-alert.yaml +0 -0
  86. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/wikidata-country-reference.yaml +0 -0
  87. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/open-data/world-bank-population-trend.yaml +0 -0
  88. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/concurrency-queue.yaml +0 -0
  89. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/concurrency-replace.yaml +0 -0
  90. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/concurrency-skip.yaml +0 -0
  91. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/connections-referenced-vs-carried.yaml +0 -0
  92. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/deadline-on-a-sensor.yaml +0 -0
  93. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/fan-out-continue.yaml +0 -0
  94. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/fan-out-fail-fast.yaml +0 -0
  95. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/fan-out-from-params.yaml +0 -0
  96. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/fan-out-nested-objects.yaml +0 -0
  97. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/fan-out-then-join.yaml +0 -0
  98. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/importance-critical.yaml +0 -0
  99. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/log-levels.yaml +0 -0
  100. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/outputs-inline-vs-storage.yaml +0 -0
  101. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/params-every-type.yaml +0 -0
  102. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/params-validation-refuses.yaml +0 -0
  103. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/pipeline-run-child.yaml +0 -0
  104. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/pipeline-run-fire-and-forget.yaml +0 -0
  105. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/pipeline-run-strict.yaml +0 -0
  106. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/pipeline-run-wait.yaml +0 -0
  107. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/poll-cadence.yaml +0 -0
  108. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/priority-layered.yaml +0 -0
  109. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/references-cheat-sheet.yaml +0 -0
  110. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/retry-budget-exhausted.yaml +0 -0
  111. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/retry-exponential-backoff.yaml +0 -0
  112. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/retry-only-transient.yaml +0 -0
  113. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/retry-with-jitter.yaml +0 -0
  114. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/rule-all-done.yaml +0 -0
  115. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/rule-all-success.yaml +0 -0
  116. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/rule-always.yaml +0 -0
  117. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/rule-one-failed.yaml +0 -0
  118. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/schedule-at-once.yaml +0 -0
  119. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/schedule-cron-timezone.yaml +0 -0
  120. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/schedule-interval.yaml +0 -0
  121. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/schedule-window-half-open.yaml +0 -0
  122. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/sensor-http-ready.yaml +0 -0
  123. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/sensor-storage-exists.yaml +0 -0
  124. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/step-names-and-keys.yaml +0 -0
  125. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/timeout-fails-the-step.yaml +0 -0
  126. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/timeout-skips-the-step.yaml +0 -0
  127. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/webhook-mapping-nested-payload.yaml +0 -0
  128. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/patterns/webhook-signed.yaml +0 -0
  129. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/README.md +0 -0
  130. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/a-gate-catching-drift.yaml +0 -0
  131. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/a-page-at-a-time.yaml +0 -0
  132. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/a-stream-to-read.yaml +0 -0
  133. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/generated-rows.yaml +0 -0
  134. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/in-the-middle-of-a-flow.yaml +0 -0
  135. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/retry-that-really-retries.yaml +0 -0
  136. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/playground/waiting-for-a-batch.yaml +0 -0
  137. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/preview/s3-parquet-to-ingestion.yaml +0 -0
  138. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/python/README.md +0 -0
  139. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/python/apply_and_run.py +0 -0
  140. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/python/ci_gate.py +0 -0
  141. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/python/connections.py +0 -0
  142. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/python/follow_logs.py +0 -0
  143. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/python/list_and_filter.py +0 -0
  144. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/queues/kafka-consume-then-transform.yaml +0 -0
  145. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/queues/kafka-produce-then-consume.yaml +0 -0
  146. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/queues/rabbitmq-consume-ack-on-success.yaml +0 -0
  147. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/queues/report-to-kafka.yaml +0 -0
  148. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/queues/report-to-rabbitmq.yaml +0 -0
  149. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/README.md +0 -0
  150. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/csv-header-rules.yaml +0 -0
  151. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/csv-to-ndjson.yaml +0 -0
  152. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/etl-csv-clean-validate-parquet.yaml +0 -0
  153. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/filter-by-predicate.yaml +0 -0
  154. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/filter-then-map-then-reduce.yaml +0 -0
  155. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-fetch-validate-post.yaml +0 -0
  156. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-follow-redirects.yaml +0 -0
  157. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-get-with-query.yaml +0 -0
  158. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-headers-and-auth-connection.yaml +0 -0
  159. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-post-file-from-storage.yaml +0 -0
  160. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-post-json-body.yaml +0 -0
  161. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-post-report.yaml +0 -0
  162. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-save-body-to-storage.yaml +0 -0
  163. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-success-status-list.yaml +0 -0
  164. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/http-timeout-override.yaml +0 -0
  165. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-dedupe-by-key.yaml +0 -0
  166. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-defaults-and-nulls.yaml +0 -0
  167. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-group-by-and-sum.yaml +0 -0
  168. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-join-two-lists.yaml +0 -0
  169. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-long-to-wide.yaml +0 -0
  170. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-nested-to-flat.yaml +0 -0
  171. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-pivot-wide-to-long.yaml +0 -0
  172. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-running-totals.yaml +0 -0
  173. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-string-cleaning.yaml +0 -0
  174. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-top-n.yaml +0 -0
  175. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-validate-in-jq-vs-schema.yaml +0 -0
  176. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/jq-window-dates.yaml +0 -0
  177. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/json-to-csv-flattening.yaml +0 -0
  178. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/large-output-to-storage.yaml +0 -0
  179. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/map-enrich-with-lookup.yaml +0 -0
  180. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/ndjson-to-parquet.yaml +0 -0
  181. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/parquet-round-trip-types.yaml +0 -0
  182. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/reconcile-two-sources.yaml +0 -0
  183. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/report-built-in.yaml +0 -0
  184. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/report-daily-digest.yaml +0 -0
  185. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/report-to-file.yaml +0 -0
  186. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/report-to-webhook.yaml +0 -0
  187. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/schema-carried.yaml +0 -0
  188. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/schema-formats.yaml +0 -0
  189. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/schema-referenced.yaml +0 -0
  190. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/schema-refuses-then-rule.yaml +0 -0
  191. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/storage-copy-dated-archive.yaml +0 -0
  192. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/storage-exists-gate.yaml +0 -0
  193. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/storage-keep-past-the-run.yaml +0 -0
  194. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/storage-write-then-read.yaml +0 -0
  195. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/webhook-post-hmac.yaml +0 -0
  196. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/recipes/webhook-post-summary.yaml +0 -0
  197. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/s3/README.md +0 -0
  198. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/s3/report-to-s3.yaml +0 -0
  199. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/s3/s3-copy-and-verify.yaml +0 -0
  200. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/s3/s3-csv-report.yaml +0 -0
  201. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/s3/s3-parquet-report.yaml +0 -0
  202. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/s3/s3-round-trip.yaml +0 -0
  203. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/schemas/README.md +0 -0
  204. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/schemas/ou-record.json +0 -0
  205. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/schemas/playground-reading.json +0 -0
  206. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/schemas/station-reading.json +0 -0
  207. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sensors/README.md +0 -0
  208. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sensors/sensor-gate.yaml +0 -0
  209. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sensors/time-window.yaml +0 -0
  210. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/showcase/README.md +0 -0
  211. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/showcase/morning-briefing.yaml +0 -0
  212. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/showcase/nightly-regional-load.yaml +0 -0
  213. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/showcase/one-region-refuses.yaml +0 -0
  214. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sql/README.md +0 -0
  215. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sql/duckdb-parquet-to-report.yaml +0 -0
  216. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sql/sql-postgres-readonly.yaml +0 -0
  217. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sql/sql-query-to-storage.yaml +0 -0
  218. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sql/sql-sqlite-roundtrip.yaml +0 -0
  219. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/sql/warehouse.sql +0 -0
  220. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/README.md +0 -0
  221. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/csv-report.yaml +0 -0
  222. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/jq-filter-and-map.yaml +0 -0
  223. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/jq-group-and-aggregate.yaml +0 -0
  224. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/jq-join-two-sources.yaml +0 -0
  225. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/jq-stream-through-storage.yaml +0 -0
  226. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/ndjson-round-trip.yaml +0 -0
  227. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/parquet-round-trip.yaml +0 -0
  228. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/xml-feed-to-ndjson.yaml +0 -0
  229. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/transform/yaml-config-to-json.yaml +0 -0
  230. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/at-one-time.yaml +0 -0
  231. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/cron-nightly.yaml +0 -0
  232. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/cron-windowed.yaml +0 -0
  233. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/document-nightly.yaml +0 -0
  234. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/interval-rolling.yaml +0 -0
  235. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/managed-and-manual.yaml +0 -0
  236. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/triggers/webhook-trigger.yaml +0 -0
  237. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/validate/README.md +0 -0
  238. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/validate/expects-a-shape.yaml +0 -0
  239. {dirigent_examples-0.20.0 → dirigent_examples-0.22.0}/src/dirigent_examples/shelves/validate/the-shape-is-wrong.yaml +0 -0
@@ -1,13 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dirigent-examples
3
- Version: 0.20.0
3
+ Version: 0.22.0
4
4
  Summary: The dirigent example corpus: the shelves of runnable documents, installed.
5
5
  License-Expression: LicenseRef-Proprietary
6
6
  License-File: LICENSE
7
7
  Classifier: Programming Language :: Python :: 3
8
8
  Classifier: Programming Language :: Python :: 3.13
9
- Requires-Dist: dirigent-client==0.20.0
10
- Requires-Dist: dirigent-plugin==0.20.0
9
+ Requires-Dist: dirigent-client==0.22.0
10
+ Requires-Dist: dirigent-plugin==0.22.0
11
11
  Requires-Python: >=3.13
12
12
  Description-Content-Type: text/markdown
13
13
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "dirigent-examples"
3
- version = "0.20.0"
3
+ version = "0.22.0"
4
4
  description = "The dirigent example corpus: the shelves of runnable documents, installed."
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.13"
@@ -11,8 +11,8 @@ classifiers = [
11
11
  "Programming Language :: Python :: 3.13",
12
12
  ]
13
13
  dependencies = [
14
- "dirigent-client==0.20.0",
15
- "dirigent-plugin==0.20.0",
14
+ "dirigent-client==0.22.0",
15
+ "dirigent-plugin==0.22.0",
16
16
  ]
17
17
 
18
18
  [project.entry-points."dirigent.plugins.v1"]
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "dirigent-examples"
3
- version = "0.20.0"
3
+ version = "0.22.0"
4
4
  description = "The dirigent example corpus: the shelves of runnable documents, installed."
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.13"
@@ -12,8 +12,8 @@ classifiers = [
12
12
  ]
13
13
  dependencies = [
14
14
  # The python/ shelf is SDK scripts a reader runs, so the corpus carries the client.
15
- "dirigent-client==0.20.0",
16
- "dirigent-plugin==0.20.0",
15
+ "dirigent-client==0.22.0",
16
+ "dirigent-plugin==0.22.0",
17
17
  ]
18
18
 
19
19
  [project.entry-points."dirigent.plugins.v1"]
@@ -18,7 +18,7 @@ The documents live on topic shelves, each with its own README:
18
18
  | [`failure/`](failure) | Retries, budgets, timeouts, tolerated failures, and the cleanup edge. |
19
19
  | [`transform/`](transform) | The reshaping verbs: jq programs and the `convert.std` codec, no allowlist anywhere. |
20
20
  | [`recipes/`](recipes) | One question per file: how to group, join, pivot, clean, convert, check, store, and call. |
21
- | [`triggers/`](triggers) | What starts a run on its own: the three clocks, and the inbound webhook. |
21
+ | [`triggers/`](triggers) | What starts a run on its own: the three clocks, the inbound webhook, and a watch on a sensor. |
22
22
  | [`sensors/`](sensors) | Steps that wait for the world: a drop landing, a clock window opening. |
23
23
  | [`queues/`](queues) | A run started by a message: a Kafka topic and a RabbitMQ queue, waited on as sensors. |
24
24
  | [`execute/`](execute) | Code on the worker: a shell step behind the allowlist. |
@@ -18,3 +18,4 @@ dg run --local examples/failure/retries.yaml
18
18
  | [step-timeout.yaml](step-timeout.yaml) | A timeout ends the attempt, and the policy decides whether another one starts. |
19
19
  | [optional-step.yaml](optional-step.yaml) | `completed_with_errors`: a step that may fail without failing the run, and the third status that says so. |
20
20
  | [error-handler.yaml](error-handler.yaml) | The cleanup edge: alert on failure, and always release the lock, however the load went. |
21
+ | [fan-out-too-wide.yaml](fan-out-too-wide.yaml) | `fan_out_max_items`: a listing wider than one grid may be fails the fan-out as rejected, before any item is written. |
@@ -0,0 +1,94 @@
1
+ # A listing wider than the instance lets one grid be, refused before any item runs.
2
+ #
3
+ # Every fan-out grid is bounded by the instance setting fan_out_max_items, 10000 unless an
4
+ # operator says otherwise. A grid fixed when the run is created is checked then, and a run
5
+ # over the bound is refused outright. A grid read from a step's output can only be checked
6
+ # when that output exists, so it is checked there: the step that would have fanned out fails
7
+ # as rejected with run.fan_out_too_wide, naming how many items it would have been and the
8
+ # bound, and nothing is written for any of them. rejected is never retried, because the same
9
+ # list would be refused the same way.
10
+ #
11
+ # The bound is there because a grid is rows: every item is a run item and an attempt, written
12
+ # in one transaction, and a listing that answers with a million entries by mistake should cost
13
+ # one refused step rather than a million rows.
14
+ #
15
+ # Hop by hop:
16
+ #
17
+ # pages eleven pages of a listing, one item per page, each a thousand generated rows.
18
+ # flatten the join: one list of every row, 11000 of them.
19
+ # each_row a fan-out over that list. 11000 is over the 10000 the instance allows, so the
20
+ # step fails as rejected and nothing runs.
21
+ # notify rule: one_failed, the error branch, which reads what went wrong.
22
+ #
23
+ # EXPECT THIS RUN TO END failed, in a few seconds, with `each_row` failed as
24
+ # run.fan_out_too_wide and `notify` run.
25
+ #
26
+ # To change it: -p pages='[1,2,3,4,5]' lists 5000 rows, under the bound, and the fan-out
27
+ # runs all 5000 of them, which takes a while.
28
+ # DIRIGENT_FAN_OUT_MAX_ITEMS=100 lowers the bound for one instance instead, and even one page
29
+ # is refused.
30
+ #
31
+ # dg run --local examples/failure/fan-out-too-wide.yaml
32
+ # DIRIGENT_FAN_OUT_MAX_ITEMS=100 dg run --local examples/failure/fan-out-too-wide.yaml -p pages='[1]'
33
+
34
+ format: dirigent/v1
35
+ kind: pipeline
36
+ code: fan-out-too-wide
37
+ name: A fan-out too wide to write
38
+ description: |
39
+ A `for_each` over a step's output that lists more than `fan_out_max_items` fails as
40
+ rejected with `run.fan_out_too_wide`, before a single item is written.
41
+
42
+ The error branch behind it still runs, and the run ends failed.
43
+
44
+ tags: [failure, transform, fan-out, rules]
45
+
46
+ requires:
47
+ blocks:
48
+ - playground.generate
49
+ - transform.jq
50
+
51
+ params:
52
+ type: object
53
+ properties:
54
+ pages:
55
+ type: array
56
+ default: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
57
+ minItems: 1
58
+ maxItems: 20
59
+ items:
60
+ type: integer
61
+ description: The thousand-row pages the listing has; eleven is over the default bound.
62
+
63
+ steps:
64
+ pages:
65
+ block: playground.generate
66
+ for_each: ${params.pages}
67
+ config:
68
+ rows: 1000
69
+ seed: 1
70
+ fields:
71
+ code: uuid4
72
+
73
+ flatten:
74
+ block: transform.jq
75
+ depends_on: [pages]
76
+ config:
77
+ input: ${steps.pages.output}
78
+ program: '[.[].records[]]'
79
+
80
+ each_row:
81
+ block: transform.jq
82
+ depends_on: [flatten]
83
+ for_each: ${steps.flatten.output.value}
84
+ config:
85
+ input: ${item}
86
+ program: '.code'
87
+
88
+ notify:
89
+ block: transform.jq
90
+ depends_on: [each_row]
91
+ rule: one_failed
92
+ config:
93
+ input: {failed: each_row}
94
+ program: '"\(.failed) was refused: the listing was wider than the instance allows"'
@@ -1,8 +1,8 @@
1
1
  # Fan-out: one step mapped over a list, with one run item per element.
2
2
  #
3
- # for_each is expanded when the run is created, so the item grid exists from the moment the
4
- # run is visible. It may read params, run, and item -- but not another step's output,
5
- # because the cardinality has to be known before anything executes.
3
+ # This for_each reads params, so it is expanded when the run is created and the item grid
4
+ # exists from the moment the run is visible. One that reads another step's output is written
5
+ # when that step has settled instead; patterns/fan-out-over-an-output.yaml shows it.
6
6
  #
7
7
  # items: continue is the item policy: a region that fails does not stop the others, its
8
8
  # failure is recorded against its own run item, and the run reports completed_with_errors
@@ -6,10 +6,9 @@
6
6
  # in Dim1, and the number in NumericValue. The envelope is OData's; the rows inside it are what
7
7
  # anyone actually wants.
8
8
  #
9
- # What this shows, beyond the fetch: how a fan-out is read back. `for_each` is expanded when the
10
- # run is created, so it can read params, run and item -- and never another step's output. And a
11
- # fan-out step stores one output: the list of its items' outputs, in item order, with a failed
12
- # item absent from it. So there is no way for item three of one step to ask item three of another
9
+ # What this shows, beyond the fetch: how a fan-out is read back. `for_each` here reads params,
10
+ # so the grid is fixed when the run is created. And a fan-out step stores one output: the list
11
+ # of its items' outputs, in item order, with a failed item absent from it. So there is no way for item three of one step to ask item three of another
13
12
  # what it produced; what a downstream step gets is the whole list at once. That is why the
14
13
  # flattening below is a single step over every envelope, and why the country is read out of
15
14
  # SpatialDim: the join key lives in the data, where a fan-in can see it.
@@ -65,6 +65,11 @@ One step definition, many run items.
65
65
  | [fan-out-then-join.yaml](fan-out-then-join.yaml) | The join, which is just a step with no `for_each` of its own | `succeeded` |
66
66
  | [fan-out-nested-objects.yaml](fan-out-nested-objects.yaml) | Elements that are objects, and `${item.limits.max_ms}` | `succeeded` |
67
67
  | [fan-out-item-wise.yaml](fan-out-item-wise.yaml) | A second fan-out over the same grid, and the item it pairs with | `succeeded` |
68
+ | [fan-out-over-an-output.yaml](fan-out-over-an-output.yaml) | List, then fan out over what was listed: a grid written when its source succeeds | `succeeded` |
69
+ | [fan-out-over-an-empty-listing.yaml](fan-out-over-an-empty-listing.yaml) | A listing that found nothing: the fan-out over it is skipped, and so is what needed it | `succeeded` |
70
+ | [fan-out-over-a-nested-path.yaml](fan-out-over-a-nested-path.yaml) | A list four fields inside an output, reached with the dotted path | `succeeded` |
71
+ | [fan-out-late-item-wise.yaml](fan-out-late-item-wise.yaml) | Adopting a listed grid, and `${steps.<name>.item.output}` over it | `succeeded` |
72
+ | [fan-out-late-after-continue.yaml](fan-out-late-after-continue.yaml) | A late grid over a fan-out under `items: continue`: only the survivors | `completed_with_errors` |
68
73
 
69
74
  ## Parameters
70
75
 
@@ -12,8 +12,8 @@
12
12
  #
13
13
  # Three things the pairing buys, and one it costs:
14
14
  #
15
- # 1. Cardinality is still fixed when the run is created. The grid comes from a step that was
16
- # itself expanded then, so nothing here waits on execution to know how wide it is.
15
+ # 1. The width is the adopted grid's. Here that grid comes from params, so it was fixed when
16
+ # the run was created; adopting a grid read from a step's output expands when that one does.
17
17
  # 2. The names must be a direct dependency: the adopted step goes in depends_on. Pairing
18
18
  # across a step that is not waited for would read an item that has not run.
19
19
  # 3. Adoption chains. A third step may map over the second's items and still reach the
@@ -0,0 +1,77 @@
1
+ # A fan-out whose failures were tolerated, feeding a fan-out over what it did produce.
2
+ #
3
+ # A fan-out step's output is the list of its items' outputs, and only the items that
4
+ # succeeded are in it. So a late grid read from a fan-out under items: continue is exactly as
5
+ # wide as the part of the batch that worked: the failed item is not a hole in the list, it is
6
+ # absent from it, and the step after it never makes an item for it.
7
+ #
8
+ # The source has to be allowed to finish for this to happen. Under the default fail_fast one
9
+ # failed item fails the whole step, and a fan-out behind it under all_success is skipped,
10
+ # with nothing expanded. Under continue the source succeeds while one item did, and the run
11
+ # ends completed_with_errors to say that something was tolerated.
12
+ #
13
+ # Hop by hop:
14
+ #
15
+ # probe one item per station. Each element says how many attempts its station fails;
16
+ # lighthouse fails one, and the default policy gives a step a single attempt, so
17
+ # that item fails and the other two succeed.
18
+ # publish a fan-out over probe's output. Two stations succeeded, so it has two items;
19
+ # the one that failed is not in the list, so there is no item for it at all.
20
+ # tally the join: how many were published, which is two.
21
+ #
22
+ # EXPECT THIS RUN TO END completed_with_errors, with one failed item under `probe`, two items
23
+ # under `publish`, and a tally of two.
24
+ #
25
+ # To change it: set lighthouse's fail to 0 and every station gets through, `publish` has
26
+ # three items, and the run succeeds.
27
+ #
28
+ # dg run --local examples/patterns/fan-out-late-after-continue.yaml
29
+
30
+ format: dirigent/v1
31
+ kind: pipeline
32
+ code: fan-out-late-after-continue
33
+ name: A late grid over what survived
34
+ description: |
35
+ A late grid read from a fan-out under `items: continue` has one item per item that
36
+ succeeded; a failed item is absent from the list, so nothing is made for it.
37
+
38
+ The run ends completed_with_errors, which says a failure was tolerated.
39
+
40
+ tags: [patterns, transform, fan-out, failure, references]
41
+
42
+ requires:
43
+ blocks:
44
+ - playground.generate
45
+ - transform.jq
46
+
47
+ steps:
48
+ probe:
49
+ block: playground.generate
50
+ for_each:
51
+ - {station: harbour, fail: 0}
52
+ - {station: lighthouse, fail: 1}
53
+ - {station: pier, fail: 0}
54
+ # One refusal does not stop the other stations, and the step still succeeds.
55
+ items: continue
56
+ config:
57
+ input: {station: "${item.station}"}
58
+ # playground.generate fails this many attempts before it succeeds, and rejected is a
59
+ # class no retry is spent on.
60
+ fail_until: ${item.fail}
61
+ fail_as: rejected
62
+
63
+ publish:
64
+ block: transform.jq
65
+ depends_on: [probe]
66
+ # probe's output is only its succeeded items' outputs, so this grid is the survivors.
67
+ for_each: ${steps.probe.output}
68
+ config:
69
+ input: ${item.records}
70
+ program: '.[0].station'
71
+
72
+ tally:
73
+ block: transform.jq
74
+ depends_on: [publish]
75
+ config:
76
+ input: ${steps.publish.output}
77
+ program: '{published: length, stations: [.[].value]}'
@@ -0,0 +1,94 @@
1
+ # Two fan-outs over one listed grid: item three of the second reads item three of the first.
2
+ #
3
+ # fan-out-item-wise.yaml pairs two fan-outs over a grid that was fixed when the run was
4
+ # created. This is the same pairing over a grid nobody knew the width of until a step had
5
+ # listed it. `measure` fans out over what `stations` listed, and `label` adopts that grid with
6
+ # for_each: ${steps.measure.items}, so each of its items reads its own match with
7
+ # ${steps.measure.item.output} rather than the whole batch.
8
+ #
9
+ # An adopter of a late grid is late too. The run is created with both `measure` and `label`
10
+ # waiting as one pending attempt each. `measure` expands when `stations` succeeds; `label`
11
+ # expands when `measure` has settled, into the same number of items at the same indices, so
12
+ # the pairing holds by position exactly as it does over a fixed grid.
13
+ #
14
+ # Hop by hop:
15
+ #
16
+ # stations the listing: three seeded stations.
17
+ # measure one item per listed station, each adding a generated reading to its record.
18
+ # label adopts measure's grid. ${item} is the same station measure had at that index,
19
+ # and ${steps.measure.item.output.records} is that item's own output -- one
20
+ # record, the station with its reading -- not the list of all three.
21
+ # board the join: one line per station, in listing order.
22
+ #
23
+ # EXPECT THIS RUN TO SUCCEED at once, with three items under `measure`, three under `label`
24
+ # at the same indices, and a board of three lines.
25
+ #
26
+ # To change it: -p stations=6 widens both grids together; they always match.
27
+ #
28
+ # dg run --local examples/patterns/fan-out-late-item-wise.yaml
29
+ # dg run --local examples/patterns/fan-out-late-item-wise.yaml -p stations=6
30
+
31
+ format: dirigent/v1
32
+ kind: pipeline
33
+ code: fan-out-late-item-wise
34
+ name: One listed item to one item
35
+ description: |
36
+ A step adopting a grid that was expanded from an output, `for_each: ${steps.<name>.items}`,
37
+ expands with the same indices once that grid has settled.
38
+
39
+ Each of its items reads its match with `${steps.<name>.item.output}`.
40
+
41
+ tags: [patterns, transform, fan-out, references]
42
+
43
+ requires:
44
+ blocks:
45
+ - playground.generate
46
+ - transform.jq
47
+
48
+ params:
49
+ type: object
50
+ properties:
51
+ stations:
52
+ type: integer
53
+ default: 3
54
+ minimum: 1
55
+ maximum: 50
56
+ description: How many stations the listing finds, which is how wide both grids become.
57
+
58
+ steps:
59
+ stations:
60
+ block: playground.generate
61
+ config:
62
+ rows: ${params.stations}
63
+ seed: 7
64
+ fields:
65
+ station: city
66
+
67
+ measure:
68
+ block: playground.generate
69
+ depends_on: [stations]
70
+ for_each: ${steps.stations.output.records}
71
+ config:
72
+ input: ${item}
73
+ fields:
74
+ celsius:
75
+ provider: pyint
76
+ args: {min_value: -20, max_value: 30}
77
+
78
+ label:
79
+ block: transform.jq
80
+ # The adopted step goes in depends_on: this grid is measure's, and it exists once measure
81
+ # has settled.
82
+ depends_on: [measure]
83
+ for_each: ${steps.measure.items}
84
+ config:
85
+ # The matching item's own output: one station, with its reading.
86
+ input: ${steps.measure.item.output.records}
87
+ program: '.[0] | "\(.station): \(.celsius) C"'
88
+
89
+ board:
90
+ block: transform.jq
91
+ depends_on: [label]
92
+ config:
93
+ input: ${steps.label.output}
94
+ program: '[.[].value]'
@@ -7,12 +7,10 @@
7
7
  # ordinary run.
8
8
  #
9
9
  # What a fan-out actually is: one step definition, N run items, one attempt each, all of them
10
- # claimable at once. The item grid exists from the moment the run is visible, because
11
- # for_each is expanded when the run is CREATED. That has a consequence worth knowing before
12
- # you reach for it: for_each may read params, run, and another fan-out's grid as
13
- # ${steps.<step>.items}, but never a step's output, because the cardinality has to be known
14
- # before anything executes. A fan-out over a list fetched at run time is not expressible, and
15
- # that is deliberate rather than missing.
10
+ # claimable at once. A literal list is expanded when the run is CREATED, so the item grid exists
11
+ # from the moment the run is visible. A list fetched at run time is the other shape: a for_each
12
+ # over ${steps.<step>.output.<field>} is written when that step has settled, and
13
+ # fan-out-over-an-output.yaml is that on its own.
16
14
  #
17
15
  # Hop by hop:
18
16
  #
@@ -0,0 +1,83 @@
1
+ # A fan-out over a list that sits deep inside another step's output.
2
+ #
3
+ # A real answer is rarely a bare list. It is an envelope -- a catalogue, a page of results, a
4
+ # network with regions inside it -- and the list worth fanning out over is a few fields down.
5
+ # `for_each` walks there with the same dotted path every reference uses: the output, then
6
+ # each field in turn, until it reaches the list. Nothing has to reshape the envelope first.
7
+ #
8
+ # The path is read when `stations` has succeeded, so it is checked then too. A field the
9
+ # output does not have fails `per_station` as rejected with reference.no_field, naming what
10
+ # was there instead; a field that holds something other than a list fails it with
11
+ # run.fan_out_not_a_list. Either way nothing is written and nothing runs.
12
+ #
13
+ # Hop by hop:
14
+ #
15
+ # catalogue a fixed envelope standing in for an API's answer: a network, its regions,
16
+ # and the stations of each region.
17
+ # per_station one item per station of the northern region, read from four fields down.
18
+ # ${item} is one station object, so ${item.code} is its code.
19
+ # collected the join: every station code, in the order the catalogue listed them.
20
+ #
21
+ # EXPECT THIS RUN TO SUCCEED at once, with three items under `per_station` and a join naming
22
+ # st-101, st-102 and st-103.
23
+ #
24
+ # To change it: write `south` for `north` in the path and the fan-out is the southern
25
+ # region's two stations; write `east`, which the catalogue does not have, and `per_station`
26
+ # fails as rejected. A reference cannot hold another reference, so the path is written out.
27
+ #
28
+ # dg run --local examples/patterns/fan-out-over-a-nested-path.yaml
29
+
30
+ format: dirigent/v1
31
+ kind: pipeline
32
+ code: fan-out-over-a-nested-path
33
+ name: Fan out over a nested path
34
+ description: |
35
+ `for_each: ${steps.<name>.output.<field>.<field>...}` reaches a list several fields inside
36
+ an upstream step's output, so an envelope needs no reshaping before it is fanned out over.
37
+
38
+ A missing field fails the fan-out as rejected, with nothing run.
39
+
40
+ tags: [patterns, transform, fan-out, references]
41
+
42
+ requires:
43
+ blocks:
44
+ - value.const
45
+ - playground.generate
46
+ - transform.jq
47
+
48
+ steps:
49
+ catalogue:
50
+ block: value.const
51
+ config:
52
+ value:
53
+ network:
54
+ name: Coastal network
55
+ regions:
56
+ north:
57
+ stations:
58
+ - {code: st-101, name: Harbour}
59
+ - {code: st-102, name: Lighthouse}
60
+ - {code: st-103, name: Pier}
61
+ south:
62
+ stations:
63
+ - {code: st-201, name: Dunes}
64
+ - {code: st-202, name: Estuary}
65
+
66
+ per_station:
67
+ block: playground.generate
68
+ depends_on: [catalogue]
69
+ # value.const answers under `value`, and the list is four fields inside that.
70
+ for_each: ${steps.catalogue.output.value.network.regions.north.stations}
71
+ config:
72
+ input: ${item}
73
+ fields:
74
+ celsius:
75
+ provider: pyint
76
+ args: {min_value: -5, max_value: 25}
77
+
78
+ collected:
79
+ block: transform.jq
80
+ depends_on: [per_station]
81
+ config:
82
+ input: ${steps.per_station.output}
83
+ program: '[.[].records[0].code]'
@@ -0,0 +1,92 @@
1
+ # A listing that came back empty, and the fan-out over it that therefore never runs.
2
+ #
3
+ # fan-out-over-an-output.yaml fans out over whatever a step listed. This is the same shape on
4
+ # the day the listing finds nothing: `stations` succeeds with no records, so `read` has no
5
+ # items to write. A late grid follows the rule every grid follows -- an empty list skips the
6
+ # step -- and a skip is a real terminal outcome, so everything behind it under the default
7
+ # all_success edge is skipped too.
8
+ #
9
+ # Nothing here failed. An empty listing is a quiet day, not an error, and the run ends
10
+ # succeeded. `read` is skipped with a reason: its one attempt carries run.fan_out_empty and
11
+ # says which step left the list empty.
12
+ #
13
+ # Hop by hop:
14
+ #
15
+ # stations the listing, asked for zero rows, so its `records` is [].
16
+ # read one item per listed station, and there are none: skipped, with nothing written.
17
+ # summary the join behind it, skipped because `read` was.
18
+ # report rule: all_done, so it runs whatever happened above it; it reads the listing,
19
+ # which is the one thing that did happen, and says how many stations there were.
20
+ #
21
+ # EXPECT THIS RUN TO SUCCEED at once, with `read` and `summary` skipped and `report` saying
22
+ # zero stations.
23
+ #
24
+ # To change it: -p stations=3 gives the listing something to find, and every step runs.
25
+ #
26
+ # dg run --local examples/patterns/fan-out-over-an-empty-listing.yaml
27
+ # dg run --local examples/patterns/fan-out-over-an-empty-listing.yaml -p stations=3
28
+
29
+ format: dirigent/v1
30
+ kind: pipeline
31
+ code: fan-out-over-an-empty-listing
32
+ name: Fan out over an empty listing
33
+ description: |
34
+ A `for_each` over a step's output that turns out to be an empty list skips the step, and
35
+ the skip carries down every `all_success` edge behind it.
36
+
37
+ The run still succeeds: an empty listing is a day with nothing to do.
38
+
39
+ tags: [patterns, transform, fan-out, references, rules]
40
+
41
+ requires:
42
+ blocks:
43
+ - playground.generate
44
+ - transform.jq
45
+
46
+ params:
47
+ type: object
48
+ properties:
49
+ stations:
50
+ type: integer
51
+ default: 0
52
+ minimum: 0
53
+ maximum: 50
54
+ description: How many stations the listing finds; zero is the case this file is about.
55
+
56
+ steps:
57
+ stations:
58
+ block: playground.generate
59
+ config:
60
+ rows: ${params.stations}
61
+ seed: 7
62
+ fields:
63
+ station: city
64
+
65
+ read:
66
+ block: playground.generate
67
+ depends_on: [stations]
68
+ # An empty list here writes no items, and a fan-out with none is skipped.
69
+ for_each: ${steps.stations.output.records}
70
+ config:
71
+ input: ${item}
72
+ fields:
73
+ celsius:
74
+ provider: pyint
75
+ args: {min_value: -20, max_value: 30}
76
+
77
+ summary:
78
+ block: transform.jq
79
+ depends_on: [read]
80
+ config:
81
+ input: ${steps.read.output}
82
+ program: 'length'
83
+
84
+ report:
85
+ block: transform.jq
86
+ depends_on: [summary]
87
+ # all_done runs once `summary` has settled, skipped included, which is what lets the run
88
+ # say what the empty day looked like.
89
+ rule: all_done
90
+ config:
91
+ input: ${steps.stations.output}
92
+ program: '{stations: .rows}'