dirigent-examples 0.18.4__tar.gz → 0.19.0__tar.gz

{dirigent_examples-0.18.4 → dirigent_examples-0.19.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: dirigent-examples
-Version: 0.18.4
+Version: 0.19.0
 Summary: The dirigent example corpus: the shelves of runnable documents, installed.
 License-Expression: LicenseRef-Proprietary
 License-File: LICENSE
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: dirigent-client==0.18.4
-Requires-Dist: dirigent-plugin==0.18.4
+Requires-Dist: dirigent-client==0.19.0
+Requires-Dist: dirigent-plugin==0.19.0
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 

{dirigent_examples-0.18.4 → dirigent_examples-0.19.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-examples"
-version = "0.18.4"
+version = "0.19.0"
 description = "The dirigent example corpus: the shelves of runnable documents, installed."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -11,8 +11,8 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-    "dirigent-client==0.18.4",
-    "dirigent-plugin==0.18.4",
+    "dirigent-client==0.19.0",
+    "dirigent-plugin==0.19.0",
 ]
 
 [project.entry-points."dirigent.plugins.v1"]

{dirigent_examples-0.18.4 → dirigent_examples-0.19.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-examples"
-version = "0.18.4"
+version = "0.19.0"
 description = "The dirigent example corpus: the shelves of runnable documents, installed."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -12,8 +12,8 @@ classifiers = [
 ]
 dependencies = [
     # The python/ shelf is SDK scripts a reader runs, so the corpus carries the client.
-    "dirigent-client==0.18.4",
-    "dirigent-plugin==0.18.4",
+    "dirigent-client==0.19.0",
+    "dirigent-plugin==0.19.0",
 ]
 
 [project.entry-points."dirigent.plugins.v1"]

{dirigent_examples-0.18.4 → dirigent_examples-0.19.0}/src/dirigent_examples/shelves/README.md

@@ -14,6 +14,7 @@ The documents live on topic shelves, each with its own README:
 | --- | --- |
 | [`graph/`](graph) | The shapes a DAG takes: lines, branches, fans, joins, and skips. |
 | [`patterns/`](patterns) | One engine behaviour per file: rules, retries, the two clocks, item policies, triggers, concurrency, and the reference language. |
+| [`playground/`](playground) | Documents that need something to happen: generated rows, a step that fails twice, a shape that drifts, a page to walk. |
 | [`failure/`](failure) | Retries, budgets, timeouts, tolerated failures, and the cleanup edge. |
 | [`transform/`](transform) | The reshaping verbs: jq programs and the `convert.std` codec, no allowlist anywhere. |
 | [`recipes/`](recipes) | One question per file: how to group, join, pivot, clean, convert, check, store, and call. |

dirigent_examples-0.19.0/src/dirigent_examples/shelves/playground/README.md

@@ -0,0 +1,42 @@
+# Playground examples
+
+Documents that need something to *happen*, not something to call. `playground.generate` is a
+node like any other: it takes an optional `input`, it emits an output, and it works at the
+start of a flow, in the middle of one, or as its last step. It reaches nothing -- no
+connection, no URL, no network, no file -- so a document here runs offline and runs the same
+on every machine.
+
+The knobs are what make it worth having. A fan-out needs a list to walk, a retry needs a step
+that fails twice, a gate needs a shape that drifts, and an outputs-to-storage example needs a
+payload over the threshold. Each of those is a knob rather than a contrivance:
+
+| Knob | What it does |
+| --- | --- |
+| `fields` | Which Faker provider fills each output field. Any provider the installed Faker has. |
+| `rows`, `locale`, `seed` | How many records, in which locale, from which seed. |
+| `page`, `size` | Hand back one page of the records, and say how many pages there are. |
+| `drift` | Emit a deliberately wrong shape: `strings`, `missing` or `extra`. |
+| `payload` | Filler of a chosen size, to push an output past the inline limit. |
+| `delay` | Take that long before answering, for a timeout or a deadline. |
+| `fail_until` | Fail that many attempts before succeeding, for a retry. |
+| `input` | A value from upstream to work from; absent, the knobs generate one. |
+
+**Everything is seeded.** The same seed gives the same records on every run, which is what
+lets a document here assert on what came out. With no seed the node draws one and reports it,
+so a surprising answer can still be reproduced.
+
+**The provider names are Faker's.** A reader cannot guess 280 of them, and a list pasted into
+a page rots. `uv run faker` prints every provider the installed Faker offers with a sample of
+each, and `uv run faker -l no_NO` prints them for one locale.
+
+A document that is teaching HTTP itself -- a request with headers and a query, a status a
+client has to handle, a redirect, a delay against a timeout -- wants a real HTTP hop instead,
+and the instance serves one under `/api/v1/playground`.
+
+| File | What it demonstrates |
+| --- | --- |
+| [generated-rows.yaml](generated-rows.yaml) | The lead case: a first step with no input, a field map, a row count and a seed. |
+| [retry-that-really-retries.yaml](retry-that-really-retries.yaml) | `fail_until` reads the engine's attempt number, so the step really fails twice and really recovers. |
+| [a-gate-catching-drift.yaml](a-gate-catching-drift.yaml) | `drift` makes the shape wrong on demand, so `validate.schema` is seen refusing it. |
+| [a-page-at-a-time.yaml](a-page-at-a-time.yaml) | `page`/`size` give a fan-out a real page to walk, and `payload` pushes the output out to storage. |
+| [in-the-middle-of-a-flow.yaml](in-the-middle-of-a-flow.yaml) | The node with an `input` from upstream, and again as the document's last step. |

dirigent_examples-0.19.0/src/dirigent_examples/shelves/playground/a-gate-catching-drift.yaml

@@ -0,0 +1,98 @@
+# A schema gate seen catching something, because the shape drifts on demand.
+#
+# A gate example is only convincing if the reader sees it refuse. `drift` makes the node emit
+# a shape that is deliberately wrong, in one of three documented ways:
+#
+#   strings   every number and boolean comes back as a string   a typed schema refuses it
+#   missing   the field map's first field is left out           a required-field schema refuses it
+#   extra     a field the map never named is added              a closed schema refuses it
+#
+# The schema is carried in this document's own `schemas:` section, so `dg run --local` runs
+# the file alone. A server refuses a carried schema, and an instance names one instead.
+#
+# Hop by hop:
+#
+#   reading   three records: a station code and a numeric value each, drifted or not.
+#   gate      validate.schema against `reading-records`. Passes the value through when it
+#             fits, and fails the run naming the path when it does not.
+#   report    reads the gate's output, so the graph records that it saw a checked shape.
+#
+# EXPECT THIS RUN TO END completed with the default, and failed with any other drift. Read
+# the gate's error on a failure: it names the path into the payload that is wrong.
+#
+#   dg run --local examples/playground/a-gate-catching-drift.yaml                    # completed
+#   dg run --local examples/playground/a-gate-catching-drift.yaml -p drift=strings   # fails at the gate
+#   dg run --local examples/playground/a-gate-catching-drift.yaml -p drift=missing   # fails at the gate
+
+format: dirigent/v1
+kind: pipeline
+code: a-gate-catching-drift
+name: A gate catching drift
+description: |
+  Generated records against a carried JSON Schema, with a knob that makes the shape wrong.
+
+  The gate passes good records through and refuses drifted ones, naming the path.
+
+tags: [playground, validate, transform, jq, params]
+
+requires:
+  blocks:
+    - playground.generate
+    - validate.schema
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    drift:
+      type: string
+      description: How the generated records depart from the shape the gate expects.
+      default: none
+      enum: [none, strings, missing, extra]
+
+schemas:
+  reading-records:
+    $schema: https://json-schema.org/draft/2020-12/schema
+    title: A page of station readings
+    type: array
+    items:
+      type: object
+      # Closed, so the `extra` drift is refused as well as the other two.
+      additionalProperties: false
+      required: [station, value]
+      properties:
+        station:
+          type: string
+        value:
+          type: number
+
+steps:
+  reading:
+    block: playground.generate
+    config:
+      rows: 3
+      seed: 11
+      drift: ${params.drift}
+      fields:
+        # `station` is the map's first field, so it is the one the `missing` drift leaves out.
+        station: license_plate
+        value:
+          provider: pyfloat
+          args:
+            min_value: 0
+            max_value: 40
+            right_digits: 1
+
+  gate:
+    block: validate.schema
+    depends_on: [reading]
+    config:
+      input: ${steps.reading.output.records}
+      schema: reading-records
+
+  report:
+    block: transform.jq
+    depends_on: [gate]
+    config:
+      input: ${steps.gate.output.value}
+      program: '{checked: length}'

dirigent_examples-0.19.0/src/dirigent_examples/shelves/playground/a-page-at-a-time.yaml

@@ -0,0 +1,90 @@
+# A fan-out with something real to walk, and an output large enough to leave the database.
+#
+# Two things a document usually has to contrive, both knobs here:
+#
+#   page / size   the node holds `rows` records and hands back one page of them, saying how
+#                 many pages there are, so a fan-out walks a page instead of a list somebody
+#                 typed into the file.
+#   payload       filler of a chosen size beside the records. An output at or under
+#                 `inline_artifact_max` (16KB by default) is stored inline in the database,
+#                 and one above it is written to storage and referenced -- so asking for 32KB
+#                 crosses that threshold on purpose rather than by accident.
+#
+# Hop by hop:
+#
+#   page_two   records 11 to 20 of a hundred, and 32KB of filler beside them, which is what
+#              pushes this step's output out to storage.
+#   per_row    one map.jq element per record on the page: a fan-out over a real page.
+#   totals     what the page was, read off the node's own output rather than counted again.
+#
+# EXPECT THIS RUN TO END completed. Read `page_two`'s output in `dg runs show`: `pages: 10`,
+# `page: 2`, ten records, and `payload_bytes: 32768`. The artifact it wrote is in the run's
+# artifact list, because the output was too large to inline.
+#
+# To change it: -p payload=1kb keeps the output inline, so the same document shows both sides
+# of the threshold.
+#
+#   dg run --local examples/playground/a-page-at-a-time.yaml
+#   dg run --local examples/playground/a-page-at-a-time.yaml -p payload=1kb
+
+format: dirigent/v1
+kind: pipeline
+code: a-page-at-a-time
+name: A page at a time, and an output over the threshold
+description: |
+  One page of a generated hundred, walked element by element, with filler that pushes the
+  step's output past the inline limit and out to storage.
+
+tags: [playground, transform, jq, map, fan-out, outputs]
+
+requires:
+  blocks:
+    - playground.generate
+    - map.jq
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    page:
+      type: integer
+      description: Which page of the hundred records to fetch.
+      default: 2
+      minimum: 1
+      maximum: 10
+    payload:
+      type: string
+      description: How much filler to return beside the records; 32kb crosses the inline limit.
+      default: 32kb
+
+steps:
+  page_two:
+    block: playground.generate
+    config:
+      rows: 100
+      size: 10
+      page: ${params.page}
+      seed: 5
+      payload: ${params.payload}
+      fields:
+        who: name
+        company: company
+
+  per_row:
+    block: map.jq
+    depends_on: [page_two]
+    config:
+      input: ${steps.page_two.output.records}
+      program: '{who: .who, at: .company}'
+
+  totals:
+    block: transform.jq
+    depends_on: [page_two, per_row]
+    config:
+      input:
+        page: ${steps.page_two.output.page}
+        pages: ${steps.page_two.output.pages}
+        rows: ${steps.page_two.output.rows}
+        payload_bytes: ${steps.page_two.output.payload_bytes}
+        mapped: ${steps.per_row.output.value}
+      program: '{page: .page, of: .pages, from: .rows, filler_bytes: .payload_bytes, mapped: (.mapped | length)}'

dirigent_examples-0.19.0/src/dirigent_examples/shelves/playground/generated-rows.yaml

@@ -0,0 +1,80 @@
+# A document's first step, with nothing upstream and nothing to call.
+#
+# playground.generate is the node for a document that needs something to happen rather than
+# something to call: no connection, no URL, no network. Its field map names a Faker provider
+# per output field, so `who: name` fills `who` from Faker's `name` provider. Every provider
+# the installed Faker has is reachable; `uv run faker` prints them all with a sample each,
+# and `uv run faker -l no_NO` prints them for one locale.
+#
+# The seed is what makes an example verifiable: the same seed gives the same rows on every
+# run, on every machine. Leave it out and the node draws one, and its output says which --
+# so a surprising answer can still be reproduced by sending that number back.
+#
+# Hop by hop:
+#
+#   people   five rows of three fields, seeded, so the run is reproducible.
+#   shape    a jq program over what came out, to show the rows are ordinary data.
+#
+# EXPECT THIS RUN TO END completed, at once. Read `people`'s output: `records` is the rows,
+# and `seed`, `locale`, `rows` and `fields` beside them are the knobs that produced them.
+#
+# To change it: -p seed=99 for different rows, -p rows=50 for more of them, and
+# -p locale=no_NO for Norwegian names and cities.
+#
+#   dg run --local examples/playground/generated-rows.yaml
+#   dg run --local examples/playground/generated-rows.yaml -p locale=no_NO
+
+format: dirigent/v1
+kind: pipeline
+code: generated-rows
+name: Generated rows, from nothing
+description: |
+  A first step that needs no input: a field map of Faker providers, a row count and a seed.
+
+  The same seed gives the same rows every run, which is what lets an example assert on them.
+
+tags: [playground, transform, jq, params]
+
+requires:
+  blocks:
+    - playground.generate
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    rows:
+      type: integer
+      description: How many records to generate.
+      default: 5
+      minimum: 1
+      maximum: 50
+    seed:
+      type: integer
+      description: The seed the records come from; the same seed gives the same records.
+      default: 42
+    locale:
+      type: string
+      description: The Faker locale the providers generate in.
+      default: en_US
+      enum: [en_US, no_NO, fr_FR, de_DE]
+
+steps:
+  people:
+    block: playground.generate
+    config:
+      rows: ${params.rows}
+      seed: ${params.seed}
+      locale: ${params.locale}
+      # A provider name per field. The field names are ours; the provider names are Faker's.
+      fields:
+        who: name
+        where: city
+        when: date_this_decade
+
+  shape:
+    block: transform.jq
+    depends_on: [people]
+    config:
+      input: ${steps.people.output.records}
+      program: "{count: length, first: .[0].who, cities: [.[].where] | unique | length}"

dirigent_examples-0.19.0/src/dirigent_examples/shelves/playground/in-the-middle-of-a-flow.yaml

@@ -0,0 +1,73 @@
+# The node in the middle of a document, and at the end of it.
+#
+# The node takes an `input` the same way every transform does: a value written inline or
+# referenced from an earlier step. What it does with one is decided by a single rule -- the
+# input is the data, so it says how many records there are and `rows` is not consulted, and
+# the field map adds columns to each element only when the document names one.
+#
+#   an input and no `fields`     the elements pass through unchanged
+#   an input and a `fields` map  each element gains those generated columns
+#   no input at all              the knobs generate the records, as in generated-rows.yaml
+#
+# Hop by hop:
+#
+#   codes      a plain list, standing in for whatever a real upstream step produced.
+#   enriched   the node in the middle: three codes in, three enriched records out, each with
+#              a generated contact on it. `rows: 99` is written on purpose, to show that a
+#              supplied input wins and the output reports `rows: 3`.
+#   audited    the node at the end: it takes the enriched records, adds nothing, and is the
+#              document's last step. A node is useful as a terminus because its output is
+#              the account of what happened -- the seed, the knobs, and the records.
+#
+# EXPECT THIS RUN TO END completed. Read `enriched`'s output: `rows: 3`, `input_rows: 3`, and
+# the three codes with a name and an email added to each.
+#
+#   dg run --local examples/playground/in-the-middle-of-a-flow.yaml
+
+format: dirigent/v1
+kind: pipeline
+code: in-the-middle-of-a-flow
+name: The node in the middle, and at the end
+description: |
+  A list from upstream, enriched with generated columns, then handed to the node again as the
+  document's last step.
+
+  A supplied input decides how many records there are; `rows` is ignored, and the output says
+  so.
+
+tags: [playground, references, conventions]
+
+requires:
+  blocks:
+    - playground.generate
+    - value.const
+
+steps:
+  codes:
+    block: value.const
+    config:
+      value:
+        - code: st-001
+        - code: st-002
+        - code: st-003
+
+  enriched:
+    block: playground.generate
+    depends_on: [codes]
+    config:
+      input: ${steps.codes.output.value}
+      # Ignored, because the input decides how many records there are. The output reports
+      # rows: 3, which is the number that actually came out.
+      rows: 99
+      seed: 3
+      fields:
+        contact: name
+        email: email
+
+  audited:
+    block: playground.generate
+    depends_on: [enriched]
+    config:
+      # No fields named, so the records pass through untouched and this step is an account of
+      # what it received rather than a change to it.
+      input: ${steps.enriched.output.records}

dirigent_examples-0.19.0/src/dirigent_examples/shelves/playground/retry-that-really-retries.yaml

@@ -0,0 +1,80 @@
+# A retry example that proves a retry, instead of staging one.
+#
+# A document teaching `retry:` used to need a remote endpoint that fails on cue, which no
+# public service offers: the usual trick is an endpoint that always fails, so the reader sees
+# a budget spent and never sees a recovery. `fail_until` closes that gap. The node reads the
+# attempt number the engine hands it and fails while that number is inside the window, so the
+# step really does fail twice and really does succeed on the third attempt.
+#
+# The failure is classified `transient`, which is what the engine retries. A `rejected`
+# failure is refused another attempt however much budget is left, which is what
+# ../patterns/retry-only-transient.yaml is about.
+#
+# Hop by hop:
+#
+#   flaky   fails attempts 1 and 2, succeeds on 3, and generates its rows on the way out.
+#   proof   reads the attempt that finally worked, straight off the node's own output.
+#
+# EXPECT THIS RUN TO END completed, in about three seconds: one second of backoff, then two.
+# Read `flaky` in `dg runs show`: three attempts, the last one successful, and `attempt: 3`
+# on the output to say so in the data as well.
+#
+# To change it: -p fail_until=3 with the budget left at 3 and the step runs out of attempts,
+# so the run fails -- which is the other half of what a budget means.
+#
+#   dg run --local examples/playground/retry-that-really-retries.yaml
+#   dg run --local examples/playground/retry-that-really-retries.yaml -p fail_until=3   # fails
+
+format: dirigent/v1
+kind: pipeline
+code: retry-that-really-retries
+name: A retry that really retries
+description: |
+  A step that fails its first two attempts and succeeds on the third, on purpose.
+
+  `fail_until` reads the engine's own attempt number, so the recovery is real rather than
+  described.
+
+tags: [playground, transform, jq, retry, failure]
+
+requires:
+  blocks:
+    - playground.generate
+    - transform.jq
+
+params:
+  type: object
+  properties:
+    fail_until:
+      type: integer
+      description: How many attempts fail before one succeeds; 3 exhausts the budget below.
+      default: 2
+      minimum: 0
+      maximum: 4
+
+steps:
+  flaky:
+    block: playground.generate
+    retry:
+      max_attempts: 3
+      backoff: 1s
+      multiplier: 2.0
+      max_backoff: 4s
+      # No jitter, so the timing above is the timing you see.
+      jitter: 0.0
+    config:
+      fail_until: ${params.fail_until}
+      rows: 3
+      seed: 7
+      fields:
+        who: name
+
+  proof:
+    block: transform.jq
+    depends_on: [flaky]
+    config:
+      input:
+        attempt: ${steps.flaky.output.attempt}
+        failed_before_it: ${steps.flaky.output.fail_until}
+        rows: ${steps.flaky.output.rows}
+      program: '{succeeded_on_attempt: .attempt, after_failures: .failed_before_it, rows: .rows}'