dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/patterns/webhook-signed.yaml

@@ -0,0 +1,144 @@
+# A signed outbound delivery, and where the secret on each side of it lives.
+#
+# webhook.post is the sending half of the webhook story; webhook-mapping-nested-payload.yaml
+# is the receiving half. sign_with names the connection whose hmac_secret signs the body, and
+# the block presents the result as X-Dirigent-Signature -- computed exactly the way dirigent's
+# own /hooks/{token} verifies one, so a dirigent instance can be the receiver on the other end
+# with nothing in between.
+#
+# THREE THINGS ARE WORTH BEING PRECISE ABOUT:
+#
+#   * The signature covers THE BYTES THAT GO ON THE WIRE. The block serialises the body
+#     itself and signs those bytes, because letting an HTTP client re-encode a body after
+#     signing produces a signature that verifies nowhere.
+#   * sign_with is separate from connection ON PURPOSE. A POST to an absolute URL can still be
+#     signed with a secret this instance holds, and signing is something the document says out
+#     loud rather than something that happens invisibly because a connection had a field set.
+#   * A secret is never in a document. It is on a connection, sealed at rest and redacted in
+#     every API response, and a document names the connection. What this file carries is a
+#     connection for a --local run to create, which is the one case where carrying one is
+#     right -- a server refuses a document that embeds a credential, because applying it would
+#     store the secret in every version of the pipeline.
+#
+# THE INBOUND DIRECTION IS AN INSTANCE SETTING, NOT A DOCUMENT ONE. A webhook a document
+# declares carries only its code and its payload mapping; whether deliveries to it must be
+# signed, and with what secret, is set where the token is minted:
+#
+#   dg webhook create <pipeline> <code> --hmac-secret "$SECRET" --map day='$.date'
+#
+# An unsigned delivery to a webhook that requires a signature is answered 401 with "this
+# webhook requires a X-Dirigent-Signature header", and a wrong one is answered 401 too, after
+# a constant-time comparison.
+#
+# Hop by hop:
+#
+#   build     assembles the notification body. Signing a value a step computed is the ordinary
+#             case; the signature is over whatever the body serialises to.
+#   notify    POSTs it signed. Postman Echo takes any body and answers with what it received,
+#             so its answer shows the header that was presented.
+#   unsigned  the same POST with no sign_with, for comparison: output.signed is false.
+#   compare   reads both back and puts the two signed flags side by side.
+#
+# EXPECT THIS RUN TO SUCCEED in about two seconds, with notify's signed true and unsigned's
+# signed false.
+#
+#   dg run --local examples/patterns/webhook-signed.yaml
+
+format: dirigent/v1
+kind: pipeline
+code: webhook-signed
+name: A signed webhook delivery
+description: |
+  `webhook.post` with `sign_with` presents `X-Dirigent-Signature` over the exact bytes it
+  sends, computed the way dirigent's own inbound `/hooks/{token}` verifies one.
+
+  The secret lives on a connection, never in the document. Requiring a signature on an
+  **inbound** webhook is an instance setting made when the token is minted, not something
+  `dirigent/v1` can say.
+
+tags: [patterns, transform, webhook, credential]
+
+requires:
+  blocks:
+    - webhook.post
+    - transform.jq
+    - value.const
+
+# Carried, because a --local run has no instance to hold a connection. Against a real
+# instance this is a connection created once, and the document names it by code:
+#   dg connection create http ops-receiver --set base_url=https://postman-echo.com \
+#     --set hmac_secret=...
+connections:
+  ops-receiver:
+    kind: http
+    config:
+      base_url: https://postman-echo.com
+      # A SecretStr: sealed on the way in, redacted on the way out, and never readable back
+      # from the API. This one is a demonstration value and protects nothing.
+      hmac_secret: a shared secret between two dirigent instances
+      timeout: 30s
+
+params:
+  type: object
+  properties:
+    dataset:
+      type: string
+      description: What the notification says finished.
+      default: cases
+    rows:
+      type: integer
+      description: How many rows it says were loaded.
+      default: 4120
+      minimum: 0
+
+steps:
+  build:
+    block: value.const
+    config:
+      value:
+        event: dataset.loaded
+        dataset: "${params.dataset}"
+        rows: "${params.rows}"
+
+  notify:
+    block: webhook.post
+    depends_on: [build]
+    config:
+      # Where it goes: the connection's base URL plus this path.
+      connection: ops-receiver
+      path: /post
+      # What signs it. The same connection here, but it need not be: a POST to one place can
+      # be signed with a secret agreed with somebody else.
+      sign_with: ops-receiver
+      body: "${steps.build.output.value}"
+      headers:
+        # Routing metadata the receiver wants. Headers are outside the signature, which
+        # covers the body's bytes only, so nothing security-bearing belongs up here.
+        x-event-kind: dataset.loaded
+
+  unsigned:
+    block: webhook.post
+    depends_on: [build]
+    # No sign_with, which is the default. The pair exists so the signed flag has something to
+    # be compared against.
+    config:
+      connection: ops-receiver
+      path: /post
+      body: "${steps.build.output.value}"
+
+  compare:
+    block: transform.jq
+    depends_on: [notify, unsigned]
+    config:
+      input:
+        signed: "${steps.notify.output.signed}"
+        unsigned: "${steps.unsigned.output.signed}"
+        # The receiver echoes the headers it was sent, so the signature the block presented is
+        # readable here. A real receiver verifies it and says nothing about it.
+        presented: "${steps.notify.output.json_body.headers}"
+      program: |
+        {
+          signed,
+          unsigned,
+          signature_header: (.presented | keys | map(select(ascii_downcase == "x-dirigent-signature")) | first)
+        }

dirigent_examples/shelves/preview/s3-parquet-to-ingestion.yaml

@@ -0,0 +1,92 @@
+# NEEDS THE dirigent-storage-s3 PACKAGE, which registers the s3:// scheme, and an
+# S3-compatible endpoint set up as examples/s3/s3-round-trip.yaml documents.
+#
+# This is the blueprint's worked example, and the thing it demonstrates is that a storage
+# backend is not a step. Nothing here says "connect to S3": s3:// is a registered scheme,
+# and every block that takes a URI can address it, including the sensor that waits for the
+# object to appear. Swapping the drop from s3:// to gs:// is a one-word edit in one line.
+#
+# The parquet never becomes a step output, and it could not: a value comes into a run only
+# through storage.read, which reads text and json and refuses a format it cannot decode.
+# What travels between the steps is the address of the object, which is what makes the
+# fan-out below cheap: each region is handed the same URI, and nothing is copied to do it.
+# The ingestion endpoint reads the object from the bucket it is already looking at.
+#
+# Hop by hop:
+#
+#   wait_for_drop   the sensor. Its output says which object landed and how big it is.
+#   push            one call per region, each carrying the address and the size rather than
+#                   the bytes.
+#   notify_failure  one_failed, so a region that could not be pushed is heard about.
+
+format: dirigent/v1
+kind: pipeline
+code: s3-parquet-to-ingestion
+name: Parquet from S3 to an ingestion endpoint
+description: Wait for the daily parquet drop, hand each region its address, alert on failure.
+
+tags: [preview, http, sensor, storage]
+
+concurrency: skip
+
+requires:
+  blocks:
+    - storage.exists
+    - http.request
+  connections:
+    - modelling-api
+    - ops-webhook
+  storage:
+    - s3
+
+params:
+  type: object
+  required: [day]
+  properties:
+    day:
+      type: string
+      format: date
+    regions:
+      type: array
+      default: [east, west]
+      items:
+        type: string
+
+steps:
+  wait_for_drop:
+    block: storage.exists
+    poll: 5m
+    deadline: 6h
+    on_timeout: skip
+    config:
+      uri: "s3://drops/climate/${params.day}.parquet"
+
+  push:
+    block: http.request
+    depends_on: [wait_for_drop]
+    for_each: "${params.regions}"
+    items: continue
+    retry:
+      max_attempts: 5
+      backoff: 30s
+    config:
+      connection: modelling-api
+      path: "/v1/ingest/${item}"
+      method: POST
+      # The object the sensor found, named rather than carried. The size is what the
+      # endpoint checks the drop against before it starts reading.
+      body:
+        day: "${params.day}"
+        region: "${item}"
+        source: "${steps.wait_for_drop.output.uri}"
+        source_bytes: "${steps.wait_for_drop.output.size}"
+
+  notify_failure:
+    block: http.request
+    depends_on: [push]
+    rule: one_failed
+    config:
+      connection: ops-webhook
+      method: POST
+      body:
+        text: "s3-parquet-to-ingestion failed for ${params.day}"

dirigent_examples/shelves/python/README.md

@@ -0,0 +1,31 @@
+# Driving an instance from Python
+
+The YAML files above are pipelines. These are programs that drive an instance: each one is a
+single file using [`dirigent-client`](../../packages/dirigent-client), the typed async SDK.
+
+Every script reads `DG_URL` and `DG_TOKEN` from the environment, exactly as `dg` does:
+
+```bash
+export DG_URL=http://127.0.0.1:3333 DG_TOKEN=...   # dg dev prints both
+uv run python examples/python/apply_and_run.py
+```
+
+| Script | What it shows |
+| --- | --- |
+| [`apply_and_run.py`](apply_and_run.py) | Plan, apply, run, wait for the terminal state, print the report. |
+| [`follow_logs.py`](follow_logs.py) | Stream a run's log entries as the workers write them. |
+| [`list_and_filter.py`](list_and_filter.py) | Query pipelines and runs by pipeline, status, and window. |
+| [`connections.py`](connections.py) | Create a credential record, check it, read it back redacted, delete it. |
+| [`error_handling.py`](error_handling.py) | Provoke each typed refusal, and handle each on its own terms. |
+| [`ci_gate.py`](ci_gate.py) | Apply and run in a build job, exiting non-zero when no run started or the run does not succeed. |
+
+`apply_and_run.py`, `follow_logs.py`, and `ci_gate.py` apply
+[`hello-world.yaml`](../hello-world.yaml), which uses `shell.run`. That block executes code
+on the worker, so the instance must allowlist it:
+
+```bash
+export DIRIGENT_ENABLED_UNSAFE_BLOCKS='["shell.run"]'
+```
+
+The REST API remains available directly; see [docs/python.md](../../docs/python.md) for the
+SDK reference and for what the raw endpoints answer with.

dirigent_examples/shelves/python/apply_and_run.py

@@ -0,0 +1,52 @@
+"""Apply a document, start a run, wait for it, and report what happened.
+
+The canonical first script: everything a CI job or a scheduler-of-schedulers does.
+
+    export DG_URL=http://127.0.0.1:3333 DG_TOKEN=...
+    uv run python examples/python/apply_and_run.py
+
+hello-world.yaml uses shell.run, which executes code on the worker, so the instance must
+allowlist it: DIRIGENT_ENABLED_UNSAFE_BLOCKS='["shell.run"]'.
+"""
+
+import asyncio
+import os
+from datetime import timedelta
+from pathlib import Path
+
+from dirigent_client import Dirigent, PlanAction, RunStatus
+
+DOCUMENT = Path(__file__).resolve().parents[1] / "hello-world.yaml"
+
+
+async def main() -> int:
+    """Apply the document, run it, and wait for the run to settle."""
+    async with Dirigent(url=os.environ["DG_URL"], token=os.environ["DG_TOKEN"]) as dg:
+        plan = await dg.pipelines.apply(DOCUMENT, dry_run=True)
+        print(f"plan: {plan.plan.action.value} {plan.plan.code}")
+        if plan.plan.action is PlanAction.INVALID:
+            for issue in plan.plan.issues:
+                print(f"  - {issue}")
+            return 1
+
+        applied = await dg.pipelines.apply(DOCUMENT)
+        print(f"applied: {applied.plan.code} version {applied.version or applied.plan.current_version}")
+
+        accepted = await dg.pipelines.run(applied.plan.code)
+        if accepted.run_id is None:
+            # Not a failure: the pipeline's concurrency policy declined a second run.
+            print(f"not started: {accepted.detail}")
+            return 0
+
+        print(f"started: run {accepted.run_id}")
+        run = await dg.runs.wait(accepted.run_id, timeout=timedelta(minutes=5))
+        print(f"finished: {run.status.value} in {run.pipeline}")
+
+        report = await dg.runs.report(run.id)
+        for step in report.steps:
+            print(f"  {step.outcome:<10} {step.step} ({step.block})")
+        return 0 if run.status is RunStatus.SUCCEEDED else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

dirigent_examples/shelves/python/ci_gate.py

@@ -0,0 +1,76 @@
+"""Apply a document and run it, exiting non-zero when no run started or the run does not succeed.
+
+The shape of a deploy job: one script, one exit code, no interpretation needed by whoever
+reads the build log.
+
+    export DG_URL=https://dirigent.example.org DG_TOKEN=...
+    uv run python examples/python/ci_gate.py pipelines/daily-load.yaml
+
+In a GitHub workflow:
+
+    - run: uv run python examples/python/ci_gate.py pipelines/daily-load.yaml
+      env:
+        DG_URL: ${{ vars.DG_URL }}
+        DG_TOKEN: ${{ secrets.DG_TOKEN }}
+"""
+
+import asyncio
+import os
+import sys
+from datetime import timedelta
+from pathlib import Path
+
+from dirigent_client import Dirigent, DirigentError, PlanAction, ProvenanceSource, RunStatus, WaitTimeout
+
+#: How long the job is willing to wait for the run before it gives up on it.
+BUDGET = timedelta(hours=2)
+
+#: Statuses this gate accepts. A run that tolerated an item failure is not a green build.
+ACCEPTED = (RunStatus.SUCCEEDED,)
+
+
+async def gate(document: Path, reference: str) -> int:
+    """Apply, run, wait, and report; the return value is the job's exit code."""
+    async with Dirigent(url=os.environ["DG_URL"], token=os.environ["DG_TOKEN"]) as dg:
+        applied = await dg.pipelines.apply(document, source=ProvenanceSource.FILE, source_ref=reference)
+        plan = applied.plan
+        if plan.action is PlanAction.INVALID:
+            print(f"::error::{document} does not validate against this instance")
+            for issue in plan.issues:
+                print(f"  {issue}")
+            return 1
+        print(f"{plan.action.value} {plan.code} -> version {applied.version or plan.current_version}")
+
+        accepted = await dg.pipelines.run(plan.code)
+        if accepted.run_id is None:
+            print(f"::error::not started: {accepted.detail}")
+            return 1
+
+        try:
+            run = await dg.runs.wait(accepted.run_id, timeout=BUDGET)
+        except WaitTimeout as timed_out:
+            print(f"::error::{timed_out.message}")
+            return 1
+
+        report = await dg.runs.report(run.id)
+        for step in report.steps:
+            print(f"  {step.outcome:<10} {step.step:<24} {step.error or ''}")
+        if run.status in ACCEPTED:
+            print(f"{run.status.value} in {(report.duration_ms or 0) / 1000:.1f}s")
+            return 0
+        print(f"::error::run {run.id} {run.status.value}: {run.error or 'see the step table above'}")
+        return 1
+
+
+def main() -> int:
+    """Read the document from the command line and run the gate over it."""
+    reference = sys.argv[1] if len(sys.argv) > 1 else str(Path(__file__).resolve().parents[1] / "hello-world.yaml")
+    try:
+        return asyncio.run(gate(Path(reference), reference))
+    except DirigentError as refusal:
+        print(f"::error::{refusal.message}")
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dirigent_examples/shelves/python/connections.py

@@ -0,0 +1,61 @@
+"""Create a connection, ask whether it answers, and remove it again.
+
+    export DG_URL=http://127.0.0.1:3333 DG_TOKEN=...
+    uv run python examples/python/connections.py
+
+A connection's kind declares which of its fields are secret by marking them SecretStr. That
+one declaration is what makes the API redact them and the engine encrypt them, so a read
+never returns a credential: this script writes one and gets the redaction marker back.
+"""
+
+import asyncio
+import os
+
+from dirigent_client import Conflict, Dirigent
+
+NAME = "example-http"
+
+
+async def main() -> int:
+    """Create a connection, check it, read it back redacted, and delete it."""
+    async with Dirigent(url=os.environ["DG_URL"], token=os.environ["DG_TOKEN"]) as dg:
+        catalog = await dg.blocks.catalog()
+        print(f"connection kinds: {', '.join(entry.id for entry in catalog.connection_kinds) or 'none'}")
+
+        created_here = False
+        try:
+            created = await dg.connections.create(
+                NAME,
+                kind="http",
+                description="An example credential; nothing behind it is real.",
+                config={
+                    # A reserved name that never resolves, so the check below fails without
+                    # this example ever reaching a real service.
+                    "base_url": "https://service.invalid",
+                    "bearer_token": "not-a-real-token",
+                },
+            )
+            created_here = True
+        except Conflict:
+            print(f"{NAME} already exists; reading it instead")
+            created = await dg.connections.get(NAME)
+
+        print(f"created: {created.code} ({created.kind})")
+        print(f"  secret fields: {', '.join(created.secret_fields) or 'none'}")
+        for field, value in created.config.items():
+            print(f"  {field}: {value}")
+
+        report = await dg.connections.check(NAME)
+        print(f"check: {'healthy' if report.healthy else 'unhealthy'} -- {report.detail or ''}")
+
+        # Only remove what this run made; a connection that was already here is left in place.
+        if created_here:
+            await dg.connections.delete(NAME)
+            print(f"deleted: {NAME}")
+        else:
+            print(f"leaving pre-existing {NAME} in place")
+        return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

dirigent_examples/shelves/python/error_handling.py

@@ -0,0 +1,84 @@
+"""Provoke each typed refusal, and show the shape of handling it.
+
+    export DG_URL=http://127.0.0.1:3333 DG_TOKEN=...
+    uv run python examples/python/error_handling.py
+
+Every exception carries the status, the URL, and the parsed problem body, so a caller can
+print one sentence, branch on a status, or read the field list a validation failure names.
+"""
+
+import asyncio
+import os
+
+from dirigent_client import (
+    Dirigent,
+    DirigentError,
+    NotDirigent,
+    NotFound,
+    Unauthorized,
+    ValidationFailed,
+)
+
+DOCUMENT = """
+format: dirigent/v1
+kind: pipeline
+code: error-handling-demo
+params:
+  type: object
+  required: [day]
+  properties:
+    day:
+      type: string
+steps:
+  gate:
+    block: time.window
+    config:
+      after: "00:00"
+      before: "23:59"
+"""
+
+
+async def main() -> int:
+    """Ask for four things that cannot work, and handle each refusal on its own terms."""
+    url = os.environ["DG_URL"]
+    token = os.environ["DG_TOKEN"]
+
+    async with Dirigent(url=url, token=token) as dg:
+        try:
+            await dg.pipelines.get("no-pipeline-has-this-code")
+        except NotFound as refusal:
+            print(f"not found: {refusal.message}  [{refusal.status} from {refusal.url}]")
+
+        await dg.pipelines.apply(DOCUMENT)
+        try:
+            await dg.pipelines.run("error-handling-demo", params={"day": 7})
+        except ValidationFailed as refusal:
+            print(f"refused: {refusal.message}")
+            for problem in refusal.problems:
+                print(f"  - {problem}")
+        finally:
+            await dg.pipelines.delete("error-handling-demo")
+
+    async with Dirigent(url=url, token="not-a-real-token") as dg:
+        try:
+            await dg.auth.whoami()
+        except Unauthorized as refusal:
+            print(f"unauthorized: {refusal.message}")
+
+    # A URL pointing at the wrong thing has two shapes. Nothing listening is a
+    # TransportError; something answering without X-Dirigent-Version -- a docs server, a
+    # proxy, another service -- is NotDirigent, and both mean the URL rather than the
+    # request is wrong. Every refusal is a DirigentError, so one clause is enough when the
+    # distinction does not matter.
+    async with Dirigent(url="http://127.0.0.1:1", token=token, retries=0) as dg:
+        try:
+            await dg.system.info()
+        except NotDirigent as refusal:
+            print(f"wrong address: {refusal.message}")
+        except DirigentError as refusal:
+            print(f"unreachable: {refusal.message}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

dirigent_examples/shelves/python/follow_logs.py

@@ -0,0 +1,39 @@
+"""Start a run and print its log entries as the workers write them.
+
+    export DG_URL=http://127.0.0.1:3333 DG_TOKEN=...
+    uv run python examples/python/follow_logs.py
+
+The tail is server-sent events; the client reopens a dropped connection from the last entry
+it yielded, so a proxy timing the stream out does not lose or repeat a line.
+"""
+
+import asyncio
+import os
+from pathlib import Path
+
+from dirigent_client import Dirigent
+
+DOCUMENT = Path(__file__).resolve().parents[1] / "hello-world.yaml"
+
+
+async def main() -> int:
+    """Apply, run, and stream the run's log entries until it settles."""
+    async with Dirigent(url=os.environ["DG_URL"], token=os.environ["DG_TOKEN"]) as dg:
+        applied = await dg.pipelines.apply(DOCUMENT)
+        accepted = await dg.pipelines.run(applied.plan.code)
+        if accepted.run_id is None:
+            print(f"not started: {accepted.detail}")
+            return 0
+
+        print(f"following run {accepted.run_id}")
+        async for entry in dg.runs.follow_logs(accepted.run_id):
+            where = entry.step_name or "-"
+            print(f"  {entry.level.value:<7} {where:<12} {entry.message}")
+
+        run = (await dg.runs.get(accepted.run_id)).run
+        print(f"{run.status.value}")
+        return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

dirigent_examples/shelves/python/list_and_filter.py

@@ -0,0 +1,52 @@
+"""Query runs by pipeline, status, and how far back to look, and print them as a table.
+
+    export DG_URL=http://127.0.0.1:3333 DG_TOKEN=...
+    uv run python examples/python/list_and_filter.py
+
+Every listing answers a page: `items`, and a `next` cursor to pass back as `after` when
+there is more. A window (`since`) and a limit are still how a report bounds itself; the
+cursor is how it reads past the first page without asking for a bigger one.
+"""
+
+import asyncio
+import os
+
+from dirigent_client import Dirigent, RunStatus
+
+#: The window a report covers, in the humane duration grammar the whole product uses.
+WINDOW = "24h"
+
+
+async def main() -> int:
+    """Print the pipelines this instance holds, and its recent runs."""
+    async with Dirigent(url=os.environ["DG_URL"], token=os.environ["DG_TOKEN"]) as dg:
+        pipelines = await dg.pipelines.list()
+        print(f"{'pipeline':<28} {'version':<8} {'active':<7} {'in flight'}")
+        for pipeline in pipelines.items:
+            version = str(pipeline.current_version or "-")
+            print(f"{pipeline.code:<28} {version:<8} {str(pipeline.active):<7} {pipeline.active_runs}")
+
+        print(f"\nruns in the last {WINDOW}")
+        print(f"{'run':<38} {'pipeline':<20} {'status':<22} {'started'}")
+        for run in (await dg.runs.list(since=WINDOW, limit=20)).items:
+            started = run.started_at.isoformat() if run.started_at else "-"
+            print(f"{run.id!s:<38} {run.pipeline:<20} {run.status.value:<22} {started}")
+
+        # Walking the cursor is how a report covers a window larger than one page.
+        failed = []
+        after = None
+        while True:
+            page = await dg.runs.list(status=RunStatus.FAILED, since=WINDOW, after=after)
+            failed.extend(page.items)
+            if page.next is None:
+                break
+            after = page.next
+
+        print(f"\n{len(failed)} failed in the last {WINDOW}")
+        for run in failed:
+            print(f"  {run.id}  {run.pipeline}  {run.error or 'no error recorded'}")
+        return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))