dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/open-data/feeds-composition.yaml

@@ -0,0 +1,120 @@
+# Two of the feeds on this shelf, started by a third: refresh the reference, then read the data.
+#
+# `pipeline.run` starts another pipeline on the same instance and, by default, waits for it. So
+# this document is an orchestration layer over documents that each stand alone: the Wikidata
+# reference table and the WHO GHO extract know nothing about each other or about this file, and
+# each is still runnable, testable and schedulable on its own.
+#
+# WHAT CROSSES THE BOUNDARY, AND WHAT DOES NOT. Parameters go down: the parent hands each child
+# exactly the ones that child's own schema declares, validated against that schema when the step
+# executes rather than when this document is applied. What comes back up is a status and a run
+# id -- not the child's data. So the country list below is written here rather than read out of
+# the reference the first child just built: a child's rows reach a parent only through storage,
+# at a path both sides agree on, and under `--local` each run's scratch is its own. On an
+# instance, giving both children a shared prefix parameter is what turns "refresh the reference"
+# into something the next child can actually read.
+#
+# `requires.pipelines` names both children, so an instance that does not hold them refuses this
+# document at apply time with the list of what to apply first, instead of failing at the step at
+# six in the morning.
+#
+# What happens, hop by hop:
+#
+#   reference  the Wikidata country table, rebuilt. It takes no parameters: what it produces is
+#             the same for everybody, which is what makes it a reference.
+#   indicators  the GHO extract for one indicator over the country list, started after the
+#             reference finished. It fans out internally with items: continue, so a country the
+#             endpoint refuses leaves that child completed_with_errors -- and `strict` is left
+#             at its default here, so that outcome is reported rather than treated as failure.
+#             Setting strict: true is how a parent says a partial child is not good enough.
+#   briefing  the receipt: which children ran, under which run ids, and how each ended. A run
+#             id is what somebody opens to see what actually happened inside a child.
+#
+# To make it yours: add a child. A briefing is a list of pipeline.run steps and a transform that
+# reads their statuses, and adding a feed is one step plus one line in requires.
+#
+#   dg run --local examples/open-data/feeds-composition.yaml \
+#     --also-apply examples/open-data/wikidata-country-reference.yaml \
+#     --also-apply examples/open-data/who-gho-indicators-to-parquet.yaml
+#
+# Against an instance the children are applied first, and then the parent is just a pipeline:
+#
+#   dg apply examples/open-data/wikidata-country-reference.yaml
+#   dg apply examples/open-data/who-gho-indicators-to-parquet.yaml
+#   dg apply examples/open-data/feeds-composition.yaml
+#   dg run feeds-composition --watch
+
+format: dirigent/v1
+kind: pipeline
+code: feeds-composition
+name: Open data briefing
+description: |
+  One document that runs two others: the **Wikidata** country reference and the **WHO GHO**
+  indicator extract, then reports how each child ended.
+
+  Parameters go down, statuses come back up, and data crosses between pipelines through
+  storage rather than through a step output.
+
+tags: [open-data, pipeline, transform, briefing, composition]
+
+requires:
+  blocks:
+    - pipeline.run
+    - transform.jq
+  pipelines:
+    - wikidata-country-reference
+    - who-gho-indicators-to-parquet
+
+params:
+  type: object
+  additionalProperties: false
+  properties:
+    indicator:
+      type: string
+      default: WHOSIS_000001
+      description: Handed to the GHO extract as its own indicator parameter.
+    countries:
+      type: array
+      default: [MWI, NPL]
+      items:
+        type: string
+      description: The countries the extract fans out over.
+
+steps:
+  reference:
+    block: pipeline.run
+    config:
+      pipeline: wikidata-country-reference
+      # No params: the child's defaults are the whole answer, and a parent that repeats them
+      # here would be a second place to change them.
+
+  indicators:
+    block: pipeline.run
+    depends_on: [reference]
+    config:
+      pipeline: who-gho-indicators-to-parquet
+      # Only what the child's schema declares. A parameter it does not have is refused when
+      # this step executes, which is the same check a person typing -p would get.
+      params:
+        indicator: ${params.indicator}
+        countries: ${params.countries}
+
+  briefing:
+    block: transform.jq
+    depends_on: [indicators]
+    config:
+      input:
+        indicator: ${params.indicator}
+        reference: ${steps.reference.output}
+        indicators: ${steps.indicators.output}
+      # A child hands back three things and no data: which pipeline ran, which run it was, and
+      # how it ended.
+      program: |
+        {
+          indicator,
+          children: [
+            {feed: .reference.pipeline, run: .reference.run_id, status: .reference.status},
+            {feed: .indicators.pipeline, run: .indicators.run_id, status: .indicators.status}
+          ],
+          all_green: ([.reference.status, .indicators.status] | all(. == "succeeded"))
+        }

dirigent_examples/shelves/open-data/gdacs-disaster-updates.yaml

@@ -0,0 +1,230 @@
+# Yesterday's disaster alerts, with the ones you already saw taken out.
+#
+# GDACS -- the Global Disaster Alert and Coordination System, run by the JRC and OCHA -- is
+# keyless and anonymous: no account, no token, no registered application. One GET answers
+# GeoJSON, {"features": [{"properties": {...}}]}, and everything an event has is under
+# `properties`.
+#
+# The window does not make the query non-overlapping. GDACS filters on the event's own active
+# range, so a cyclone that opened last week and has not closed comes back in today's answer and
+# in tomorrow's, and an event that is revised keeps its `eventid` while `datemodified` moves.
+# Both are the same problem: the query cannot tell you what is new. The fix is a list of what
+# yesterday's run saw, and the honest consequence is visible in the graph: yesterday's file is
+# read behind a sensor, so on the very first day there is nothing to compare against, `seen`
+# skips, and everything behind it skips too. Day one posts nothing and ends succeeded; day two
+# posts what day one did not have. There is no state block, so a marker file is the state, and a
+# skipped read is how a pipeline says "there was no marker".
+#
+# What happens, hop by hop:
+#
+#   dates     the window turned into the two dates -- the ones the query filters between, and
+#             the ones the files below are named by. GDACS wants a plain day here, not the
+#             instant a window edge carries. It settles the prefix the lists are kept under
+#             as well, since the default one is only known once the run has started.
+#   fetch     one GET. The two list filters are semicolon-joined in a single parameter, which is
+#             this API's own convention rather than a general one.
+#   today     the events flattened to rows, carried as a value because the comparison needs them
+#             as data.
+#   record    storage.write, this day's list, put down before anything is compared so tomorrow's
+#             run has something to read even if the post below fails.
+#   seen      yesterday's list, if there is one. Five seconds, then skip: an absent file is the
+#             expected answer on day one and after any gap in the schedule.
+#   recalled  storage.read, yesterday's file brought back into the run as a value. A value
+#             goes out to storage only through a write and comes in only through a read, so
+#             the marker costs a step at each end.
+#   previous  yesterday's ids, taken out of what that read produced.
+#   fresh     the difference. The comparison is over ids alone, because a revised event keeps
+#             its id and moves its date.
+#   notify    what is actually new, posted. Postman Echo by default so the shape of the post is
+#             visible without a receiver; point it at yours.
+#
+# To make it yours: narrow `eventlist` and `alertlevel` to what you act on, and give `store` a
+# prefix that outlives one run. The default is `${run.scratch}`, which is deleted with the run,
+# so under it every day is day one.
+#
+# Rehearsing both days from the CLI takes one directory holding the instance, and a `store`
+# under that directory's artifact root so the second run finds what the first wrote:
+#
+#   dg run --local --root /tmp/dirigent-gdacs examples/open-data/gdacs-disaster-updates.yaml \
+#     -p store=file:///tmp/dirigent-gdacs/artifacts/gdacs --window 2026-09-05..2026-09-06
+#   dg run --local --root /tmp/dirigent-gdacs examples/open-data/gdacs-disaster-updates.yaml \
+#     -p store=file:///tmp/dirigent-gdacs/artifacts/gdacs --window 2026-09-06..2026-09-07
+#
+# Day one skips at `seen` and posts nothing; day two posts the difference.
+
+format: dirigent/v1
+kind: pipeline
+code: gdacs-disaster-updates
+name: GDACS disaster updates
+description: |
+  Disaster events active inside the run's window, read from the **GDACS** event list,
+  deduplicated against the previous day's list and posted onward.
+
+  Keyless and anonymous: nothing to register, nothing to hold.
+
+tags: [open-data, http, sensor, storage, transform, webhook, schedule]
+
+requires:
+  blocks:
+    - transform.jq
+    - http.request
+    - storage.write
+    - storage.exists
+    - storage.read
+    - webhook.post
+
+params:
+  type: object
+  additionalProperties: false
+  properties:
+    eventlist:
+      type: string
+      default: EQ;TC;FL
+      description: >-
+        Event types, semicolon-separated as GDACS reads them: EQ earthquake, TC tropical
+        cyclone, FL flood, VO volcano, DR drought, WF wildfire.
+    alertlevel:
+      type: string
+      default: Green;Orange;Red
+      description: Alert levels, semicolon-separated. Orange;Red is the pair worth waking for.
+    store:
+      type: string
+      default: ""
+      description: >-
+        A prefix the daily lists are written under, such as
+        file:///var/lib/dirigent/gdacs. Empty keeps them in the run's own scratch, where
+        tomorrow's run cannot read them.
+    webhook_url:
+      type: string
+      default: https://postman-echo.com/post
+      description: Where the digest goes.
+
+steps:
+  dates:
+    block: transform.jq
+    config:
+      input:
+        start: ${run.window.start}
+        end: ${run.window.end}
+        store: ${params.store}
+        scratch: ${run.scratch}
+      # The two days the query filters between, and the two file names. Yesterday is the day
+      # before the window opened, which is the file the previous firing wrote. The prefix is
+      # settled here too: a reference resolves once, so a default that names another one has
+      # to be worked out in a step rather than written in the parameter.
+      program: |
+        def as_date: .[0:19] + "Z" | fromdateiso8601;
+        {
+          day: .start[0:10],
+          from_date: .start[0:10],
+          to_date: .end[0:10],
+          previous_day: (.start | as_date | . - 86400 | strftime("%Y-%m-%d")),
+          store: (if .store == "" then .scratch + "/gdacs" else .store end)
+        }
+
+  fetch:
+    block: http.request
+    depends_on: [dates]
+    config:
+      url: https://www.gdacs.org/gdacsapi/api/events/geteventlist/SEARCH
+      query:
+        # Both filters are lists written into one parameter, joined with semicolons. This is
+        # GDACS's own convention, not a general one.
+        eventlist: ${params.eventlist}
+        alertlevel: ${params.alertlevel}
+        # A plain day at each edge. The filter is an overlap against the event's own from/to
+        # range, so an event still running when the window opened is included.
+        fromDate: ${steps.dates.output.value.from_date}
+        toDate: ${steps.dates.output.value.to_date}
+        # The API's own page ceiling. A single day worldwide is far short of it; a longer
+        # window is not, and would need pageNumber walked.
+        pageSize: 100
+
+  today:
+    block: transform.jq
+    depends_on: [fetch]
+    config:
+      input: ${steps.fetch.output.body}
+      # GeoJSON, so the event is under properties. The id is the type and the number together:
+      # GDACS's own detail URL takes both, and the number alone is a sequence per event type.
+      program: |
+        [.features[]
+         | .properties as $p
+         | {
+             id: "\($p.eventtype)-\($p.eventid)",
+             title: $p.name,
+             url: $p.url.report,
+             event_type: $p.eventtype,
+             alert_level: $p.alertlevel,
+             country: $p.country,
+             from_date: $p.fromdate,
+             modified_at: $p.datemodified
+           }]
+
+  record:
+    block: storage.write
+    depends_on: [dates, today]
+    config:
+      # Written before anything is compared, so tomorrow's run has a list to read even if the
+      # post below fails. The day names the file, which is what makes "yesterday" a path.
+      target: ${steps.dates.output.value.store}/${steps.dates.output.value.day}.json
+      value: ${steps.today.output.value}
+
+  seen:
+    block: storage.exists
+    depends_on: [dates, record]
+    poll: 1s
+    deadline: 5s
+    # No file for yesterday means no previous run, or a gap in the schedule. Both are quiet
+    # days rather than failures, and everything downstream skips with this step.
+    on_timeout: skip
+    config:
+      uri: ${steps.dates.output.value.store}/${steps.dates.output.value.previous_day}.json
+
+  recalled:
+    block: storage.read
+    depends_on: [seen]
+    config:
+      # The path the sensor above just found, read as the value yesterday's run wrote.
+      source: ${steps.seen.output.uri}
+
+  previous:
+    block: transform.jq
+    depends_on: [recalled]
+    config:
+      input: ${steps.recalled.output.value}
+      # Only the ids: the comparison is about identity, and yesterday's alert levels are
+      # yesterday's problem.
+      program: |
+        [.[].id]
+
+  fresh:
+    block: transform.jq
+    depends_on: [today, previous]
+    config:
+      input:
+        today: ${steps.today.output.value}
+        seen: ${steps.previous.output.value}
+      # A revised event keeps its id and moves its datemodified, so identity is the only safe
+      # key. An escalation from Green to Red is therefore not news here; a new eventid is.
+      program: |
+        . as {$today, $seen}
+        | [$today[] | select(.id as $id | $seen | index($id) | not)]
+
+  notify:
+    block: webhook.post
+    depends_on: [dates, fresh]
+    config:
+      url: ${params.webhook_url}
+      body:
+        kind: gdacs-updates
+        day: ${steps.dates.output.value.day}
+        events: ${steps.fresh.output.value}
+
+triggers:
+  schedules:
+    - code: daily
+      name: Every morning, for the day that closed
+      description: Reads the previous day, which is the window this firing carries.
+      cron: "0 7 * * *"
+      timezone: UTC

dirigent_examples/shelves/open-data/github-releases-relay.yaml

@@ -0,0 +1,225 @@
+# New releases of a repository, relayed onward, with the rate limit treated as a fact of life.
+#
+# GitHub's REST API serves public repositories without a token: GET /repos/{owner}/{repo}/releases
+# answers a list, newest first, each release carrying `tag_name`, `name`, `published_at`,
+# `prerelease`, `draft` and `html_url`. Unauthenticated calls are limited to 60 requests per hour
+# per IP, and that is the whole budget -- shared with everything else on that address. A schedule
+# firing every ten minutes spends 6 of them; one firing hourly spends 1, which is the right
+# cadence for something that publishes a release a month. A token raises the limit to 5000, and
+# is the only reason to give this pipeline a connection.
+#
+# The floor is a parameter here, and the marker is written but not read. `since` says what has
+# already been relayed, the last step records the newest stamp this run saw, and pointing the
+# next firing's `since` at that marker is the read half of the pattern -- the same sensor-plus-
+# transform shape any marker read has. Splitting it this way keeps the two halves legible: this
+# document is about what "new" means and what a rate limit costs, not about state.
+#
+# What happens, hop by hop:
+#
+#   releases  one GET, with the API version pinned by the Accept header. GitHub asks every
+#             client for a User-Agent and refuses requests without one.
+#   candidates  the releases flattened, each row carrying the floor and the two exclusions it
+#             will be judged by. A jq program never has values spliced into its text, so
+#             anything a filter compares against has to arrive as data on the row.
+#   fresh     filter.jq: published after the floor, not a draft, and not a prerelease unless
+#             the run asked for them. Three comparisons, each between two fields of the row.
+#   digest    the rows a person would read, with the carried limits dropped again.
+#   ordered   the digest in publication order, which is the order the file below holds.
+#   staged    storage.write, the digest as one json object. A converter reads a URI and writes
+#             a URI, so a value the run is holding is put down before it can be converted.
+#   written   json to ndjson, because an empty list has to encode as an empty file for the
+#             gate below to see the difference.
+#   matched   the gate: nothing new is a skip, not a failure, and the relay skips with it.
+#   relay     the digest, posted. Postman Echo by default so this runs for real.
+#   marker    the newest stamp seen, plus what the rate limit had left, computed under
+#             rule: all_done so a quiet run still records where it got to. GitHub reports the
+#             remaining budget in a response header, which is worth keeping beside the stamp:
+#             when the relay goes quiet, that number says whether it is the repo or the quota.
+#   remember  that marker written to storage, which is the half the next firing would read.
+#
+# To make it yours: set repo, move `since` forward to when you started caring, and point
+# webhook_url at your receiver. For a private repository or a heavier cadence, put a token in
+# a connection and give the step `connection:` instead of `url:`.
+#
+#   dg run --local examples/open-data/github-releases-relay.yaml
+#   dg run --local examples/open-data/github-releases-relay.yaml -p repo=jqlang/jq -p since=2030-01-01T00:00:00Z
+
+format: dirigent/v1
+kind: pipeline
+code: github-releases-relay
+name: GitHub releases relay
+description: |
+  Releases of a public repository published since a floor, read from **GitHub's REST API**
+  without a token, and relayed onward -- only when there are any.
+
+  Unauthenticated calls share a budget of 60 requests an hour per IP, which is what decides
+  the cadence a schedule can have.
+
+tags: [open-data, http, sensor, storage, transform, webhook, filter, relay, starter]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+    - filter.jq
+    - map.jq
+    - convert.std
+    - storage.write
+    - storage.exists
+    - webhook.post
+
+params:
+  type: object
+  additionalProperties: false
+  properties:
+    repo:
+      type: string
+      default: jqlang/jq
+      description: owner/name of a public repository.
+    since:
+      type: string
+      default: "2020-01-01T00:00:00Z"
+      description: Only releases published after this instant are relayed.
+    include_prereleases:
+      type: boolean
+      default: false
+    per_page:
+      type: integer
+      default: 30
+      maximum: 100
+      description: One page is one request against a 60-per-hour budget; ask for enough.
+    webhook_url:
+      type: string
+      default: https://postman-echo.com/post
+
+steps:
+  releases:
+    block: http.request
+    config:
+      url: https://api.github.com/repos/${params.repo}/releases
+      headers:
+        # Pinning the API version here is what stops a future default from reshaping the
+        # payload under a pipeline that has been running for a year.
+        Accept: application/vnd.github+json
+        X-GitHub-Api-Version: "2022-11-28"
+        User-Agent: dirigent-example
+      query:
+        per_page: ${params.per_page}
+
+  candidates:
+    block: transform.jq
+    depends_on: [releases]
+    config:
+      input:
+        releases: ${steps.releases.output.body}
+        since: ${params.since}
+        include_prereleases: ${params.include_prereleases}
+      # A draft has no published_at at all, so the null is replaced with a stamp that loses
+      # every comparison rather than being compared as null.
+      program: |
+        . as {$since, $include_prereleases}
+        | [.releases[]
+           | {
+               tag: .tag_name,
+               title: (.name // .tag_name),
+               published_at: (.published_at // "0000-01-01T00:00:00Z"),
+               draft: .draft,
+               prerelease: .prerelease,
+               url: .html_url,
+               since: $since,
+               include_prereleases: $include_prereleases
+             }]
+
+  fresh:
+    block: filter.jq
+    depends_on: [candidates]
+    config:
+      input: ${steps.candidates.output.value}
+      # ISO 8601 stamps in the same zone compare correctly as strings, which is the one thing
+      # that format is designed to make true.
+      program: |
+        .published_at > .since
+        and (.draft | not)
+        and (.include_prereleases or (.prerelease | not))
+
+  digest:
+    block: map.jq
+    depends_on: [fresh]
+    config:
+      input: ${steps.fresh.output.value}
+      program: |
+        del(.since, .include_prereleases)
+
+  ordered:
+    block: transform.jq
+    depends_on: [digest]
+    config:
+      input: ${steps.digest.output.value}
+      program: |
+        sort_by(.published_at)
+
+  staged:
+    block: storage.write
+    depends_on: [ordered]
+    config:
+      target: ${run.scratch}/fresh.json
+      value: ${steps.ordered.output.value}
+
+  written:
+    block: convert.std
+    depends_on: [staged]
+    config:
+      source: ${steps.staged.output.uri}
+      target: ${run.scratch}/fresh.ndjson
+      from: json
+      to: ndjson
+
+  matched:
+    block: storage.exists
+    depends_on: [written]
+    poll: 1s
+    deadline: 3s
+    on_timeout: skip
+    config:
+      uri: ${steps.written.output.target}
+      min_size: 1b
+
+  relay:
+    block: webhook.post
+    depends_on: [digest, matched]
+    config:
+      url: ${params.webhook_url}
+      body:
+        kind: github-releases
+        repo: ${params.repo}
+        since: ${params.since}
+        releases: ${steps.digest.output.value}
+
+  marker:
+    block: transform.jq
+    depends_on: [releases, digest, matched]
+    # all_done: a run that relayed nothing still saw the repository, and still knows what the
+    # budget had left.
+    rule: all_done
+    config:
+      input:
+        repo: ${params.repo}
+        releases: ${steps.digest.output.value}
+        # Header names arrive lower-cased, which is what HTTP/2 makes of them.
+        rate_limit_remaining: ${steps.releases.output.headers.x-ratelimit-remaining}
+      # The newest stamp seen, which is what the next run's `since` should be. `max` over an
+      # empty list is null, so a quiet run reports null rather than inventing a floor.
+      program: |
+        {
+          repo,
+          rate_limit_remaining,
+          relayed: (.releases | length),
+          newest_published_at: ([.releases[].published_at] | max)
+        }
+
+  remember:
+    block: storage.write
+    depends_on: [marker]
+    config:
+      target: ${run.scratch}/markers/releases.json
+      value: ${steps.marker.output.value}