dirigent-examples 0.15.0__py3-none-any.whl

dirigent_examples/shelves/open-data/open-meteo-weekly-report.yaml

@@ -0,0 +1,142 @@
+# A week of weather for one point, from a public archive, ending as a csv in storage.
+#
+# Open-Meteo's archive API serves reanalysed historical weather for any coordinate, with no
+# key, no account and no quota worth worrying about. It answers columnar: one `daily.time`
+# array of dates and one array per variable, aligned by index. That shape is convenient for a
+# chart and wrong for everything else, so the first thing this pipeline does is turn it into
+# rows.
+#
+# What happens, hop by hop:
+#
+#   covered   the run's window, an interval of instants, turned into the two dates the API
+#             asks for. The window is half-open -- start included, end excluded -- and
+#             Open-Meteo's end_date is inclusive, so the end is pulled back one day here
+#             rather than being quietly off by one in the output.
+#   fetch     one GET. The response is the columnar payload above, held inline: a week of
+#             three variables is a few kilobytes, so nothing needs to go to storage yet.
+#   rows      the transposition. `daily.time` indexes everything, so walking its indices and
+#             reading each variable at the same offset is the whole job.
+#   staged    storage.write, the rows as one json object. A converter reads one URI and writes
+#             another, so a value the run is holding is put down before it is re-encoded.
+#   report    the codec, json to csv, from that object to the one a person opens.
+#
+# The archive lags real weather by about five days, so a schedule that reads the week that
+# just closed is reading a week the archive already has. That is why the cadence below fires
+# on Monday for the week before, and why the ad hoc window in the run line is in the past.
+#
+# To make it yours: point latitude and longitude at your site, name the variables you care
+# about in `daily`, and change the schedule's timezone to the one your week is counted in.
+# For many points, this is the pipeline a fan-out calls once per site.
+#
+#   dg run --local examples/open-data/open-meteo-weekly-report.yaml --window 2026-08-17..2026-08-24
+#   dg run --local examples/open-data/open-meteo-weekly-report.yaml --window 2026-08-17..2026-08-24 \
+#     -p latitude=-1.29 -p longitude=36.82
+
+format: dirigent/v1
+kind: pipeline
+code: open-meteo-weekly-report
+name: Weekly weather report
+description: |
+  A week of daily temperature and precipitation for one coordinate, read from the
+  **Open-Meteo archive API**, transposed from columns to rows and written as csv.
+
+  The week is the run's window, so the same document serves the Monday schedule and a
+  backfill of any week the archive holds.
+
+tags: [open-data, http, storage, transform, csv, schedule, starter]
+
+requires:
+  blocks:
+    - transform.jq
+    - http.request
+    - storage.write
+    - convert.std
+
+params:
+  type: object
+  additionalProperties: false
+  properties:
+    latitude:
+      type: number
+      default: -13.98
+      description: Degrees north; the default is Lilongwe, Malawi.
+    longitude:
+      type: number
+      default: 33.79
+      description: Degrees east.
+    daily:
+      type: string
+      default: temperature_2m_max,temperature_2m_min,precipitation_sum
+      description: The archive's daily variables, comma-separated, as the API names them.
+
+steps:
+  covered:
+    block: transform.jq
+    config:
+      input:
+        start: ${run.window.start}
+        end: ${run.window.end}
+      # fromdateiso8601 wants a Z, and a window edge carries a numeric offset, so the string
+      # is cut to seconds and given one. The subtraction is the half-open-to-inclusive fix.
+      program: |
+        def as_date: .[0:19] + "Z" | fromdateiso8601;
+        {
+          start_date: .start[0:10],
+          end_date: (.end | as_date | . - 86400 | strftime("%Y-%m-%d"))
+        }
+
+  fetch:
+    block: http.request
+    depends_on: [covered]
+    config:
+      url: https://archive-api.open-meteo.com/v1/archive
+      query:
+        latitude: ${params.latitude}
+        longitude: ${params.longitude}
+        start_date: ${steps.covered.output.value.start_date}
+        end_date: ${steps.covered.output.value.end_date}
+        daily: ${params.daily}
+        # UTC rather than the station's local zone, so a row's date means the same thing here
+        # as it does in every other pipeline on this shelf.
+        timezone: UTC
+
+  rows:
+    block: transform.jq
+    depends_on: [fetch]
+    config:
+      input: ${steps.fetch.output.body}
+      # The arrays under daily are parallel, so the index is the join key. reduce over the
+      # keys rather than naming the variables keeps this program correct when the `daily`
+      # parameter asks for a different set.
+      program: |
+        . as $body
+        | ($body.daily | keys_unsorted - ["time"]) as $vars
+        | [range($body.daily.time | length) as $i
+           | reduce $vars[] as $var
+               ({date: $body.daily.time[$i]}; .[$var] = $body.daily[$var][$i])]
+
+  staged:
+    block: storage.write
+    depends_on: [rows]
+    config:
+      target: ${run.scratch}/weather/${run.window.start}.json
+      value: ${steps.rows.output.value}
+
+  report:
+    block: convert.std
+    depends_on: [staged]
+    config:
+      source: ${steps.staged.output.uri}
+      # The window start names the file, so a backfill of ten weeks writes ten objects and
+      # overwrites none of them.
+      target: ${run.scratch}/weather/${run.window.start}.csv
+      from: json
+      to: csv
+
+triggers:
+  schedules:
+    - code: weekly
+      name: Monday morning, for the week before
+      description: Fires once the archive has settled on the week that just closed.
+      cron: "0 6 * * 1"
+      timezone: UTC

dirigent_examples/shelves/open-data/overpass-health-facilities.yaml

@@ -0,0 +1,154 @@
+# Every hospital and clinic OpenStreetMap knows about inside a box, as csv and as parquet.
+#
+# Overpass is a query engine over live OSM data, keyless and public. Its query language is its
+# own -- `[out:json];(node[...](bbox);way[...](bbox););out center;` -- and it answers
+# {"version": ..., "elements": [...]}. An element is a node, which carries lat and lon itself,
+# or a way, which is a shape and carries a computed `center` instead because `out center` asked
+# for one. Everything else about a facility lives in `tags`, a free-form map: `name`, `amenity`,
+# `healthcare`, `operator`, whatever a mapper typed.
+#
+# On the request: Overpass takes the query as the body of a POST, written in its own language
+# and not in JSON. `http.request` sends a string body as it stands and serialises anything else
+# as JSON, so the query goes in `body` as text -- a JSON-quoted query is a parse error at
+# Overpass, not a query -- and `content_type` says what the server is being handed.
+#
+# What happens, hop by hop:
+#
+#   fetch     one POST carrying the query, holding the answer inline. The box below is a city,
+#             which is a few hundred elements.
+#   rows      the flattening. Nodes and ways are reconciled to one lat/lon pair, and the four
+#             tags worth having become columns; an element with no name still becomes a row,
+#             because an unnamed clinic is a fact about the map rather than a broken record.
+#   staged    storage.write, the rows as one json object. A converter reads one URI and writes
+#             another, so both codecs below read this one object.
+#   csv       for a person: opens in anything, loses the types.
+#   parquet   for a pipeline: keeps the types, and is what any later analysis wants to read.
+#             The same rows are written twice on purpose; that is the point of having both
+#             codecs, and neither one re-fetches anything.
+#
+# Overpass is a shared public instance with a per-IP quota, and it answers a slow query with a
+# 504 rather than queueing it. The retry below is not decoration: it is how this pipeline
+# behaves on an ordinary busy afternoon. Heavy or scheduled use belongs on your own instance.
+#
+# To make it yours: move the four bbox edges, and change amenities to whatever OSM calls the
+# thing you are mapping -- `pharmacy`, `doctors`, `school`. The regex in the query is what
+# makes the list a list.
+#
+#   dg run --local examples/open-data/overpass-health-facilities.yaml
+#   dg run --local examples/open-data/overpass-health-facilities.yaml \
+#     -p south=27.65 -p west=85.25 -p north=27.78 -p east=85.40 -p amenities='pharmacy|doctors'
+
+format: dirigent/v1
+kind: pipeline
+code: overpass-health-facilities
+name: Health facilities from OpenStreetMap
+description: |
+  Hospitals and clinics inside a bounding box, read from the **Overpass API**, flattened into
+  rows and written twice: csv for a person and parquet for the next pipeline.
+
+tags: [open-data, http, storage, transform, csv, parquet, starter]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+    - storage.write
+    - convert.std
+    - convert.arrow
+
+params:
+  type: object
+  additionalProperties: false
+  properties:
+    south:
+      type: number
+      default: -14.05
+    west:
+      type: number
+      default: 33.70
+    north:
+      type: number
+      default: -13.90
+    east:
+      type: number
+      default: 33.90
+      description: The four edges of the box; the default is Lilongwe, Malawi.
+    amenities:
+      type: string
+      default: hospital|clinic
+      description: An Overpass regex alternation of amenity values.
+
+steps:
+  fetch:
+    block: http.request
+    # A public instance under load answers 504 in seconds rather than queueing the query, so
+    # the wait between tries is long enough for the queue in front of it to drain.
+    retry:
+      max_attempts: 3
+      backoff: 30s
+      multiplier: 2.0
+    config:
+      url: https://overpass-api.de/api/interpreter
+      method: POST
+      # The [timeout:60] inside the query is Overpass's own budget for running it; the
+      # timeout below is how long this step waits for the whole exchange. The inner
+      # one has to be the smaller of the two, or the server is still working when the
+      # client has given up.
+      body: |
+        [out:json][timeout:60];
+        (
+          node["amenity"~"^(${params.amenities})$"](${params.south},${params.west},${params.north},${params.east});
+          way["amenity"~"^(${params.amenities})$"](${params.south},${params.west},${params.north},${params.east});
+        );
+        out center;
+      # What the interpreter is handed: OverpassQL as plain text, said by the step rather than
+      # left for the client to guess.
+      content_type: text/plain; charset=utf-8
+      timeout: 2m
+
+  rows:
+    block: transform.jq
+    depends_on: [fetch]
+    config:
+      input: ${steps.fetch.output.body}
+      # A way has no coordinates of its own; `out center` gives it a centroid, and `// ` is
+      # what reconciles the two shapes into one pair of columns.
+      program: |
+        [.elements[]
+         | {
+             osm_kind: .type,
+             osm_id: .id,
+             name: (.tags.name // null),
+             amenity: (.tags.amenity // null),
+             healthcare: (.tags.healthcare // null),
+             operator: (.tags.operator // null),
+             latitude: (.lat // .center.lat),
+             longitude: (.lon // .center.lon)
+           }]
+        | sort_by(.name // "")
+
+  staged:
+    block: storage.write
+    depends_on: [rows]
+    config:
+      target: ${run.scratch}/facilities.json
+      value: ${steps.rows.output.value}
+
+  csv:
+    block: convert.std
+    depends_on: [staged]
+    config:
+      source: ${steps.staged.output.uri}
+      target: ${run.scratch}/facilities.csv
+      from: json
+      to: csv
+
+  parquet:
+    block: convert.arrow
+    depends_on: [staged]
+    config:
+      # The same object the csv is made from, read a second time: two codecs, one source.
+      source: ${steps.staged.output.uri}
+      target: ${run.scratch}/facilities.parquet
+      from: json
+      to: parquet

dirigent_examples/shelves/open-data/usgs-earthquakes-alert.yaml

@@ -0,0 +1,208 @@
+# Everything the ground did in the last day, cut down to what is near you and big enough to care.
+#
+# The USGS summary feeds are keyless GeoJSON, updated every minute: all_day.geojson is a
+# FeatureCollection of every event in the past 24 hours, a couple of hundred of them on an
+# ordinary day. Each feature carries the magnitude, the place, and the epoch-milliseconds time
+# in `properties`, and the position as [longitude, latitude, depth_km] in `geometry.coordinates`
+# -- longitude first, which is the GeoJSON order and the opposite of how people say it.
+#
+# The interesting part is not the fetch, it is the quiet day. An alert that posts "nothing
+# happened" every hour is an alert nobody reads, so this pipeline has to be able to do nothing.
+# There is no `if` in the format, and there is no conditional edge; what there is is a sensor
+# that can skip, and a step behind a skipped sensor is skipped with it. So the digest is written
+# to storage first, and `storage.exists` is asked for it with a floor of one byte: an empty
+# digest writes an empty file, the floor is never met, the sensor times out and skips, the
+# webhook skips behind it, and the run still ends succeeded. A matching event writes bytes, the
+# very first poke sees them, and the post goes out.
+#
+# What happens, hop by hop:
+#
+#   feed      one GET, held inline: the whole day is well under a megabyte.
+#   candidates  every feature flattened into a row, with the great-circle distance worked out
+#             and the two thresholds copied onto it. jq has the trigonometry, so geography
+#             needs no subprocess and no allowlist entry. The thresholds ride along as data
+#             because that is the only way the filter below can see them: a jq program is
+#             compiled once and never has a value spliced into its text.
+#   nearby    filter.jq, comparing each row's own numbers against the limits it carries.
+#   digest    map.jq drops the limits again, leaving the row a person reads.
+#   ordered   the digest, biggest magnitude first, which is the order the file below holds.
+#   staged    storage.write, that list as one json object. A converter reads one URI and writes
+#             another, so a value the run is holding is put down before it is re-encoded.
+#   written   json to ndjson. ndjson is what makes the gate work: an empty list encodes as an
+#             empty file, where an empty json array would still be two bytes.
+#   matched   the gate. Three seconds is plenty when the file is already there; on a quiet
+#             day it is three seconds of waiting and then a clean skip.
+#   notify    the digest, posted. It points at Postman Echo so this example runs for real
+#             without standing anything up; on your instance it is your own receiver, and
+#             `sign_with` naming a connection with an hmac_secret is how the receiver knows
+#             the post came from you.
+#
+# To make it yours: set latitude, longitude and radius_km to your area of responsibility, raise
+# min_magnitude until the alert is worth waking up for, and point webhook_url at your receiver.
+# A schedule firing hourly is the natural cadence; the feed only ever holds a day.
+#
+#   dg run --local examples/open-data/usgs-earthquakes-alert.yaml
+#   dg run --local examples/open-data/usgs-earthquakes-alert.yaml -p min_magnitude=7 -p radius_km=50
+
+format: dirigent/v1
+kind: pipeline
+code: usgs-earthquakes-alert
+name: Earthquakes near a point
+description: |
+  The **USGS** past-day earthquake feed, filtered by magnitude and great-circle distance from a
+  point, posted onward as a digest -- and posted only when something matched.
+
+  A quiet day ends `succeeded` with the gate and the post skipped, which is what an alert
+  pipeline has to be able to do.
+
+tags: [open-data, http, sensor, storage, transform, webhook, filter, starter]
+
+requires:
+  blocks:
+    - http.request
+    - filter.jq
+    - map.jq
+    - transform.jq
+    - storage.write
+    - convert.std
+    - storage.exists
+    - webhook.post
+
+params:
+  type: object
+  additionalProperties: false
+  properties:
+    latitude:
+      type: number
+      default: 36.0
+      description: Degrees north of the point everything is measured from.
+    longitude:
+      type: number
+      default: -119.5
+      description: Degrees east; the default point is California's Central Valley.
+    radius_km:
+      type: number
+      default: 800
+      description: How far from that point an event still counts.
+    min_magnitude:
+      type: number
+      default: 2.0
+      description: Raise this until the digest is worth reading.
+    webhook_url:
+      type: string
+      default: https://postman-echo.com/post
+      description: Where the digest goes; Postman Echo so the example posts for real.
+
+steps:
+  feed:
+    block: http.request
+    # The feed is regenerated every minute and served from a cache that is occasionally slow;
+    # a second try is cheaper than a missed hour.
+    retry:
+      max_attempts: 2
+      backoff: 10s
+    config:
+      url: https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.geojson
+
+  candidates:
+    block: transform.jq
+    depends_on: [feed]
+    config:
+      # Everything the program needs arrives as data: the features, the point, and the two
+      # limits. Nothing about this input is spliced into the program's text.
+      input:
+        features: ${steps.feed.output.body.features}
+        latitude: ${params.latitude}
+        longitude: ${params.longitude}
+        radius_km: ${params.radius_km}
+        min_magnitude: ${params.min_magnitude}
+      # GeoJSON coordinates are [longitude, latitude, depth], longitude first. USGS times are
+      # epoch milliseconds, so a digest a person reads has to divide before it formats.
+      program: |
+        def radians: . * 3.141592653589793 / 180;
+        . as {$latitude, $longitude, $radius_km, $min_magnitude}
+        | [.features[]
+           | .geometry.coordinates as $c
+           | ((($latitude - $c[1]) | radians / 2 | sin)) as $dp
+           | ((($longitude - $c[0]) | radians / 2 | sin)) as $dl
+           | ($dp * $dp + ($c[1] | radians | cos) * ($latitude | radians | cos) * $dl * $dl) as $a
+           | {
+               id: .id,
+               magnitude: (.properties.mag // -1),
+               place: .properties.place,
+               at: (.properties.time / 1000 | todate),
+               depth_km: $c[2],
+               distance_km: (2 * 6371 * (($a | sqrt) | asin) | round),
+               url: .properties.url,
+               radius_km: $radius_km,
+               min_magnitude: $min_magnitude
+             }]
+
+  nearby:
+    block: filter.jq
+    depends_on: [candidates]
+    config:
+      input: ${steps.candidates.output.value}
+      # Every number in this comparison is a field of the row being tested, which is why the
+      # program is three words long and needs no parameters of its own.
+      program: |
+        .magnitude >= .min_magnitude and .distance_km <= .radius_km
+
+  digest:
+    block: map.jq
+    depends_on: [nearby]
+    config:
+      input: ${steps.nearby.output.value}
+      # The limits did their job in the filter; what goes out is the event.
+      program: |
+        del(.radius_km, .min_magnitude)
+
+  ordered:
+    block: transform.jq
+    depends_on: [digest]
+    config:
+      input: ${steps.digest.output.value}
+      program: |
+        sort_by(-.magnitude)
+
+  staged:
+    block: storage.write
+    depends_on: [ordered]
+    config:
+      target: ${run.scratch}/matched.json
+      value: ${steps.ordered.output.value}
+
+  written:
+    block: convert.std
+    depends_on: [staged]
+    config:
+      source: ${steps.staged.output.uri}
+      target: ${run.scratch}/matched.ndjson
+      from: json
+      to: ndjson
+
+  matched:
+    block: storage.exists
+    depends_on: [written]
+    poll: 1s
+    deadline: 3s
+    # The whole point: nothing matched is not a failure, it is a run with nothing to say.
+    on_timeout: skip
+    config:
+      uri: ${steps.written.output.target}
+      # One byte is the difference between an empty digest and a digest.
+      min_size: 1b
+
+  notify:
+    block: webhook.post
+    depends_on: [digest, matched]
+    config:
+      url: ${params.webhook_url}
+      body:
+        kind: earthquake-digest
+        near:
+          - ${params.latitude}
+          - ${params.longitude}
+        radius_km: ${params.radius_km}
+        min_magnitude: ${params.min_magnitude}
+        events: ${steps.digest.output.value}

dirigent_examples/shelves/open-data/who-gho-indicators-to-parquet.yaml

@@ -0,0 +1,163 @@
+# One WHO indicator, several countries, one table, one parquet file, and a manifest over the batch.
+#
+# The WHO Global Health Observatory serves its whole indicator catalogue as OData, keyless:
+# /api/<INDICATOR_CODE> answers {"@odata.context": ..., "value": [ ... ]}, one object per
+# observation, each carrying the country in SpatialDim, the year in TimeDim, the disaggregation
+# in Dim1, and the number in NumericValue. The envelope is OData's; the rows inside it are what
+# anyone actually wants.
+#
+# What this shows, beyond the fetch: how a fan-out is read back. `for_each` is expanded when the
+# run is created, so it can read params, run and item -- and never another step's output. And a
+# fan-out step stores one output: the list of its items' outputs, in item order, with a failed
+# item absent from it. So there is no way for item three of one step to ask item three of another
+# what it produced; what a downstream step gets is the whole list at once. That is why the
+# flattening below is a single step over every envelope, and why the country is read out of
+# SpatialDim: the join key lives in the data, where a fan-in can see it.
+#
+# What happens, hop by hop:
+#
+#   fetch     one GET per country. items: continue means a country the indicator has no data
+#             for, or one the endpoint is slow for, does not take the rest of the batch down
+#             with it -- and its absence from this step's output list is what the manifest
+#             below counts.
+#   rows      the fan-in: every envelope this step is handed, opened at `.value` and flattened
+#             into one table sorted by country and year.
+#   staged    storage.write, that table as one json object. A converter reads one URI and
+#             writes another, so a value the run is holding is put down before it is
+#             re-encoded.
+#   parquet   convert.arrow, json to parquet. Parquet is bytes, so it never travels as a value:
+#             the step names a source URI and a target URI and nothing is carried between them.
+#   manifest  what the run has to say for itself: how many countries were asked for, how many
+#             answered, how many rows landed, and where the file is.
+#   index     the manifest written beside the parquet, which is the object another pipeline
+#             lists a dataset from.
+#
+# To make it yours: change indicator to any code from https://ghoapi.azureedge.net/api/Indicator
+# and countries to the ISO3 codes you report on. Point the targets at s3:// on an instance with
+# an object store, and the manifest becomes a dataset index other pipelines read.
+#
+#   dg run --local examples/open-data/who-gho-indicators-to-parquet.yaml
+#   dg run --local examples/open-data/who-gho-indicators-to-parquet.yaml \
+#     -p indicator=WHOSIS_000015 -p countries='["NPL","BGD"]'
+
+format: dirigent/v1
+kind: pipeline
+code: who-gho-indicators-to-parquet
+name: WHO GHO indicator to parquet
+description: |
+  One **GHO** indicator fetched once per country, flattened out of the OData envelopes into one
+  table and written as parquet, with a manifest counting what came back.
+
+  A fan-out's results are read back as one list, because a fan-out step stores one output and
+  `for_each` cannot read an upstream step's output at all.
+
+tags: [open-data, http, storage, transform, fan-out, parquet, starter]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+    - storage.write
+    - convert.arrow
+
+params:
+  type: object
+  additionalProperties: false
+  properties:
+    indicator:
+      type: string
+      default: WHOSIS_000001
+      description: A GHO indicator code; the default is life expectancy at birth.
+    countries:
+      type: array
+      default: [MWI, NPL, ETH]
+      items:
+        type: string
+        description: ISO3 country code, as GHO writes it in SpatialDim.
+
+steps:
+  fetch:
+    block: http.request
+    for_each: ${params.countries}
+    # A country with no data for this indicator answers an empty value list rather than an
+    # error, so what this policy really tolerates is the endpoint being slow or briefly down.
+    items: continue
+    # The GHO endpoint sits behind a CDN that occasionally takes seconds to answer a cold
+    # indicator; one retry costs nothing and saves the batch.
+    retry:
+      max_attempts: 3
+      backoff: 5s
+    config:
+      url: https://ghoapi.azureedge.net/api/${params.indicator}
+      query:
+        # OData's filter language, not a URL convention: the quotes around the code are part
+        # of the expression and the server parses them.
+        $filter: SpatialDim eq '${item}'
+      timeout: 1m
+
+  rows:
+    block: transform.jq
+    depends_on: [fetch]
+    config:
+      # One fan-out step, one output: the list of what its items answered. A country whose
+      # item failed is simply not in it.
+      input: ${steps.fetch.output}
+      # NumericValue rather than Value: Value is the display string, "48.0 [46.7-49.6]", and
+      # a column that has to be parsed before it can be added up is not a number.
+      program: |
+        [.[]
+         | .body.value[]
+         | {country: .SpatialDim,
+            year: .TimeDim,
+            dimension: .Dim1,
+            value: .NumericValue,
+            low: .Low,
+            high: .High}]
+        | sort_by(.country, .year, .dimension)
+
+  staged:
+    block: storage.write
+    depends_on: [rows]
+    config:
+      target: ${run.scratch}/rows/${params.indicator}.json
+      value: ${steps.rows.output.value}
+
+  parquet:
+    block: convert.arrow
+    depends_on: [staged]
+    config:
+      source: ${steps.staged.output.uri}
+      target: ${run.scratch}/parquet/${params.indicator}.parquet
+      from: json
+      to: parquet
+
+  manifest:
+    block: transform.jq
+    depends_on: [fetch, rows, parquet]
+    config:
+      input:
+        indicator: ${params.indicator}
+        requested: ${params.countries}
+        answered: ${steps.fetch.output}
+        rows: ${steps.rows.output.value}
+        file: ${steps.parquet.output.target}
+        bytes: ${steps.parquet.output.bytes_written}
+      # `answered` is shorter than `requested` exactly when an item failed, which is the only
+      # place in the run where that difference is visible as data rather than as a status.
+      program: |
+        {
+          indicator,
+          file,
+          bytes,
+          requested: (.requested | length),
+          answered: (.answered | length),
+          rows: (.rows | length),
+          countries: ([.rows[].country] | unique)
+        }
+
+  index:
+    block: storage.write
+    depends_on: [manifest]
+    config:
+      target: ${run.scratch}/parquet/${params.indicator}-manifest.json
+      value: ${steps.manifest.output.value}