dirigent-integration 0.23.2__py3-none-any.whl

dirigent_integration/__init__.py

@@ -0,0 +1,29 @@
+"""The control center's own contribution: the cross-boundary example shelves.
+
+This distribution contributes no blocks and no connection kinds. Every block its documents
+name comes from a pack the integration assembles; what is only ever authored here is the
+document that puts two packs in one flow, and this is how that corpus reaches an instance.
+"""
+
+from collections.abc import Sequence
+from importlib.resources import files
+from importlib.resources.abc import Traversable
+
+from dirigent_plugin import extension
+
+#: The directory the cross-boundary shelves live in, inside this distribution.
+SHELVES_DIRECTORY = "shelves"
+
+
+class IntegrationPlugin:
+    """The plugin object the host discovers under the dirigent.plugins.v1 entry-point group."""
+
+    @extension
+    def examples(self) -> Sequence[Traversable]:
+        """Contribute the cross-boundary shelves this distribution carries, and nothing else."""
+        return [files(__package__ or "dirigent_integration") / SHELVES_DIRECTORY]
+
+
+plugin = IntegrationPlugin()
+
+__all__ = ["SHELVES_DIRECTORY", "IntegrationPlugin", "plugin"]

dirigent_integration/shelves/dhis2-analytics-to-csv-report.yaml

@@ -0,0 +1,174 @@
+# An analytics query turned into a csv somebody can open, and a webhook saying it is there.
+#
+# DHIS2 answers /api/analytics with a grid: a `headers` list naming the columns, a `rows` list
+# of positional arrays, and a `metaData` block holding the display names for every uid that
+# appears. That is a good wire format and a poor report -- nothing outside DHIS2 knows that
+# column 2 is the value -- so the middle of this pipeline is the reshaping, and it is the part
+# worth reading. Three packs meet here: dhis2 for the query, builtin for the jq, the csv codec
+# and the webhook, storage-s3 for where the report lands.
+#
+# What happens, hop by hop:
+#
+#   query     dhis2.analytics_query, aggregate mode, broken down by indicator and period with
+#             the org unit fixed as a filter. The grid comes back as `body`, a value.
+#   rows      the grid becomes a list of flat objects, one per grid row, with the uids
+#             resolved to names out of metaData: {indicator, period, value}. That is the
+#             shape a csv has, and the grid is not.
+#   staged    storage.write, that value as one json object. A converter reads one URI and
+#             writes another, so a value the run is holding is put down before it is
+#             re-encoded -- and that write is the only door a value leaves a run by.
+#   report    convert.std json -> csv, reading the staged object and writing to s3://. The
+#             header line is the object keys. Output: source, target, bytes_written.
+#   summary   how many rows and where they went, built before the notification so the webhook
+#             body is a value and not four references.
+#   notify    webhook.post of that summary to whatever URL the caller named. Nothing is signed
+#             here; use sign_with when the receiver verifies an HMAC.
+#
+# To make it yours: point indicators at your own uids (semicolons separate them, which is
+# DHIS2's own syntax for several items on one dimension), change period to a fixed period or
+# another relative one, and give notify_url a receiver. The default is Postman Echo, which
+# reflects what it is posted, so the run shows the body that went out without a receiver of
+# your own. Which connection backs s3:// is an instance setting (storage_connections:
+# {s3: <code>}).
+#
+#   dg run --local examples/dhis2-analytics-to-csv-report.yaml
+#   dg run --local examples/dhis2-analytics-to-csv-report.yaml -p period=LAST_4_QUARTERS
+
+format: dirigent/v1
+kind: pipeline
+code: dhis2-analytics-to-csv-report
+name: An analytics grid as a csv report
+description: Query DHIS2 analytics, flatten the grid into rows, write a csv to S3, and announce it.
+
+tags: [dhis2, s3, csv, cross-boundary]
+
+requires:
+  blocks:
+    - dhis2.analytics_query
+    - transform.jq
+    - storage.write
+    - convert.std
+    - webhook.post
+
+# The public DHIS2 play server, carried inline so this document runs standalone. An instance
+# you own names a connection it holds instead of carrying one.
+connections:
+  dhis2-demo:
+    kind: dhis2
+    config:
+      base_url: https://play.im.dhis2.org/stable-2-43-1
+      basic_username: admin
+      basic_password: district
+      timeout: 60s
+
+params:
+  type: object
+  properties:
+    indicators:
+      type: string
+      # ANC 1 coverage and ANC 3 coverage on the play server. Semicolons are DHIS2's own
+      # separator for several items on one dimension, so this stays a single string.
+      default: Uvn6LCg7dVU;OdiHJayrsKo
+    period:
+      type: string
+      default: LAST_12_MONTHS
+      description: A relative period keyword or an ISO period, as DHIS2's pe dimension takes.
+    org_unit:
+      type: string
+      default: ImspTQPwCqd
+    bucket:
+      type: string
+      default: dirigent-reports
+    notify_url:
+      type: string
+      default: https://postman-echo.com/post
+      description: Where the summary is posted; the default reflects it back for inspection.
+
+steps:
+  query:
+    block: dhis2.analytics_query
+    config:
+      connection: dhis2-demo
+      mode: aggregate
+      # dimension axes are what the grid is broken down by; filter axes fix a dimension the
+      # grid is not broken down by. Putting ou in filter is what makes the report two columns
+      # wide instead of three.
+      dimension:
+        - dx:${params.indicators}
+        - pe:${params.period}
+      filter:
+        - ou:${params.org_unit}
+      # UID keeps the grid's own cells as uids and the display names in metaData, which the
+      # next step resolves. Asking DHIS2 for NAME here would flatten the report but lose the
+      # uid, and a report a machine reads again wants both.
+      output_id_scheme: UID
+
+  rows:
+    block: transform.jq
+    depends_on: [query]
+    config:
+      input: ${steps.query.output.body}
+      # The grid is positional, so the column positions are looked up by header name rather
+      # than hard-coded: DHIS2 has moved them between versions. metaData.items maps every uid
+      # in the answer to its display name, which is what turns dx:Uvn6LCg7dVU into a label.
+      program: |
+        . as $grid
+        | ([$grid.headers[].name] | index("dx")) as $dx
+        | ([$grid.headers[].name] | index("pe")) as $pe
+        | ([$grid.headers[].name] | index("value")) as $value
+        | $grid.metaData.items as $names
+        | [ $grid.rows[]
+          | {
+              indicator: ($names[.[$dx]].name // .[$dx]),
+              period: ($names[.[$pe]].name // .[$pe]),
+              value: .[$value]
+            }
+          ]
+
+  staged:
+    block: storage.write
+    depends_on: [rows]
+    config:
+      # The run's own scratch space: these bytes exist only so the converter has a URI to
+      # read, and the report the caller keeps is the csv below.
+      target: ${run.scratch}/analytics/${params.period}.json
+      value: ${steps.rows.output.value}
+
+  report:
+    block: convert.std
+    depends_on: [staged]
+    config:
+      source: ${steps.staged.output.uri}
+      from: json
+      to: csv
+      # The report outlives the run, so ${run.scratch} is out: retention sweeps a run's prefix
+      # with the run. ${artifacts}/<path> is the portable way to keep something -- the instance's
+      # own storage root, which nothing sweeps -- and a named bucket is the other case, which is
+      # this one: the csv has to land in the reporting bucket people already fetch reports from.
+      target: s3://${params.bucket}/dhis2/analytics/${params.period}.csv
+
+  summary:
+    block: transform.jq
+    depends_on: [report]
+    config:
+      input:
+        rows: ${steps.rows.output.value}
+        csv: ${steps.report.output.target}
+        bytes: ${steps.report.output.bytes_written}
+      # Counting happens here rather than in the notification because a webhook body should be
+      # a value that was computed, not four references a receiver has to trust were resolved.
+      program: |
+        {report: .csv, report_bytes: .bytes, row_count: (.rows | length)}
+
+  notify:
+    block: webhook.post
+    depends_on: [summary]
+    retry:
+      max_attempts: 3
+      backoff: 30s
+    config:
+      url: ${params.notify_url}
+      body: ${steps.summary.output.value}
+      # A receiver that queues the notification answers 202, which is delivered as far as this
+      # step is concerned: posting it is what the step is responsible for.
+      success_status: [200, 201, 202]

dirigent_integration/shelves/dhis2-export-to-s3.yaml

@@ -0,0 +1,66 @@
+# Export a DHIS2 data value set and archive it in object storage.
+#
+# This pipeline is the integration's own: it belongs to no single pack, because it spans two.
+# The read is a dhis2.data_value_set_export from the dhis2 pack; the write and the archive land
+# on the s3:// scheme the dirigent-storage-s3 pack contributes. dirigent-core knows neither
+# pack -- only an assembled environment where both are installed can run this at all.
+#
+# Three hops, and what each one hands on:
+#
+#   export   dhis2.data_value_set_export. The set comes back as `body`, a value in the run.
+#   stage    storage.write, that value to a URI. A block does not write storage itself: a
+#            value leaves a run through this one door. Output: uri, bytes_written,
+#            content_type.
+#   archive  storage.copy, the object the write named onto a second, dated key. A copy moves
+#            bytes nobody has to look at, so it is bounded by the storage and not by the
+#            worker.
+#
+# Which connection backs the s3:// scheme is an instance setting (storage_connections:
+# {s3: <code>}), so the document carries only the dhis2 connection it names directly.
+#
+#   dg run --local examples/dhis2-export-to-s3.yaml
+
+format: dirigent/v1
+kind: pipeline
+code: dhis2-export-to-s3
+name: Export a data value set to S3
+description: Read a DHIS2 data value set, write it into object storage, then archive a dated copy.
+
+tags: [dhis2, s3, cross-boundary]
+
+requires:
+  blocks:
+    - dhis2.data_value_set_export
+    - storage.write
+    - storage.copy
+
+connections:
+  dhis2-demo:
+    kind: dhis2
+    config:
+      base_url: https://play.im.dhis2.org/stable-2-43-1
+      basic_username: admin
+      basic_password: district
+      timeout: 60s
+
+steps:
+  export:
+    block: dhis2.data_value_set_export
+    config:
+      connection: dhis2-demo
+      data_set: BfMAe6Itzgt
+      period: "202507"
+      org_unit: vSbt6cezomG
+  stage:
+    block: storage.write
+    depends_on: [export]
+    config:
+      target: s3://dirigent-exports/dhis2/BfMAe6Itzgt-202507.json
+      value: ${steps.export.output.body}
+  archive:
+    block: storage.copy
+    depends_on: [stage]
+    config:
+      # Where the write said the object landed, not the same key typed twice.
+      source: ${steps.stage.output.uri}
+      target: s3://dirigent-archive/dhis2/BfMAe6Itzgt-202507.json

dirigent_integration/shelves/dhis2-metadata-snapshot.yaml

@@ -0,0 +1,143 @@
+# A dated snapshot of the metadata a downstream system needs, and then the data that uses it.
+#
+# Every DHIS2 export is a pile of uids. What the uids mean -- which data element, which
+# facility, which category combination -- lives in metadata that changes underneath a
+# warehouse without warning. So a load that will still be readable in a year snapshots the
+# metadata beside the data, at the same moment, and this is that step. It spans dhis2 for the
+# reads, builtin for the reshape and the composition, and storage-s3 for where it lands, which
+# is why no single pack could own it.
+#
+# What happens, hop by hop:
+#
+#   collections  one dhis2.metadata read per resource named in params. The block resolves the
+#                instance's version on connect and reads through that version's accessor, so
+#                the same document reads a 2.42 or a 2.43 instance unchanged. Each answer is
+#                the collection under its own key: {"dataElements": [...]}.
+#   snapshot     the fan-out read back as one list and zipped against the resource names into a
+#                single document, {"dataElements": [...], "organisationUnits": [...]}.
+#   store        storage.write, that document as one object in s3. A jq step hands on a value
+#                and a value leaves a run through this one door, so the write is its own hop.
+#                One object, one fetch, everything a reader needs to resolve a uid.
+#   counts       the same list, counted: what each collection held on the day of the snapshot.
+#                This is the number worth alerting on -- a data element count that halved
+#                overnight is a broken credential, not a tidy-up.
+#   then_export  pipeline.run of dhis2-export-to-s3, the example beside this one. Composition
+#                is the point: the metadata is snapshotted first, the data is read second, and
+#                a reader of that data has the metadata that was true when it was read.
+#
+# To make it yours: change resources to the collections your consumers actually resolve, and
+# point pipeline at your own export. `wait: true` keeps this run open until the child settles,
+# so a failed export fails the snapshot run too; set it false to fire and forget. The child is
+# resolved by code on the instance, so it must be applied before this document runs -- which is
+# why it is not named in requires here, where a local validation holds no instance to check
+# against. Which connection backs s3:// is an instance setting (storage_connections: {s3: <code>}).
+#
+#   dg apply examples/dhis2-export-to-s3.yaml
+#   dg apply examples/dhis2-metadata-snapshot.yaml
+#   dg run dhis2-metadata-snapshot --watch
+
+format: dirigent/v1
+kind: pipeline
+code: dhis2-metadata-snapshot
+name: A metadata snapshot, then the data
+description: Snapshot DHIS2 metadata collections into S3, then run the export that uses them.
+
+tags: [dhis2, s3, composition, cross-boundary]
+
+requires:
+  blocks:
+    - dhis2.metadata
+    - transform.jq
+    - storage.write
+    - pipeline.run
+
+# The public DHIS2 play server, carried inline so this document runs standalone. An instance
+# you own names a connection it holds instead.
+connections:
+  dhis2-demo:
+    kind: dhis2
+    config:
+      base_url: https://play.im.dhis2.org/stable-2-43-1
+      basic_username: admin
+      basic_password: district
+      timeout: 60s
+
+params:
+  type: object
+  properties:
+    resources:
+      type: array
+      # DHIS2 collection names, spelled as they appear in the API path. These two are what a
+      # warehouse needs to make an export readable: what was measured, and where.
+      default: [dataElements, organisationUnits]
+      items:
+        type: string
+    fields:
+      type: string
+      default: id,name,shortName
+      description: The DHIS2 fields selector applied to every collection in the snapshot.
+    bucket:
+      type: string
+      default: dirigent-exports
+
+steps:
+  collections:
+    block: dhis2.metadata
+    for_each: ${params.resources}
+    config:
+      connection: dhis2-demo
+      resource: ${item}
+      fields: ${params.fields}
+      # Paging off: a snapshot wants the whole collection, and a half-read one is worse than
+      # no snapshot at all.
+      paging: false
+
+  snapshot:
+    block: transform.jq
+    depends_on: [collections]
+    config:
+      input:
+        resources: ${params.resources}
+        pages: ${steps.collections.output}
+      # The fan-out's output is its items' outputs in item order, so it lines up with the
+      # resource list positionally. That alignment is only safe because the step above has no
+      # `items: continue` -- a tolerated failure would drop an element and silently shift
+      # every collection one name to the left.
+      program: |
+        [.resources, [.pages[].body]]
+        | transpose
+        | map({(.[0]): (.[1][.[0]] // [])})
+        | add
+
+  store:
+    block: storage.write
+    depends_on: [snapshot]
+    config:
+      target: s3://${params.bucket}/dhis2/metadata/snapshot.json
+      value: ${steps.snapshot.output.value}
+
+  counts:
+    block: transform.jq
+    depends_on: [collections]
+    config:
+      input:
+        resources: ${params.resources}
+        pages: ${steps.collections.output}
+      program: |
+        [.resources, [.pages[].body]]
+        | transpose
+        | map({resource: .[0], rows: ((.[1][.[0]] // []) | length)})
+
+  then_export:
+    block: pipeline.run
+    # Both branches, so the run does not start the export before the snapshot it is supposed to
+    # be described by has actually landed.
+    depends_on: [store, counts]
+    config:
+      pipeline: dhis2-export-to-s3
+      # Waiting makes this run the whole story: one run id covers the metadata and the data,
+      # and a failed export is visible here rather than only in the child.
+      wait: true
+      # A child that finished completed_with_errors is a partial export, and a snapshot paired
+      # with a partial export should not report success.
+      strict: true

dirigent_integration/shelves/dhis2-tracker-weekly-window.yaml

@@ -0,0 +1,162 @@
+# A weekly tracker extract that reads the week that just closed, not the moment it started.
+#
+# A schedule answers "when does this start". A window answers "what does this run cover", and
+# for an extract that is the only question that matters: the Monday-morning run is meant to
+# collect last week, not everything the instance has ever recorded. Every schedule-fired run
+# carries the interval that just closed as ${run.window.start} up to but not including
+# ${run.window.end}, and consecutive firings tile the timeline with no overlap and no gap.
+#
+# Four packs meet here: dhis2 for the tracker read, builtin for the jq and the ndjson codec,
+# parquet for the encoding, storage-s3 for both destinations.
+#
+# What happens, hop by hop:
+#
+#   read      dhis2.tracker, kind: events, scoped to a programme and an org unit subtree, with
+#             updated_after set to the window's start. DHIS2 answers a page: the objects under
+#             `instances` and a `page` block beside them.
+#   in_window the upper bound. DHIS2's tracker read takes updated_after and nothing that means
+#             updated_before, so the open end of the window is applied here instead. Without
+#             it, an event edited while the run was in flight would be collected twice: once by
+#             this firing and again by the next one.
+#   staged    storage.write, the windowed events as one json object in s3://. A converter reads
+#             one URI and writes another, so a value the run is holding is put down first --
+#             and that write is the only door a value leaves a run by.
+#   as_ndjson convert.std json -> ndjson, beside it. One event per line is the format a reader
+#             streams rather than parses whole, and it is what the parquet encoder reads next.
+#   as_parquet  convert.arrow ndjson -> parquet, beside that. Parquet is bytes and never
+#             travels as a value: source and target are URIs and nothing is carried between
+#             them. Each converter's output names the target it wrote.
+#   archive   a dated second copy, keyed by the window rather than by the clock, so a backfill
+#             of an old week writes that week's key and not today's.
+#
+# To make it yours: change program and org_unit to your own, adjust the cron and the timezone,
+# and point bucket at a bucket you own. Which connection backs s3:// is an instance setting
+# (storage_connections: {s3: <code>}).
+#
+# The window is a property of the run, not of this document -- nothing below declares it, the
+# cadence computes it -- so an ad hoc run has to be handed one, and a run carrying no window
+# refuses the reference rather than quietly asking DHIS2 for everything it has:
+#
+#   dg run --local examples/dhis2-tracker-weekly-window.yaml --window 2026-06-01..2026-06-08
+#   dg apply examples/dhis2-tracker-weekly-window.yaml
+#   dg backfill dhis2-tracker-weekly-window --schedule weekly \
+#       --from 2026-01-05T03:00:00Z --to 2026-03-02T03:00:00Z --dry-run
+
+format: dirigent/v1
+kind: pipeline
+code: dhis2-tracker-weekly-window
+name: A week of tracker events, as parquet
+description: Read the tracker events of the week that just closed and archive them as parquet.
+
+tags: [dhis2, parquet, s3, schedule, cross-boundary]
+
+requires:
+  blocks:
+    - dhis2.tracker
+    - transform.jq
+    - storage.write
+    - convert.std
+    - convert.arrow
+    - storage.copy
+
+# The public DHIS2 play server, carried inline so this document runs standalone. An instance
+# you own names a connection it holds instead.
+connections:
+  dhis2-demo:
+    kind: dhis2
+    config:
+      base_url: https://play.im.dhis2.org/stable-2-43-1
+      basic_username: admin
+      basic_password: district
+      timeout: 60s
+
+params:
+  type: object
+  properties:
+    program:
+      type: string
+      default: IpHINAT79UW
+      description: The tracker programme to read; the default is the play server's child programme.
+    org_unit:
+      type: string
+      default: ImspTQPwCqd
+    bucket:
+      type: string
+      default: dirigent-exports
+
+steps:
+  read:
+    block: dhis2.tracker
+    timeout: 30m
+    config:
+      connection: dhis2-demo
+      kind: events
+      program: ${params.program}
+      org_unit: ${params.org_unit}
+      # The org unit is a root, so the read has to descend it; SELECTED would return the
+      # events recorded at the root itself, which is almost none of them.
+      ou_mode: DESCENDANTS
+      # The lower bound of the window, handed to DHIS2 so the instance does the filtering
+      # rather than sending a year of events across for us to discard.
+      updated_after: ${run.window.start}
+      fields: event,program,programStage,orgUnit,occurredAt,updatedAt,status
+      page_size: 500
+
+  in_window:
+    block: transform.jq
+    depends_on: [read]
+    config:
+      input:
+        events: ${steps.read.output.body.instances}
+        until: ${run.window.end}
+      # Half-open: >= start came from DHIS2, < end is applied here. Both ends written out is
+      # what stops an event that was edited during the run being collected twice.
+      program: |
+        .until as $until
+        | [.events[] | select(.updatedAt < $until)]
+
+  staged:
+    block: storage.write
+    depends_on: [in_window]
+    config:
+      target: s3://${params.bucket}/dhis2/tracker/${params.program}/${run.window.start}.json
+      value: ${steps.in_window.output.value}
+
+  as_ndjson:
+    block: convert.std
+    depends_on: [staged]
+    config:
+      source: ${steps.staged.output.uri}
+      from: json
+      to: ndjson
+      target: s3://${params.bucket}/dhis2/tracker/${params.program}/${run.window.start}.ndjson
+
+  as_parquet:
+    block: convert.arrow
+    depends_on: [as_ndjson]
+    config:
+      source: ${steps.as_ndjson.output.target}
+      from: ndjson
+      to: parquet
+      target: s3://${params.bucket}/dhis2/tracker/${params.program}/${run.window.start}.parquet
+
+  archive:
+    block: storage.copy
+    depends_on: [as_parquet]
+    config:
+      source: ${steps.as_parquet.output.target}
+      # Keyed by the window's start, so re-running or backfilling a week overwrites that week's
+      # archive object and nothing else. Keying it by the wall clock would file a backfill of
+      # January under the day the backfill happened to run.
+      target: s3://dirigent-archive/dhis2/tracker/${params.program}/${run.window.start}.parquet
+
+triggers:
+  schedules:
+    - code: weekly
+      name: Monday morning, Oslo time
+      description: Reads the week that closed at three on Monday morning.
+      cron: "0 3 * * 1"
+      # The zone belongs to the schedule, so the window it derives is a week of Oslo's clock
+      # rather than a fixed 168 hours: the daylight-saving weeks come out right without
+      # anything in the steps above knowing that daylight saving exists.
+      timezone: Europe/Oslo