dirigent-integration 0.23.2__py3-none-any.whl

dirigent_integration/shelves/inbound/webhook-payload-to-tracker-event.yaml

@@ -0,0 +1,176 @@
+# An event posted by an outside system, landing in DHIS2 the moment it is captured.
+#
+# The other examples on this shelf poll or wait. This one is pushed: a mobile capture app,
+# an EMR, a lab middleware POSTs one encounter to this pipeline's webhook and the run starts
+# with that encounter already in its parameters. Nothing is scheduled and nothing is
+# scanned.
+#
+# WHERE THIS LANDS, AND WHY IT IS NOT A TRACKER EVENT. An event-shaped capture is what
+# arrives, but `dhis2.tracker` in this pack READS `/api/tracker` -- tracked entities,
+# enrollments, events -- and has no write side, so there is nothing to hand an event to
+# today. The encounter is therefore summed into the aggregate figures it contributes to and
+# imported as a data value set, which is what an aggregate DHIS2 wants from an event stream
+# anyway. When a tracker write block lands, only the reshape and the import change; the
+# intake, the mapping table and the guard do not.
+#
+# The hops, and the shape at each one:
+#
+#   1. the POST     -> {encounter: {facility, occurred_at, items: [{code, quantity}]}}
+#   2. the mapping  -> three params, each one path into that body and nothing else
+#   3. reshape      -> {dataValues: [{dataElement, period, orgUnit, value}]}, one value per
+#                      distinct item code, quantities summed
+#   4. gate         -> the same, refused unless the ids and the period are DHIS2's own
+#   5. import       -> the instance's summary
+#
+# THE MAPPING IS THE WHOLE SECURITY BOUNDARY. A caller reaches exactly the three parameters
+# named under params_from_payload, and each mapped value is validated against the params
+# schema like any other run's. A payload carrying a connection code, a dry_run: false, or a
+# different org unit reaches none of them, because none of them is mapped.
+#
+# The period is derived rather than accepted. A capture knows when it happened; it does not
+# get to say which DHIS2 month it belongs to, because that is the receiving instance's
+# vocabulary and a caller that gets it wrong would silently write into the wrong month.
+#
+# Locally, the payload is supplied as parameters, which is exactly what a delivery does:
+#
+#   dg run --local examples/inbound/webhook-payload-to-tracker-event.yaml \
+#     -p facility=DiszpKrYNg8 -p occurred_at=2026-06-14T09:20:00Z
+#
+# Against an instance, where the endpoint exists:
+#
+#   dg apply examples/inbound/webhook-payload-to-tracker-event.yaml
+#   dg webhook rotate-token webhook-payload-to-tracker-event capture-posted
+#   curl -X POST "$DG_URL/hooks/$TOKEN" -d '{"encounter": {
+#     "facility": "DiszpKrYNg8", "occurred_at": "2026-06-14T09:20:00Z",
+#     "items": [{"code": "EXPOSED_REGISTERED", "quantity": 2}]}}'
+
+format: dirigent/v1
+kind: pipeline
+code: webhook-payload-to-tracker-event
+name: A pushed encounter into DHIS2
+description: |
+  An outside system POSTs one captured encounter; the run reshapes it into a DHIS2 data
+  value set and imports it.
+
+  The pack's `dhis2.tracker` block reads `/api/tracker` and does not write it, so an
+  event-shaped capture lands as the aggregate figures it contributes to.
+
+tags: [inbound, dhis2, webhook, transform, validate, cross-boundary]
+
+requires:
+  blocks:
+    - transform.jq
+    - validate.schema
+    - dhis2.data_value_set_import
+
+connections:
+  dhis2-demo:
+    kind: dhis2
+    config:
+      base_url: https://play.im.dhis2.org/stable-2-43-1
+      basic_username: admin
+      basic_password: district
+      timeout: 60s
+
+params:
+  type: object
+  properties:
+    facility:
+      type: string
+      description: The uid of the facility the encounter was captured at.
+      default: DiszpKrYNg8
+    occurred_at:
+      type: string
+      format: date-time
+      description: When the encounter happened, which is what the DHIS2 period is derived from.
+      default: "2026-06-14T09:20:00Z"
+    items:
+      type: array
+      description: What was given or done, in the sending system's own item codes.
+      default:
+        - { code: EXPOSED_REGISTERED, quantity: 2 }
+        - { code: NVP_WITHIN_72H, quantity: 1 }
+        - { code: EXPOSED_REGISTERED, quantity: 1 }
+      items:
+        type: object
+        required: [code, quantity]
+        properties:
+          code: { type: string }
+          quantity: { type: integer }
+    dry_run:
+      type: boolean
+      description: Whether the instance validates the import and writes nothing.
+      default: true
+
+schemas:
+  data-value-set:
+    type: object
+    required: [dataValues]
+    properties:
+      dataValues:
+        type: array
+        minItems: 1
+        items:
+          type: object
+          required: [dataElement, period, orgUnit, value]
+          properties:
+            dataElement: { type: string, format: dhis2-uid }
+            period: { type: string, format: dhis2-period }
+            orgUnit: { type: string, format: dhis2-uid }
+            value: { type: string }
+
+steps:
+  # One encounter can carry the same item twice -- two children registered in one visit --
+  # so the items are grouped and summed before they become data values. Importing them as
+  # two values would have the second silently replace the first rather than add to it.
+  reshape:
+    block: transform.jq
+    config:
+      input:
+        facility: "${params.facility}"
+        occurred_at: "${params.occurred_at}"
+        items: "${params.items}"
+        lookup:
+          EXPOSED_REGISTERED: x0PshcPLSk1
+          NVP_WITHIN_72H: wZqi8EXN5x4
+      program: |
+        .lookup as $lookup
+        | .facility as $ou
+        # A monthly DHIS2 period is YYYYMM, which is the first seven characters of an ISO
+        # instant with the dash taken out. The capture's own clock decides the month.
+        | (.occurred_at[0:7] | gsub("-"; "")) as $period
+        | {dataValues: [
+            .items
+            | group_by(.code)[]
+            | {dataElement: $lookup[.[0].code],
+               period: $period,
+               orgUnit: $ou,
+               value: ([.[].quantity] | add | tostring)}
+          ]}
+
+  # An item code the table does not know left dataElement null, and null is not a uid: the
+  # gate is what turns a sender's typo into a failed run instead of a refused import.
+  gate:
+    block: validate.schema
+    depends_on: [reshape]
+    config:
+      input: "${steps.reshape.output.value}"
+      schema: data-value-set
+
+  import:
+    block: dhis2.data_value_set_import
+    depends_on: [gate]
+    config:
+      connection: dhis2-demo
+      data_values: "${steps.gate.output.value}"
+      dry_run: "${params.dry_run}"
+
+triggers:
+  webhooks:
+    - code: capture-posted
+      name: One encounter, pushed
+      description: The capture app POSTs an encounter as it is recorded.
+      params_from_payload:
+        facility: "$.encounter.facility"
+        occurred_at: "$.encounter.occurred_at"
+        items: "$.encounter.items"

dirigent_integration/shelves/inbound/weekly-window-pull-and-import.yaml

@@ -0,0 +1,175 @@
+# A weekly pull bounded by the run's own window, not by "now".
+#
+# THE WINDOW IS THE POINT. A schedule says when the run starts; the window says what the run
+# covers. Monday's run is meant to read the week that just closed, and it must read exactly
+# that week whether it fires on time, fires an hour late after a worker restart, or is
+# backfilled next year. ${run.window.start} and ${run.window.end} are that interval, and
+# they are half-open -- start included, end excluded -- so consecutive firings tile the
+# timeline with no overlap and no gap. A pull bounded by "now minus seven days" has neither
+# property.
+#
+# Nothing in this document declares the window: the cadence at the bottom computes it. So an
+# ad hoc run has to be given one, and a run carrying none refuses the reference rather than
+# resolving it to nothing and asking the outside system for its entire history.
+#
+# The hops, and the shape at each one:
+#
+#   1. pull       -> the USGS earthquake catalogue, a GeoJSON FeatureCollection:
+#                    {type, metadata, features: [...]} -- one feature per event in the week
+#   2. reshape    -> {dataSet, dataValues: [{dataElement, period, orgUnit, value}]}: one
+#                    value, the week's event count, filed against the ISO week the window
+#                    opened in
+#   3. import     -> the instance's summary
+#   4. signed_off -> the sensor that waits for a person to sign the week off
+#
+# A public catalogue that takes an explicit time range is what makes this runnable with no
+# setup; the mechanism is the same for a district register or a lab system that takes
+# ?from=&to=. The catalogue's event counts stand in for a surveillance system's weekly case
+# counts -- that pairing is a stand-in and nothing else here is.
+#
+# A count of zero is a measurement and is imported as one -- the week genuinely had no
+# qualifying event. That is the opposite of the empty cell in csv-drop-to-data-values.yaml,
+# where nothing was reported at all.
+#
+#   dg run --local examples/inbound/weekly-window-pull-and-import.yaml \
+#     --window 2026-08-03..2026-08-10
+#   dg apply examples/inbound/weekly-window-pull-and-import.yaml
+#   dg backfill weekly-window-pull-and-import --schedule monday-morning \
+#     --from 2026-06-01T06:00:00Z --to 2026-08-31T06:00:00Z --dry-run
+
+format: dirigent/v1
+kind: pipeline
+code: weekly-window-pull-and-import
+name: A weekly windowed pull into DHIS2
+description: |
+  A Monday schedule whose runs pull the week that just closed -- `${run.window.start}` up
+  to but not including `${run.window.end}` -- reshape it into a data value set, import it,
+  and wait for the week to be signed off.
+
+tags: [inbound, dhis2, http, schedule, transform, cross-boundary]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+    - dhis2.data_value_set_import
+    - dhis2.data_set_complete
+
+connections:
+  dhis2-demo:
+    kind: dhis2
+    config:
+      base_url: https://play.im.dhis2.org/stable-2-43-1
+      basic_username: admin
+      basic_password: district
+      timeout: 60s
+  catalogue:
+    kind: http
+    config:
+      base_url: https://earthquake.usgs.gov
+      timeout: 60s
+
+params:
+  type: object
+  properties:
+    min_magnitude:
+      type: number
+      description: What counts as an event worth reporting; the schedule pins it per firing.
+      default: 5
+    data_element:
+      type: string
+      description: The uid the weekly count lands on; a weekly surveillance element.
+      default: UsSUX0cpKsH
+    org_unit:
+      type: string
+      description: The facility the weekly count is reported for.
+      default: DiszpKrYNg8
+    data_set:
+      type: string
+      description: The weekly data set the count belongs to, and whose sign-off closes it.
+      default: Nyh6laLdBEJ
+    dry_run:
+      type: boolean
+      description: Whether the instance validates the import and writes nothing.
+      default: true
+
+steps:
+  # Both ends of the interval go on the wire, because the outside system is the one that
+  # must not double-count: >= start and < end is the same half-open rule the window is.
+  pull:
+    block: http.request
+    config:
+      connection: catalogue
+      path: /fdsnws/event/1/query
+      method: GET
+      query:
+        format: geojson
+        starttime: "${run.window.start}"
+        endtime: "${run.window.end}"
+        minmagnitude: "${params.min_magnitude}"
+
+  # A DHIS2 weekly period is YYYYWnn, and getting there from an instant has two traps.
+  # strptime alone leaves the weekday and year-day fields empty, so the week number comes
+  # out wrong unless the parse is pushed through mktime|gmtime first; and DHIS2 writes W6,
+  # not W06, so the zero-padded %V is put through tonumber. The date is truncated to ten
+  # characters so the same program reads both an instant and a plain date.
+  reshape:
+    block: transform.jq
+    depends_on: [pull]
+    config:
+      input:
+        events: "${steps.pull.output.body}"
+        window_start: "${run.window.start}"
+        data_element: "${params.data_element}"
+        org_unit: "${params.org_unit}"
+        data_set: "${params.data_set}"
+      program: |
+        (.window_start[0:10] | strptime("%Y-%m-%d") | mktime | gmtime) as $opened
+        # The envelope names its data set because this element belongs to two of them, and
+        # an import that cannot tell which set a value is for refuses the whole document.
+        | {dataSet: .data_set,
+           dataValues: [{
+            dataElement: .data_element,
+            period: "\($opened | strftime("%G"))W\($opened | strftime("%V") | tonumber)",
+            orgUnit: .org_unit,
+            value: (.events.features | length | tostring)
+          }]}
+
+  import:
+    block: dhis2.data_value_set_import
+    depends_on: [reshape]
+    config:
+      connection: dhis2-demo
+      data_values: "${steps.reshape.output.value}"
+      dry_run: "${params.dry_run}"
+      # A backfilled week must land on top of whatever a previous run put there, because
+      # the catalogue revises events after the fact and the newer read is the truer one.
+      import_strategy: CREATE_AND_UPDATE
+
+  # Observes, never registers -- the same sensor, and the same reasoning, as
+  # parquet-lakehouse-to-dhis2.yaml. The period is read back off the document that was
+  # imported rather than derived a second time: one derivation, one truth.
+  signed_off:
+    block: dhis2.data_set_complete
+    depends_on: [import]
+    poll: 30m
+    deadline: 72h
+    on_timeout: skip
+    config:
+      connection: dhis2-demo
+      data_set: "${params.data_set}"
+      period: "${steps.reshape.output.value.dataValues.0.period}"
+      org_unit: "${params.org_unit}"
+
+triggers:
+  schedules:
+    - code: monday-morning
+      name: Monday, Freetown time
+      description: Reads the week that closed at six on Monday morning.
+      cron: "0 6 * * 1"
+      # The zone belongs to the schedule, so the window it derives is a week of the
+      # reporting country's clock rather than a fixed 168 hours from a server's idea of
+      # midnight.
+      timezone: Africa/Freetown
+      params:
+        min_magnitude: 5

dirigent_integration/shelves/parquet-to-dhis2-import.yaml

@@ -0,0 +1,195 @@
+# The other direction: a parquet file somebody drops in a bucket, ending up inside DHIS2.
+#
+# dhis2-values-to-parquet.yaml reads DHIS2 and writes parquet. This is the return leg, and it
+# is the one that needs a gate: bytes arriving from outside are not a data value set until
+# something has checked that they are. Three packs meet here -- storage-s3 for the drop,
+# parquet for the decode, builtin for the reshape and the schema gate, dhis2 for the write --
+# which is why it lives in the integration and in none of them.
+#
+# What happens, hop by hop:
+#
+#   dropped   storage.exists is a sensor: each poke is one cheap head request, "not yet" is
+#             the expected answer, and the run parks rather than holding a worker. min_size
+#             is what stops it opening a half-written object a producer is still uploading.
+#   decode    convert.arrow parquet -> json, into the run's scratch space. Parquet is bytes and
+#             never travels as a value, so both ends of a conversion are URIs: source and
+#             target. The json it writes is an array of row objects, one per data value,
+#             because that is what a parquet table is -- rows and columns, with no envelope.
+#   load      storage.read, that json object as a value. storage.read is the one door a value
+#             comes in by, and max_size is how much of it this step will hold: an object past
+#             it is refused rather than read half way.
+#   shape     the list is wrapped: DHIS2 wants {"dataValues": [...]}, the file carries the bare
+#             list. This is the whole impedance mismatch between a columnar file and a DHIS2
+#             payload, and it is one line of jq.
+#   gate      validate.schema against the carried schema below. Every dataElement and orgUnit
+#             must be a real DHIS2 uid and every period a real DHIS2 period, checked by the
+#             dhis2-uid and dhis2-period formats the dhis2 pack contributes -- so a typo is a
+#             refused run and not a partially-written instance. The gate has no `rule`, so the
+#             default all_success applies: the import below simply never starts if this fails.
+#   import    dhis2.data_value_set_import of the value the gate handed on. atomic_mode ALL
+#             means one conflicted value refuses the whole set, which is the right default for
+#             a file: half an import is worse than none.
+#   signed_off  dhis2.data_set_complete is a sensor, not a writer -- it waits for a human in
+#             DHIS2 to mark the set complete for that period and org unit. That is the honest
+#             end of a load: the data is in, and the run stays open until somebody stands
+#             behind it. on_timeout: skip ends the run cleanly when nobody does.
+#
+# To make it yours: point drop_uri at the object your producer writes, name your own data set,
+# period and org unit, and turn dry_run off once you are importing into an instance you own.
+# It is on by default because the play server is shared: the import is fully exercised and
+# nothing is written. Which connection backs s3:// is an instance setting
+# (storage_connections: {s3: <code>}).
+#
+#   dg run --local examples/parquet-to-dhis2-import.yaml
+#   dg run --local examples/parquet-to-dhis2-import.yaml -p dry_run=false
+
+format: dirigent/v1
+kind: pipeline
+code: parquet-to-dhis2-import
+name: A parquet drop imported into DHIS2
+description: Wait for a parquet object, check it, then import it as a DHIS2 data value set.
+
+tags: [dhis2, parquet, s3, validate, cross-boundary]
+
+requires:
+  blocks:
+    - storage.exists
+    - convert.arrow
+    - storage.read
+    - transform.jq
+    - validate.schema
+    - dhis2.data_value_set_import
+    - dhis2.data_set_complete
+
+# The public DHIS2 play server, carried inline so this document runs standalone. Against an
+# instance you own, name a connection that instance holds instead of carrying one.
+connections:
+  dhis2-demo:
+    kind: dhis2
+    config:
+      base_url: https://play.im.dhis2.org/stable-2-43-1
+      basic_username: admin
+      basic_password: district
+      timeout: 60s
+
+# Carried in the document, so the gate needs nothing handed to it and a --local run checks the
+# same shape a server would. dhis2-uid and dhis2-period are formats the dhis2 pack contributes
+# to the catalog; an instance without that pack cannot run this document at all, which is the
+# point of naming them in requires.
+schemas:
+  dhis2-data-value-set:
+    type: object
+    required: [dataValues]
+    additionalProperties: false
+    properties:
+      dataValues:
+        type: array
+        minItems: 1
+        items:
+          type: object
+          required: [dataElement, period, orgUnit, value]
+          properties:
+            dataElement: { type: string, format: dhis2-uid }
+            period: { type: string, format: dhis2-period }
+            orgUnit: { type: string, format: dhis2-uid }
+            categoryOptionCombo: { type: string, format: dhis2-uid }
+            attributeOptionCombo: { type: string, format: dhis2-uid }
+            value: { type: string }
+            comment: { type: string }
+
+params:
+  type: object
+  properties:
+    drop_uri:
+      type: string
+      default: s3://dirigent-incoming/dhis2/data-values.parquet
+      description: The object the upstream producer writes; this run starts when it appears.
+    data_set:
+      type: string
+      default: BfMAe6Itzgt
+    period:
+      type: string
+      default: "202507"
+    org_unit:
+      type: string
+      default: vSbt6cezomG
+    dry_run:
+      type: boolean
+      default: true
+      description: On by default because the play server is shared; off on an instance you own.
+
+steps:
+  dropped:
+    block: storage.exists
+    poll: 1m
+    deadline: 12h
+    # Nobody dropped a file inside the window: that is a quiet day, not a broken pipeline.
+    on_timeout: skip
+    config:
+      uri: ${params.drop_uri}
+      # A producer uploading a large object makes the key visible before the bytes are all
+      # there. A floor stops the decode reading a truncated file.
+      min_size: 1kb
+
+  decode:
+    block: convert.arrow
+    depends_on: [dropped]
+    config:
+      source: ${params.drop_uri}
+      from: parquet
+      to: json
+      # The run's own scratch space: these bytes exist only so the read below has something to
+      # decode, and the object the producer dropped is left exactly as it was.
+      target: ${run.scratch}/data-values.json
+
+  load:
+    block: storage.read
+    depends_on: [decode]
+    config:
+      # Where the conversion said it wrote, not the same path typed twice.
+      source: ${steps.decode.output.target}
+      # A data value set that fits a single import call also fits a step output; a file bigger
+      # than this is refused here rather than truncated into a wrong one.
+      max_size: 64mb
+
+  shape:
+    block: transform.jq
+    depends_on: [load]
+    config:
+      input: ${steps.load.output.value}
+      # The object literal is the envelope DHIS2 requires and a parquet table has no room for.
+      program: |
+        {dataValues: .}
+
+  gate:
+    block: validate.schema
+    depends_on: [shape]
+    config:
+      input: ${steps.shape.output.value}
+      schema: dhis2-data-value-set
+
+  import:
+    block: dhis2.data_value_set_import
+    depends_on: [gate]
+    config:
+      connection: dhis2-demo
+      # Read from the gate rather than from shape: the reference is what proves, in the
+      # document itself, that nothing reaches DHIS2 without passing the schema.
+      data_values: ${steps.gate.output.value}
+      dry_run: ${params.dry_run}
+      import_strategy: CREATE_AND_UPDATE
+      # One refused value refuses the whole file. A partially imported month is harder to
+      # reason about than a failed run, and the file can simply be dropped again.
+      atomic_mode: ALL
+
+  signed_off:
+    block: dhis2.data_set_complete
+    depends_on: [import]
+    poll: 30m
+    deadline: 72h
+    on_timeout: skip
+    config:
+      connection: dhis2-demo
+      data_set: ${params.data_set}
+      period: ${params.period}
+      org_unit: ${params.org_unit}