dirigent-integration 0.23.2__py3-none-any.whl

dirigent_integration/shelves/inbound/http-json-to-data-values.yaml

@@ -0,0 +1,151 @@
+# The shortest honest inbound path: a public JSON API, one reshape, one gated import.
+#
+# The hops, and the shape at each one:
+#
+#   1. pull       -> the World Bank indicator API answers [metadata, [observations]]:
+#                    a two-element array whose second element is the rows, each row
+#                    {countryiso3code, date: "2023", value: 8791092, ...}
+#   2. reshape    -> {dataValues: [{dataElement, period, orgUnit, value}]}, which is the
+#                    envelope /api/dataValueSets takes. Every value is a STRING there,
+#                    even a count, which is why the program says tostring
+#   3. gate       -> the same envelope, refused unless every id is a real DHIS2 UID and
+#                    every period a real DHIS2 period
+#   4. import     -> the instance's own summary: status and the four counts
+#
+# What makes it yours: point source_url at your API, and set data_element and org_unit to
+# uids from your instance. The mapping in the reshape is the only part that knows both
+# worlds, and it is six lines -- which is the whole argument for keeping the pull and the
+# import as blocks rather than as code.
+#
+# The pairing is a real one, which is why the defaults are these: the World Bank's
+# population figure is yearly, and the demo's Total Population element sits in a yearly
+# data set assigned to Ngelehun CHC. A yearly figure sent at a monthly element is refused
+# by the instance, and no schema catches that -- only the instance holds the period type of
+# a data set, which is exactly what the dry run below is for.
+#
+# dry_run defaults to true, so a first run asks the instance to validate the document and
+# write nothing. The play server is shared, so leave it true there.
+#
+#   dg run --local examples/inbound/http-json-to-data-values.yaml
+#   dg run --local examples/inbound/http-json-to-data-values.yaml -p country=NGA
+
+format: dirigent/v1
+kind: pipeline
+code: http-json-to-data-values
+name: A public JSON API into DHIS2
+description: |
+  Pull a public JSON feed, reshape it into a DHIS2 data value set, gate it on DHIS2's own
+  id and period formats, and hand it to `dhis2.data_value_set_import` as a dry run.
+
+tags: [inbound, dhis2, http, transform, validate, cross-boundary]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+    - validate.schema
+    - dhis2.data_value_set_import
+
+connections:
+  dhis2-demo:
+    kind: dhis2
+    config:
+      base_url: https://play.im.dhis2.org/stable-2-43-1
+      basic_username: admin
+      basic_password: district
+      timeout: 60s
+
+params:
+  type: object
+  properties:
+    source_url:
+      type: string
+      description: The outside feed. Anything answering JSON works; the reshape is what changes.
+      default: https://api.worldbank.org/v2/country/SLE/indicator/SP.POP.TOTL
+    country:
+      type: string
+      description: The ISO3 code spliced into source_url when it is left at its default.
+      default: SLE
+    data_element:
+      type: string
+      description: The uid every pulled observation lands on; Total Population, yearly.
+      default: WUg3MYWQ7pt
+    org_unit:
+      type: string
+      description: The facility the figure is reported for, and one the data set is assigned to.
+      default: DiszpKrYNg8
+    dry_run:
+      type: boolean
+      description: Whether the instance validates the import and writes nothing.
+      default: true
+
+# Carried with the document, so a --local run needs nothing applied first. dhis2-uid and
+# dhis2-period are format checkers the dhis2 pack contributes: they turn "a string" into
+# "a string DHIS2 will actually accept", which is the difference between failing here and
+# failing halfway through an import.
+schemas:
+  data-value-set:
+    type: object
+    required: [dataValues]
+    properties:
+      dataValues:
+        type: array
+        minItems: 1
+        items:
+          type: object
+          required: [dataElement, period, orgUnit, value]
+          properties:
+            dataElement: { type: string, format: dhis2-uid }
+            period: { type: string, format: dhis2-period }
+            orgUnit: { type: string, format: dhis2-uid }
+            value: { type: string }
+
+steps:
+  # No connection, because the outside system is not a credential this instance holds: an
+  # absolute url is the honest form for a public endpoint. A feed behind a key becomes an
+  # http connection instead, and only this step changes.
+  pull:
+    block: http.request
+    config:
+      url: "${params.source_url}"
+      method: GET
+      query:
+        format: json
+        per_page: 5
+
+  # Both worlds meet in one program, so the mapping is readable in one place. The select
+  # drops years the API has no figure for: an absent observation must not become a zero,
+  # because zero is a measurement and absence is not.
+  reshape:
+    block: transform.jq
+    depends_on: [pull]
+    config:
+      input:
+        observations: "${steps.pull.output.body}"
+        data_element: "${params.data_element}"
+        org_unit: "${params.org_unit}"
+      program: |
+        .data_element as $de
+        | .org_unit as $ou
+        | {dataValues: [
+            .observations[1][]
+            | select(.value != null)
+            | {dataElement: $de, period: .date, orgUnit: $ou, value: (.value | tostring)}
+          ]}
+
+  # The gate is also the waypoint: everything downstream reads the gate's output rather
+  # than the reshape's, so the shape every later step received is the one that was checked.
+  gate:
+    block: validate.schema
+    depends_on: [reshape]
+    config:
+      input: "${steps.reshape.output.value}"
+      schema: data-value-set
+
+  import:
+    block: dhis2.data_value_set_import
+    depends_on: [gate]
+    config:
+      connection: dhis2-demo
+      data_values: "${steps.gate.output.value}"
+      dry_run: "${params.dry_run}"

dirigent_integration/shelves/inbound/outside-to-dhis2-with-checks.yaml

@@ -0,0 +1,246 @@
+# The defensive version: every guard an inbound pipeline earns, and what each one is for.
+#
+# The other examples on this shelf show one mechanism each. This one is what a pipeline
+# looks like once it has been in production for a month and every guard has been paid for
+# by an incident.
+#
+# The guards, in the order the run meets them, and the failure each one buys off:
+#
+#   1. reachable  -- an http.ready sensor before the pull. A source that is restarting
+#                    answers 502 for ninety seconds, and a pull that runs into it is a
+#                    failed run and a page at 03:00. A sensor waits instead: "not yet" is
+#                    the expected answer, and each poke is one cheap read. on_timeout: skip
+#                    ends the run as skipped rather than failed, because a source that was
+#                    down all night is not this pipeline being broken.
+#   2. raw_gate   -- a schema on what arrived, BEFORE anything reads it. This is the guard
+#                    people leave out, and it is the one that catches the real disaster: a
+#                    source that has quietly started answering an error page, an empty
+#                    envelope, or a renamed field. Without it, the reshape below produces a
+#                    beautifully-formed data value set full of nulls.
+#   3. value_gate -- a schema on what was produced, AFTER the reshape. The raw gate checked
+#                    the sender; this one checks us. A jq program is code, and code with no
+#                    test between it and a live instance is code nobody has checked.
+#   4. rehearse   -- the import with dry_run: true. DHIS2 validates the whole document,
+#                    against its own metadata, and reports what it would have refused --
+#                    which no schema can know, because only the instance holds the list of
+#                    uids that exist and the periods that are still open.
+#   5. commit     -- the same import for real, behind the default all_success edge. It runs
+#                    only because the rehearsal came back clean; a rehearsal that failed
+#                    leaves this step skipped, and nothing was written.
+#   6. signed_off -- the completeness sensor, which observes and does not register.
+#
+# THE TWO SCHEMAS ARE NOT THE SAME SHAPE and neither is redundant. One describes the
+# outside system's vocabulary, the other describes DHIS2's, and the reshape between them is
+# the only thing that knows both. When a run fails, which gate it failed at says whose
+# problem it is: the raw gate is the sender's, the value gate is ours, the rehearsal is the
+# instance's metadata.
+#
+# The commit step is a rehearsal too until you say otherwise: commit_dry_run defaults to
+# true so this document is safe against the shared play server, and the second import
+# becomes real by setting it false against an instance of your own.
+#
+#   dg run --local examples/inbound/outside-to-dhis2-with-checks.yaml
+#   dg run --local examples/inbound/outside-to-dhis2-with-checks.yaml -p commit_dry_run=false
+
+format: dirigent/v1
+kind: pipeline
+code: outside-to-dhis2-with-checks
+name: An inbound pipeline with every guard
+description: |
+  A readiness sensor, a schema on what arrived, a schema on what was built, a rehearsed
+  import, the real import behind it, and the completeness sensor -- each guard buying off a
+  different failure.
+
+tags: [inbound, dhis2, http, sensor, transform, validate, cross-boundary]
+
+requires:
+  blocks:
+    - http.ready
+    - http.request
+    - transform.jq
+    - validate.schema
+    - dhis2.data_value_set_import
+    - dhis2.data_set_complete
+
+connections:
+  dhis2-demo:
+    kind: dhis2
+    config:
+      base_url: https://play.im.dhis2.org/stable-2-43-1
+      basic_username: admin
+      basic_password: district
+      timeout: 60s
+  source:
+    kind: http
+    config:
+      base_url: https://api.worldbank.org
+      timeout: 30s
+
+params:
+  type: object
+  properties:
+    indicator_path:
+      type: string
+      description: The outside feed's path on the source connection.
+      default: /v2/country/SLE/indicator/SP.POP.TOTL
+    data_element:
+      type: string
+      description: The uid every pulled observation lands on; Total Population, yearly.
+      default: WUg3MYWQ7pt
+    org_unit:
+      type: string
+      description: The facility the figure is reported for, and one the data set is assigned to.
+      default: DiszpKrYNg8
+    data_set:
+      type: string
+      description: The data set whose sign-off closes the reporting year.
+      default: aLpVgfXiz0f
+    commit_dry_run:
+      type: boolean
+      description: Whether the second import is still a rehearsal. False makes it real.
+      default: true
+
+schemas:
+  # The sender's shape, in the sender's vocabulary. It asserts the two facts the reshape
+  # depends on and nothing more: there is at least one observation, and each one carries a
+  # date and either a number or an explicit null. A schema that also pinned the fields the
+  # reshape ignores would fail every time the source added a column.
+  source-observations:
+    type: array
+    minItems: 1
+    items:
+      type: object
+      required: [date]
+      properties:
+        date: { type: string }
+        value: { type: [number, "null"] }
+
+  # DHIS2's shape, in DHIS2's vocabulary, with the pack's own format checkers doing the
+  # work: dhis2-uid and dhis2-period turn "a string" into "a string this instance will
+  # accept", which is what makes the gate worth having over a required-fields check.
+  data-value-set:
+    type: object
+    required: [dataValues]
+    properties:
+      dataValues:
+        type: array
+        minItems: 1
+        items:
+          type: object
+          required: [dataElement, period, orgUnit, value]
+          properties:
+            dataElement: { type: string, format: dhis2-uid }
+            period: { type: string, format: dhis2-period }
+            orgUnit: { type: string, format: dhis2-uid }
+            value: { type: string }
+
+steps:
+  # A readiness probe is not the pull: it asks the cheapest question the source answers,
+  # and expect_status left empty means any 2xx counts.
+  reachable:
+    block: http.ready
+    poll: 30s
+    deadline: 30m
+    on_timeout: skip
+    config:
+      connection: source
+      # A sensor has no query block: readiness is one fixed question, so what varies about
+      # it belongs in the path.
+      path: /v2/country/SLE?format=json
+      # A source that answers 200 with a maintenance page is caught here rather than three
+      # steps later, because ready means "answering correctly", not "answering".
+      contains: Sierra Leone
+
+  pull:
+    block: http.request
+    depends_on: [reachable]
+    config:
+      connection: source
+      path: "${params.indicator_path}"
+      method: GET
+      query:
+        format: json
+        per_page: 5
+      # A source having a bad day can answer megabytes of HTML. The cap fails the step
+      # honestly instead of holding it all in the worker.
+      max_response: 4mb
+
+  # The envelope is [metadata, observations], so the gate is pointed at the half that
+  # matters: an index into an existing structure is unambiguous, and checking the rows says
+  # more than checking that an array has two elements.
+  raw_gate:
+    block: validate.schema
+    depends_on: [pull]
+    config:
+      input: "${steps.pull.output.body.1}"
+      schema: source-observations
+
+  # Reads the GATE's output, not the pull's. Both hold the same bytes, and only one of them
+  # has been checked -- so every step past a gate provably received the shape it asserts.
+  reshape:
+    block: transform.jq
+    depends_on: [raw_gate]
+    config:
+      input:
+        observations: "${steps.raw_gate.output.value}"
+        data_element: "${params.data_element}"
+        org_unit: "${params.org_unit}"
+      program: |
+        .data_element as $de
+        | .org_unit as $ou
+        | {dataValues: [
+            .observations[]
+            | select(.value != null)
+            | {dataElement: $de, period: .date, orgUnit: $ou, value: (.value | tostring)}
+          ]}
+
+  value_gate:
+    block: validate.schema
+    depends_on: [reshape]
+    config:
+      input: "${steps.reshape.output.value}"
+      schema: data-value-set
+
+  # The instance's own verdict, at no cost: it parses the document, checks every uid and
+  # period against its metadata, and writes nothing. Its summary comes back parsed, so a
+  # conflict is a named object and value rather than a wall of JSON.
+  rehearse:
+    block: dhis2.data_value_set_import
+    depends_on: [value_gate]
+    config:
+      connection: dhis2-demo
+      data_values: "${steps.value_gate.output.value}"
+      dry_run: true
+      # ALL is deliberate here and only here: a rehearsal that quietly tolerates bad rows
+      # tells you nothing, so the rehearsal is asked to refuse on any conflict at all.
+      atomic_mode: ALL
+
+  # No rule: is written, because the default all_success is exactly the guard wanted -- a
+  # rehearsal that failed leaves this step skipped and the instance untouched. Writing
+  # rule: all_done here would undo every guard above it in one line.
+  commit:
+    block: dhis2.data_value_set_import
+    depends_on: [rehearse]
+    config:
+      connection: dhis2-demo
+      # The same document the rehearsal was given, from the same waypoint. Rebuilding it
+      # here would mean the thing that was validated and the thing that is written are two
+      # different objects that merely ought to agree.
+      data_values: "${steps.value_gate.output.value}"
+      dry_run: "${params.commit_dry_run}"
+      import_strategy: CREATE_AND_UPDATE
+
+  signed_off:
+    block: dhis2.data_set_complete
+    depends_on: [commit]
+    poll: 30m
+    deadline: 72h
+    on_timeout: skip
+    config:
+      connection: dhis2-demo
+      data_set: "${params.data_set}"
+      # The most recent period the pull covered -- the source lists newest first -- read
+      # back off the checked document rather than named a second time: a period written
+      # twice is a period that will disagree once.
+      period: "${steps.value_gate.output.value.dataValues.0.period}"
+      org_unit: "${params.org_unit}"

dirigent_integration/shelves/inbound/parquet-lakehouse-to-dhis2.yaml

@@ -0,0 +1,158 @@
+# A lakehouse table aggregated down to what DHIS2 stores, then imported.
+#
+# The lakehouse and DHIS2 disagree about grain. A warehouse keeps one row per encounter,
+# typed and columnar, because analysis wants the detail; DHIS2's aggregate side keeps one
+# figure per data element, period and org unit, because that is what a report is. The whole
+# integration is that reduction, and it is one jq program.
+#
+# The hops, and the shape at each one:
+#
+#   1. decode     -> the parquet file re-encoded as a json object in the run's scratch space:
+#                    rows of {org_unit, period, element, count}, one per encounter, thousands
+#                    of them
+#   2. rows       -> that object read into the run as a value, bounded by max_size
+#   3. aggregate  -> {dataValues: [...]}, one value per (element, period, org unit) triple,
+#                    counts summed -- tens of rows, not thousands
+#   4. import     -> the instance's summary
+#   5. signed_off -> a sensor, and the only step here that does not act: it waits for a
+#                    person to mark the data set complete for that month
+#
+# Parquet is bytes and never travels as a value, so both ends of convert.arrow are URIs and
+# storage.read is what brings the decoded json in. THE REDUCTION HAPPENS AS SOON AS THE ROWS
+# ARE IN THE RUN, which is what keeps the run's memory bounded by the number of distinct
+# triples rather than by the number of encounters. A table too large to hold is aggregated in
+# the warehouse and exported already reduced; this pipeline is unchanged by that.
+#
+# WHY THE LAST STEP IS A SENSOR. `dhis2.data_set_complete` observes a completeness
+# registration; it does not make one. Reading it here closes the loop honestly: the import
+# put the figures in, and the run then holds until somebody with the authority to say so
+# has signed the month off, or skips when nobody does inside the deadline. A pipeline that
+# registered completeness itself would be asserting, on a machine's authority, that a
+# month's reporting is finished.
+#
+# What makes it yours: the warehouse export's uri and its column names, and the data set
+# the month is signed off against. The s3:// scheme needs an s3 connection bound to it as
+# an instance setting (storage_connections: {s3: <code>}).
+#
+#   dg run --local examples/inbound/parquet-lakehouse-to-dhis2.yaml -p period=202606
+
+format: dirigent/v1
+kind: pipeline
+code: parquet-lakehouse-to-dhis2
+name: A lakehouse table into DHIS2
+description: |
+  Read a parquet export of encounter-grain rows, aggregate it to DHIS2's grain, import it,
+  and wait for the month to be signed off.
+
+tags: [inbound, dhis2, parquet, storage, transform, cross-boundary]
+
+requires:
+  blocks:
+    - convert.arrow
+    - storage.read
+    - transform.jq
+    - dhis2.data_value_set_import
+    - dhis2.data_set_complete
+  storage:
+    - s3
+
+connections:
+  dhis2-demo:
+    kind: dhis2
+    config:
+      base_url: https://play.im.dhis2.org/stable-2-43-1
+      basic_username: admin
+      basic_password: district
+      timeout: 60s
+
+params:
+  type: object
+  properties:
+    bucket:
+      type: string
+      description: The bucket the warehouse exports into.
+      default: dirigent-lakehouse
+    period:
+      type: string
+      description: The DHIS2 month this export covers, and the object it is named as.
+      default: "202606"
+    data_set:
+      type: string
+      description: The data set whose sign-off closes the month.
+      default: Rl58JxmKJo2
+    org_unit:
+      type: string
+      description: The org unit the sign-off is read for; the values carry their own.
+      default: DiszpKrYNg8
+    dry_run:
+      type: boolean
+      description: Whether the instance validates the import and writes nothing.
+      default: true
+
+steps:
+  decode:
+    block: convert.arrow
+    config:
+      source: s3://${params.bucket}/encounters/${params.period}.parquet
+      from: parquet
+      to: json
+      # The run's own scratch space: the warehouse's object is left as it is, and these bytes
+      # exist only so the read below has json to decode.
+      target: ${run.scratch}/encounters-${params.period}.json
+
+  rows:
+    block: storage.read
+    depends_on: [decode]
+    config:
+      source: ${steps.decode.output.target}
+      # Parquet is compact and JSON is not, so the decoded object is several times the file.
+      # This is the number to raise when a month stops fitting, and the signal to aggregate
+      # upstream instead when raising it stops being reasonable: an object past it is refused
+      # rather than truncated into a wrong one.
+      max_size: 64mb
+
+  # The reduction. group_by needs its key sorted together, so the triple is built first and
+  # grouped on as a whole; the sum is over what each group holds. A count column that
+  # arrived as a string from the warehouse would sum as concatenation, hence tonumber.
+  aggregate:
+    block: transform.jq
+    depends_on: [rows]
+    config:
+      input: "${steps.rows.output.value}"
+      program: |
+        [.[] | {key: [.element, .period, .org_unit], count: (.count | tonumber)}]
+        | group_by(.key)
+        | {dataValues: [
+            .[]
+            | {dataElement: .[0].key[0],
+               period: .[0].key[1],
+               orgUnit: .[0].key[2],
+               value: ([.[].count] | add | tostring)}
+          ]}
+
+  import:
+    block: dhis2.data_value_set_import
+    depends_on: [aggregate]
+    config:
+      connection: dhis2-demo
+      data_values: "${steps.aggregate.output.value}"
+      dry_run: "${params.dry_run}"
+      # The warehouse is the source of truth for these figures, so a re-run of the same
+      # month must overwrite rather than refuse: an export corrected upstream is meant to
+      # land, and CREATE alone would leave the old figure standing.
+      import_strategy: CREATE_AND_UPDATE
+
+  # Each poke is one short read. on_timeout: skip is what makes the wait a report rather
+  # than a failure -- a month nobody has signed off by the deadline is a fact about the
+  # month, not a broken pipeline.
+  signed_off:
+    block: dhis2.data_set_complete
+    depends_on: [import]
+    poll: 15m
+    deadline: 72h
+    on_timeout: skip
+    config:
+      connection: dhis2-demo
+      data_set: "${params.data_set}"
+      period: "${params.period}"
+      org_unit: "${params.org_unit}"