dirigent-integration 0.23.2__py3-none-any.whl

dirigent_integration/shelves/inbound/csv-drop-to-data-values.yaml

@@ -0,0 +1,176 @@
+# A csv dropped in a bucket, picked up and imported: the file-drop integration.
+#
+# The commonest inbound shape there is. Somebody's system -- a district DHIS2-less register,
+# a lab machine, an export somebody mails weekly -- writes a csv into object storage, and
+# nothing tells the pipeline it happened. So the pipeline waits.
+#
+# The hops, and the shape at each one:
+#
+#   1. arrived  -> a sensor, not a read: it answers only once an object matching the glob is
+#                  there and at least min_size, and hands on that object's uri
+#   2. decode   -> the csv re-encoded as a json object in the run's scratch space: a
+#                  conversion reads a URI and writes a URI, and carries nothing between them
+#   3. table    -> that object read into the run as a value, which is the one door a value
+#                  comes in by
+#   4. rows     -> the table with each row carrying the data element uid the lookup gave it,
+#                  so the code-to-uid translation is done once and visibly
+#   5. values   -> one data value per row, DHIS2's four fields and nothing else
+#   6. kept     -> the same list minus the rows that carry no measurement
+#   7. import   -> the instance's summary
+#
+# The lookup is the part every deployment rewrites. It sits in its own value.const step
+# because that is where a reader looks for it: the sending system's column codes on the
+# left, this instance's uids on the right, and a row whose code is not in the table falls
+# out at the filter rather than importing under a null id.
+#
+# THE FILTER IS NOT TIDYING. An empty cell means the facility did not report, and a 0 means
+# it reported none. Importing the first as the second invents data, so the empty ones are
+# dropped and the gap stays a gap.
+#
+# What makes it yours: the bucket and prefix, the lookup table, and the period column. The
+# s3:// scheme needs an s3 connection bound to it as an instance setting
+# (storage_connections: {s3: <code>}), exactly as examples/s3 in the dirigent repository
+# sets up; the drop directory can equally be file:// on a worker's disk.
+#
+#   dg run --local examples/inbound/csv-drop-to-data-values.yaml -p day=2026-06-01
+
+format: dirigent/v1
+kind: pipeline
+code: csv-drop-to-data-values
+name: A csv drop into DHIS2
+description: |
+  Wait for a csv to land in object storage, decode it, translate the sender's codes into
+  DHIS2 uids, drop the rows that carry no measurement, and import what is left.
+
+tags: [inbound, dhis2, storage, transform, cross-boundary]
+
+requires:
+  blocks:
+    - storage.exists
+    - convert.std
+    - storage.read
+    - value.const
+    - transform.jq
+    - map.jq
+    - filter.jq
+    - dhis2.data_value_set_import
+  storage:
+    - s3
+
+connections:
+  dhis2-demo:
+    kind: dhis2
+    config:
+      base_url: https://play.im.dhis2.org/stable-2-43-1
+      basic_username: admin
+      basic_password: district
+      timeout: 60s
+
+params:
+  type: object
+  required: [day]
+  properties:
+    day:
+      type: string
+      format: date
+      description: The drop directory the sender writes into, one per day.
+    bucket:
+      type: string
+      description: The bucket the sending system drops into.
+      default: dirigent-inbound
+    dry_run:
+      type: boolean
+      description: Whether the instance validates the import and writes nothing.
+      default: true
+
+steps:
+  # A glob in the final segment, because the sender names the file and we do not. min_size
+  # is the guard against a half-written object: a multipart upload is visible before it is
+  # complete, and a zero-byte placeholder is not a csv.
+  arrived:
+    block: storage.exists
+    poll: 1m
+    deadline: 6h
+    on_timeout: skip
+    config:
+      uri: s3://${params.bucket}/drops/${params.day}/*.csv
+      min_size: 1b
+
+  # The drop the sender wrote is left exactly as it is: the conversion reads it and writes
+  # the json beside it in the run's own scratch space.
+  decode:
+    block: convert.std
+    depends_on: [arrived]
+    config:
+      source: "${steps.arrived.output.uri}"
+      from: csv
+      to: json
+      target: "${run.scratch}/drop-${params.day}.json"
+
+  # The read is where the drop enters the run, and max_size is the ceiling on what one drop
+  # may be: an object past it is refused rather than half read.
+  table:
+    block: storage.read
+    depends_on: [decode]
+    config:
+      source: "${steps.decode.output.target}"
+      max_size: 16mb
+
+  # The sending system's vocabulary, translated. One place, one step, one thing to hand a
+  # DHIS2 administrator when a new indicator is added.
+  lookup:
+    block: value.const
+    config:
+      value:
+        PMTCT_EXPOSED_REGISTERED: x0PshcPLSk1
+        PMTCT_NVP_72H: wZqi8EXN5x4
+
+  # The uid is attached here, while the whole table is in scope; the per-row verbs below
+  # cannot see the lookup, because a map program sees one element and nothing else.
+  rows:
+    block: transform.jq
+    depends_on: [table, lookup]
+    config:
+      input:
+        csv: "${steps.table.output.value}"
+        lookup: "${steps.lookup.output.value}"
+      program: |
+        .lookup as $lookup
+        | .csv
+        | [.[] | . + {dataElement: $lookup[.element_code]}]
+
+  # Exactly one data value per row, and the list is as long as the one above it -- which is
+  # what map.jq guarantees and a transform.jq program would not.
+  values:
+    block: map.jq
+    depends_on: [rows]
+    config:
+      input: "${steps.rows.output.value}"
+      program: |
+        {dataElement, period: .period, orgUnit: .org_unit, value: .value}
+
+  # A subset, nothing edited. Two ways in: an unmapped code left dataElement null, and an
+  # empty cell left value "". jq truthiness is not applied here, so both tests are written
+  # out rather than leaning on a bare .value.
+  kept:
+    block: filter.jq
+    depends_on: [values]
+    config:
+      input: "${steps.values.output.value}"
+      program: |
+        .dataElement != null and .value != "" and .value != null
+
+  import:
+    block: dhis2.data_value_set_import
+    depends_on: [kept]
+    config:
+      connection: dhis2-demo
+      # The envelope is assembled here rather than in a step of its own: the list is the
+      # only part any earlier step had an opinion about.
+      data_values:
+        dataValues: "${steps.kept.output.value}"
+      dry_run: "${params.dry_run}"
+      # NONE takes the rows the instance accepts and reports the rest as conflicts. ALL,
+      # the default, refuses the whole file over one bad row -- the right choice when the
+      # drop is one facility's month and the wrong one when it is a district's backlog.
+      atomic_mode: NONE

dirigent_integration/shelves/inbound/fan-out-per-facility-import.yaml

@@ -0,0 +1,202 @@
+# One pull per facility, one import for the district, and a report of who did not answer.
+#
+# A district collects from facilities that are up at different times and run different
+# software. The pull therefore fans out -- one call per facility, each its own attempt --
+# and the import does not.
+#
+# WHY THE FAN CONVERGES AT THE IMPORT. Two reasons, and both are worth knowing:
+#
+#   for_each is expanded when the run is created, so it may read params, run and item but
+#   never another step's output: the cardinality has to exist before anything executes. A
+#   step fanned over "the facilities that answered" is therefore not expressible, and
+#   should not be -- it would make the shape of the run depend on the weather.
+#
+#   A data value set carries orgUnit per value, so one document can hold every facility at
+#   once. Fifteen imports of one facility each would be fifteen round trips, fifteen
+#   summaries to reconcile, and no gain.
+#
+# The hops, and the shape at each one:
+#
+#   1. pull        -> fanned: one http.request per facility. The step's output is the LIST
+#                     of its items' outputs, in item order, and A FAILED ITEM IS ABSENT
+#                     FROM IT -- not null, absent. So the list is shorter than the facility
+#                     list exactly when something went wrong
+#   2. per_facility-> fanned the same way, over the same params: one facility's rows,
+#                     already carrying its orgUnit
+#   3. batch       -> {dataValues: [...]}, every facility's values in one envelope
+#   4. import      -> one summary for the district
+#   5. reconcile   -> asked for, answered, missing -- computed from the two lists rather
+#                     than from anything the import said
+#   6. notify      -> that reconciliation POSTed where a person will see it
+#
+# items: continue is what makes a facility that is down a line in the report instead of the
+# end of the run. Under the default, fail_fast, one unreachable facility would take the
+# other fourteen with it.
+#
+# The step after a tolerated fan-out still runs, because a fan-out that lost items counts
+# as succeeded and the run ends completed_with_errors. If EVERY facility fails, the step
+# failed, and everything downstream is skipped -- including the reconciliation, which is
+# the one case where the run's own status is the report.
+#
+# What makes it yours: the facilities list, the endpoint each one answers on, and where the
+# reconciliation is posted.
+#
+#   dg run --local examples/inbound/fan-out-per-facility-import.yaml
+#   dg run --local examples/inbound/fan-out-per-facility-import.yaml \
+#     -p facilities='[{"code":"ngelehun-chc","org_unit":"DiszpKrYNg8","reported":42}]'
+
+format: dirigent/v1
+kind: pipeline
+code: fan-out-per-facility-import
+name: A facility fan-out into one district import
+description: |
+  Pull one facility at a time, tolerate the ones that are down, import what answered as a
+  single data value set, and post a reconciliation of asked-for against imported.
+
+tags: [inbound, dhis2, http, graph, transform, cross-boundary]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+    - dhis2.data_value_set_import
+    - webhook.post
+
+connections:
+  dhis2-demo:
+    kind: dhis2
+    config:
+      base_url: https://play.im.dhis2.org/stable-2-43-1
+      basic_username: admin
+      basic_password: district
+      timeout: 60s
+
+params:
+  type: object
+  properties:
+    facilities:
+      type: array
+      description: The facilities to collect from; each carries the uid its values land on.
+      default:
+        - { code: ngelehun-chc, org_unit: DiszpKrYNg8, reported: 42 }
+        - { code: gbenikoro-mchp, org_unit: y77LiPqLMoq, reported: 17 }
+        - { code: dodo-kortuma-chp, org_unit: rwfuVQHnZJ5, reported: 8 }
+      items:
+        type: object
+        required: [code, org_unit, reported]
+        properties:
+          code: { type: string }
+          org_unit: { type: string }
+          # What that facility's register would answer. It rides in the request and comes
+          # back in the echo, so the example has a figure to import without pretending a
+          # public endpoint holds anyone's health data.
+          reported: { type: integer }
+    period:
+      type: string
+      description: The DHIS2 month being collected.
+      default: "202606"
+    data_element:
+      type: string
+      description: The uid each facility's figure lands on.
+      default: x0PshcPLSk1
+    report_url:
+      type: string
+      description: Where the reconciliation is POSTed.
+      default: https://postman-echo.com/post
+    dry_run:
+      type: boolean
+      description: Whether the instance validates the import and writes nothing.
+      default: true
+
+steps:
+  # ${item} is one element of the params list, so ${item.code} reaches into it. The facility
+  # endpoint stands in as an echo service that answers whatever it is asked -- the shape
+  # this step teaches is one call per facility, not what that call returns.
+  pull:
+    block: http.request
+    for_each: "${params.facilities}"
+    items: continue
+    config:
+      url: https://postman-echo.com/get
+      method: GET
+      query:
+        facility: "${item.code}"
+        period: "${params.period}"
+        reported: "${item.reported}"
+
+  # Fanned over the same params list, so item and the pull's list line up index for index
+  # -- while every item succeeded. That is exactly why the reconciliation below counts the
+  # pull's own output rather than assuming this step saw everything.
+  per_facility:
+    block: transform.jq
+    depends_on: [pull]
+    for_each: "${params.facilities}"
+    items: continue
+    config:
+      input:
+        facility: "${item}"
+        answers: "${steps.pull.output}"
+        period: "${params.period}"
+        data_element: "${params.data_element}"
+      program: |
+        . as {$facility, $answers, $period, $data_element}
+        | [$answers[] | select(.body.args.facility == $facility.code)] as $mine
+        | [$mine[]
+           | {dataElement: $data_element,
+              period: $period,
+              orgUnit: $facility.org_unit,
+              value: (.body.args.reported | tostring)}]
+
+  # The fan-in. A fanned step's output is a list of its items' output objects, so the value
+  # each one carries is unwrapped once here before the lists are concatenated.
+  batch:
+    block: transform.jq
+    depends_on: [per_facility]
+    config:
+      input: "${steps.per_facility.output}"
+      program: |
+        {dataValues: [.[] | .value[]]}
+
+  import:
+    block: dhis2.data_value_set_import
+    depends_on: [batch]
+    config:
+      connection: dhis2-demo
+      data_values: "${steps.batch.output.value}"
+      dry_run: "${params.dry_run}"
+      # One facility's bad row must not cost the district its import; the conflicts come
+      # back in the summary and are reported rather than thrown away.
+      atomic_mode: NONE
+
+  # The honest reconciliation reads BOTH sides: what was asked for is the params list, what
+  # answered is the pull's surviving items, and the difference is the facilities nobody has
+  # heard from. The import summary cannot supply that -- it only ever saw what arrived.
+  reconcile:
+    block: transform.jq
+    depends_on: [import]
+    config:
+      input:
+        asked: "${params.facilities}"
+        answered: "${steps.pull.output}"
+        summary: "${steps.import.output}"
+        period: "${params.period}"
+      program: |
+        [.answered[] | .body.args.facility] as $codes
+        | {period: .period,
+           facilities_asked: (.asked | length),
+           facilities_answered: ($codes | length),
+           missing: [.asked[] | select(.code as $c | $codes | index($c) | not) | .code],
+           imported: .summary.imported,
+           updated: .summary.updated,
+           ignored: .summary.ignored,
+           conflicts: (.summary.conflicts | length)}
+
+  # A collection run nobody reads is a collection run nobody trusts. The body is the
+  # reconciliation exactly as computed; sign_with names a connection holding an hmac_secret
+  # when the receiver checks signatures.
+  notify:
+    block: webhook.post
+    depends_on: [reconcile]
+    config:
+      url: "${params.report_url}"
+      body: "${steps.reconcile.output.value}"

dirigent_integration/shelves/inbound/fhir-observations-to-data-values.yaml

@@ -0,0 +1,193 @@
+# A FHIR Observation feed reduced to DHIS2 aggregate values.
+#
+# FHIR keeps one Observation per measurement, coded in LOINC, about one patient. DHIS2's
+# aggregate side keeps one figure per data element, period and org unit. So the integration
+# is three decisions, and every FHIR-to-aggregate integration makes the same three:
+#
+#   the LOINC code   -> the data element, through a table this document carries. LOINC is
+#                       the sending vocabulary and a DHIS2 uid is the receiving one; nothing
+#                       derives one from the other, so the table is the mapping and there is
+#                       no clever way around writing it.
+#   effectiveDateTime-> the period, truncated to the DHIS2 month. FHIR timestamps an
+#                       instant; DHIS2 stores a reporting month.
+#   the org unit     -> NOT from the payload. A public FHIR server's Observations are about
+#                       patients, and a patient is not a facility. Which facility this feed
+#                       belongs to is knowledge the puller has and the payload does not, so
+#                       it is a parameter.
+#
+# The hops, and the shape at each one:
+#
+#   1. pull       -> a FHIR Bundle: {resourceType: "Bundle", entry: [{resource: {...}}]}
+#   2. flatten    -> one row per entry that has both a mapped LOINC code and a numeric
+#                    value; everything else is dropped, silently and on purpose
+#   3. aggregate  -> {dataValues: [...]}, one value per (element, period) pair, summed
+#   4. gate       -> refused unless the ids and periods are DHIS2's own
+#   5. import     -> the instance's summary
+#
+# HOW A COUNT IS AGGREGATED IS A CLINICAL DECISION, not a technical one. Summing is right
+# for doses given and wrong for a body temperature; an averaged indicator needs a different
+# last line, and the pack cannot know which. This one counts observations, which is the
+# reduction that is always defensible: how many were recorded.
+#
+# The default pull is the public HAPI test server, so the example runs with no setup. It
+# holds whatever the world has posted to it, so the feed is unpredictable in content and
+# perfectly predictable in shape -- which is the half this example teaches. If a run finds
+# nothing mappable, the gate refuses an empty set rather than importing nothing quietly.
+#
+# What makes it yours: fhir_base_url pointed at your server, the lookup table, and the
+# org unit.
+#
+#   dg run --local examples/inbound/fhir-observations-to-data-values.yaml
+#   dg run --local examples/inbound/fhir-observations-to-data-values.yaml -p count=50
+
+format: dirigent/v1
+kind: pipeline
+code: fhir-observations-to-data-values
+name: FHIR Observations into DHIS2
+description: |
+  Pull a FHIR Observation bundle, map LOINC codes onto DHIS2 data elements, reduce the
+  observations to monthly counts, and import them as a data value set.
+
+tags: [inbound, dhis2, http, transform, validate, cross-boundary]
+
+requires:
+  blocks:
+    - http.request
+    - transform.jq
+    - validate.schema
+    - dhis2.data_value_set_import
+
+connections:
+  dhis2-demo:
+    kind: dhis2
+    config:
+      base_url: https://play.im.dhis2.org/stable-2-43-1
+      basic_username: admin
+      basic_password: district
+      timeout: 60s
+  # The outside end is a connection rather than a bare url because a real FHIR server is
+  # authenticated and versioned, and both belong to the server rather than to the step.
+  fhir-server:
+    kind: http
+    config:
+      base_url: https://hapi.fhir.org/baseR4
+      timeout: 60s
+
+params:
+  type: object
+  properties:
+    count:
+      type: integer
+      description: How many Observations one page asks for; kept small so the demo is cheap.
+      default: 20
+      minimum: 1
+      maximum: 200
+    org_unit:
+      type: string
+      description: The facility this feed reports for, which the payload cannot say.
+      default: DiszpKrYNg8
+    loinc_to_data_element:
+      type: object
+      description: LOINC code on the left, DHIS2 data element uid on the right.
+      default:
+        "8302-2": x0PshcPLSk1
+        "29463-7": wZqi8EXN5x4
+    dry_run:
+      type: boolean
+      description: Whether the instance validates the import and writes nothing.
+      default: true
+
+schemas:
+  data-value-set:
+    type: object
+    required: [dataValues]
+    properties:
+      dataValues:
+        type: array
+        minItems: 1
+        items:
+          type: object
+          required: [dataElement, period, orgUnit, value]
+          properties:
+            dataElement: { type: string, format: dhis2-uid }
+            period: { type: string, format: dhis2-period }
+            orgUnit: { type: string, format: dhis2-uid }
+            value: { type: string }
+
+steps:
+  # _format=json because a FHIR server negotiates content and will answer XML to a client
+  # that does not say; _count bounds the page. A real feed pages with _getpages, which is a
+  # loop this example deliberately does not have.
+  pull:
+    block: http.request
+    config:
+      connection: fhir-server
+      path: /Observation
+      method: GET
+      query:
+        _count: "${params.count}"
+        _format: json
+
+  # One program, because the mapping needs the code and the timestamp of the same resource
+  # at once. Entries are dropped rather than defaulted at three points: no LOINC coding, no
+  # mapped uid, no effective instant. An observation that cannot be placed in a month at a
+  # data element is not an observation this instance can store.
+  flatten:
+    block: transform.jq
+    depends_on: [pull]
+    config:
+      input:
+        bundle: "${steps.pull.output.body}"
+        lookup: "${params.loinc_to_data_element}"
+      program: |
+        .lookup as $lookup
+        | [ .bundle.entry // []
+            | .[]
+            | .resource
+            | select(.resourceType == "Observation")
+            | (.effectiveDateTime // .issued) as $when
+            | select($when != null)
+            | (.code.coding // [])
+            | .[]
+            | select(.system == "http://loinc.org")
+            | select($lookup[.code] != null)
+            | {dataElement: $lookup[.code], period: ($when[0:7] | gsub("-"; ""))}
+          ]
+
+  # Counting, not summing a measured quantity: see the header. group_by leaves one group
+  # per distinct pair, and its length is how many observations landed in it.
+  aggregate:
+    block: transform.jq
+    depends_on: [flatten]
+    config:
+      input:
+        rows: "${steps.flatten.output.value}"
+        org_unit: "${params.org_unit}"
+      program: |
+        .org_unit as $ou
+        | {dataValues: [
+            .rows
+            | group_by([.dataElement, .period])[]
+            | {dataElement: .[0].dataElement,
+               period: .[0].period,
+               orgUnit: $ou,
+               value: (length | tostring)}
+          ]}
+
+  # minItems: 1 in the schema is the real work here: a feed that happened to carry nothing
+  # this instance knows fails the run visibly instead of importing an empty envelope and
+  # reporting success.
+  gate:
+    block: validate.schema
+    depends_on: [aggregate]
+    config:
+      input: "${steps.aggregate.output.value}"
+      schema: data-value-set
+
+  import:
+    block: dhis2.data_value_set_import
+    depends_on: [gate]
+    config:
+      connection: dhis2-demo
+      data_values: "${steps.gate.output.value}"
+      dry_run: "${params.dry_run}"