dirigent-integration 0.23.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. dirigent_integration/__init__.py +29 -0
  2. dirigent_integration/py.typed +0 -0
  3. dirigent_integration/shelves/dhis2-analytics-to-csv-report.yaml +174 -0
  4. dirigent_integration/shelves/dhis2-export-to-s3.yaml +66 -0
  5. dirigent_integration/shelves/dhis2-metadata-snapshot.yaml +143 -0
  6. dirigent_integration/shelves/dhis2-tracker-weekly-window.yaml +162 -0
  7. dirigent_integration/shelves/dhis2-values-per-org-unit-to-parquet.yaml +186 -0
  8. dirigent_integration/shelves/dhis2-values-to-parquet.yaml +64 -0
  9. dirigent_integration/shelves/fhir/README.md +79 -0
  10. dirigent_integration/shelves/fhir/dhis2-to-fhir-observations.yaml +212 -0
  11. dirigent_integration/shelves/fhir/fhir-capture-bundle-to-data-values.yaml +230 -0
  12. dirigent_integration/shelves/fhir/fhir-conceptmap-driven-mapping.yaml +222 -0
  13. dirigent_integration/shelves/fhir/fhir-encounter-to-event.yaml +248 -0
  14. dirigent_integration/shelves/fhir/fhir-measure-report-to-analytics-check.yaml +205 -0
  15. dirigent_integration/shelves/fhir/fhir-nightly-window-sync.yaml +182 -0
  16. dirigent_integration/shelves/fhir/fhir-patient-to-tracked-entity.yaml +225 -0
  17. dirigent_integration/shelves/fhir/fhir-questionnaire-response-to-data-values.yaml +187 -0
  18. dirigent_integration/shelves/fhir/fhir-subscription-webhook-to-dhis2.yaml +218 -0
  19. dirigent_integration/shelves/inbound/README.md +32 -0
  20. dirigent_integration/shelves/inbound/csv-drop-to-data-values.yaml +176 -0
  21. dirigent_integration/shelves/inbound/fan-out-per-facility-import.yaml +202 -0
  22. dirigent_integration/shelves/inbound/fhir-observations-to-data-values.yaml +193 -0
  23. dirigent_integration/shelves/inbound/http-json-to-data-values.yaml +151 -0
  24. dirigent_integration/shelves/inbound/outside-to-dhis2-with-checks.yaml +246 -0
  25. dirigent_integration/shelves/inbound/parquet-lakehouse-to-dhis2.yaml +158 -0
  26. dirigent_integration/shelves/inbound/webhook-payload-to-tracker-event.yaml +176 -0
  27. dirigent_integration/shelves/inbound/weekly-window-pull-and-import.yaml +175 -0
  28. dirigent_integration/shelves/parquet-to-dhis2-import.yaml +195 -0
  29. dirigent_integration-0.23.2.dist-info/METADATA +231 -0
  30. dirigent_integration-0.23.2.dist-info/RECORD +33 -0
  31. dirigent_integration-0.23.2.dist-info/WHEEL +4 -0
  32. dirigent_integration-0.23.2.dist-info/entry_points.txt +3 -0
  33. dirigent_integration-0.23.2.dist-info/licenses/LICENSE +15 -0
@@ -0,0 +1,186 @@
1
+ # One export per organisation unit, flattened into one parquet table, with a manifest.
2
+ #
3
+ # The single-org-unit version of this is dhis2-values-to-parquet.yaml beside it. This is the
4
+ # shape a real extraction has: a list of facilities or districts, one read each, one table at
5
+ # the end, and something that says what actually landed. It spans three packs -- dhis2 for the
6
+ # reads, parquet for the encoding, storage-s3 for both destinations -- so it belongs to none of
7
+ # them and is authored here.
8
+ #
9
+ # WHY ONE TABLE AND NOT ONE FILE PER ORG UNIT. A value moves through step outputs, and a value
10
+ # leaves a run through storage.write; a block does not write storage itself. A fan-out step
11
+ # stores one output -- the list of its items' outputs, in item order, with a failed item absent
12
+ # from it -- and `for_each` is expanded when the run is created, so it can read params, run and
13
+ # item and never an upstream step's output. Item three of one step therefore cannot ask item
14
+ # three of another what it produced: a second fan cannot pick up the export its twin made. So
15
+ # the fan-in is a single step over every export, and the org unit is read out of each data
16
+ # value where the join key can be seen, rather than out of a storage path that used to carry it.
17
+ #
18
+ # What happens, hop by hop:
19
+ #
20
+ # export one dhis2.data_value_set_export per org unit. Each item answers `body`, the
21
+ # data value set as a value. items: continue means one facility going quiet --
22
+ # a credential that cannot read it, an instance that times out on one subtree --
23
+ # does not cost the others their export; the run ends completed_with_errors and
24
+ # the failed item is simply absent from this step's output list.
25
+ # rows the fan-in: every set this step is handed, opened at .dataValues and flattened
26
+ # into one table sorted by org unit and data element.
27
+ # staged storage.write, that table as one json object. A converter reads one URI and
28
+ # writes another, so a value the run is holding is put down before it is
29
+ # re-encoded. Output: uri, bytes_written, content_type.
30
+ # parquet convert.arrow, json to parquet. Parquet is bytes and never travels as a value:
31
+ # the step names a source URI and a target URI and carries nothing between them.
32
+ # Output: source, target, bytes_written.
33
+ # manifest what the run has to say for itself: how many org units were asked for, how many
34
+ # answered, how many rows landed, and where the file is.
35
+ # index that manifest written beside the parquet, at a stable URI. This is the artifact
36
+ # a downstream reader opens first to learn what a run produced without listing a
37
+ # bucket.
38
+ # archive a second, dated copy of the manifest, so the history of what each run landed
39
+ # survives the next run overwriting the current one.
40
+ #
41
+ # To make it yours: change org_units, data_set and period, and point bucket at a bucket you
42
+ # own. Which connection backs the s3:// scheme is an instance setting
43
+ # (storage_connections: {s3: <code>}), so nothing about S3 appears in this document beyond
44
+ # the URIs. The dhis2 connection below is the public play server, carried inline so the
45
+ # document runs standalone; an instance you own names its own connection instead.
46
+ #
47
+ # dg run --local examples/dhis2-values-per-org-unit-to-parquet.yaml
48
+ # dg run --local examples/dhis2-values-per-org-unit-to-parquet.yaml -p period=202506
49
+
50
+ format: dirigent/v1
51
+ kind: pipeline
52
+ code: dhis2-values-per-org-unit-to-parquet
53
+ name: A parquet table over a batch of org units
54
+ description: Export a data value set per organisation unit, flatten the batch into one parquet table, and manifest what landed.
55
+
56
+ tags: [dhis2, parquet, s3, cross-boundary]
57
+
58
+ requires:
59
+ blocks:
60
+ - dhis2.data_value_set_export
61
+ - transform.jq
62
+ - storage.write
63
+ - convert.arrow
64
+ - storage.copy
65
+
66
+ # The public DHIS2 play server, carried inline so this document runs on its own. A server
67
+ # apply is where a connection is named rather than carried.
68
+ connections:
69
+ dhis2-demo:
70
+ kind: dhis2
71
+ config:
72
+ base_url: https://play.im.dhis2.org/stable-2-43-1
73
+ basic_username: admin
74
+ basic_password: district
75
+ timeout: 60s
76
+
77
+ params:
78
+ type: object
79
+ properties:
80
+ org_units:
81
+ type: array
82
+ # Facilities and districts that exist on the play server. The list is the fan-out's
83
+ # width: four elements here are four run items and four reads.
84
+ default: [vSbt6cezomG, DiszpKrYNg8, O6uvpzGd5pu, ImspTQPwCqd]
85
+ items:
86
+ type: string
87
+ data_set:
88
+ type: string
89
+ default: BfMAe6Itzgt
90
+ period:
91
+ type: string
92
+ default: "202507"
93
+ description: An ISO period identifier, such as 202507 or 2026Q1.
94
+ bucket:
95
+ type: string
96
+ default: dirigent-exports
97
+
98
+ steps:
99
+ export:
100
+ block: dhis2.data_value_set_export
101
+ for_each: ${params.org_units}
102
+ items: continue
103
+ config:
104
+ connection: dhis2-demo
105
+ data_set: ${params.data_set}
106
+ period: ${params.period}
107
+ org_unit: ${item}
108
+
109
+ rows:
110
+ block: transform.jq
111
+ depends_on: [export]
112
+ config:
113
+ # One fan-out step, one output: the list of what its items answered. An org unit whose
114
+ # item failed is simply not in it, and an org unit with nothing reported answers a set
115
+ # with no dataValues, which is why the flattening tolerates both.
116
+ input: ${steps.export.output}
117
+ program: |
118
+ [.[]
119
+ | .body
120
+ | (.dataValues // [])[]
121
+ | {org_unit: .orgUnit,
122
+ data_element: .dataElement,
123
+ category_option_combo: .categoryOptionCombo,
124
+ period: .period,
125
+ value: .value,
126
+ stored_by: .storedBy,
127
+ last_updated: .lastUpdated}]
128
+ | sort_by(.org_unit, .data_element, .category_option_combo)
129
+
130
+ staged:
131
+ block: storage.write
132
+ depends_on: [rows]
133
+ config:
134
+ target: s3://${params.bucket}/dhis2/${params.data_set}/${params.period}/values.json
135
+ value: ${steps.rows.output.value}
136
+
137
+ parquet:
138
+ block: convert.arrow
139
+ depends_on: [staged]
140
+ config:
141
+ source: ${steps.staged.output.uri}
142
+ from: json
143
+ to: parquet
144
+ target: s3://${params.bucket}/dhis2/${params.data_set}/${params.period}/values.parquet
145
+
146
+ manifest:
147
+ block: transform.jq
148
+ depends_on: [export, rows, parquet]
149
+ config:
150
+ # `answered` is shorter than `asked_for` exactly when an item failed, which is the only
151
+ # place in the run where that difference is visible as data rather than as a status.
152
+ input:
153
+ data_set: ${params.data_set}
154
+ period: ${params.period}
155
+ asked_for: ${params.org_units}
156
+ answered: ${steps.export.output}
157
+ rows: ${steps.rows.output.value}
158
+ file: ${steps.parquet.output.target}
159
+ bytes: ${steps.parquet.output.bytes_written}
160
+ program: |
161
+ {
162
+ data_set,
163
+ period,
164
+ file,
165
+ bytes,
166
+ asked_for: (.asked_for | length),
167
+ answered: (.answered | length),
168
+ rows: (.rows | length),
169
+ org_units: ([.rows[].org_unit] | unique)
170
+ }
171
+
172
+ index:
173
+ block: storage.write
174
+ depends_on: [manifest]
175
+ config:
176
+ target: s3://${params.bucket}/dhis2/${params.data_set}/${params.period}/_manifest.json
177
+ value: ${steps.manifest.output.value}
178
+
179
+ archive:
180
+ block: storage.copy
181
+ depends_on: [index]
182
+ config:
183
+ source: ${steps.index.output.uri}
184
+ # Keyed by the run, so today's manifest does not overwrite yesterday's. The current
185
+ # manifest above stays at a stable URI for anything that just wants the latest.
186
+ target: s3://dirigent-archive/dhis2/${params.data_set}/${params.period}/${run.id}.json
@@ -0,0 +1,64 @@
1
+ # Export a DHIS2 data value set and re-encode it as parquet in object storage.
2
+ #
3
+ # Three packs meet in this one pipeline, which is why it lives here and in no pack: the read
4
+ # is dhis2.data_value_set_export from the dhis2 pack, the re-encode is convert.arrow from the
5
+ # parquet pack, and both ends address the s3:// scheme the dirigent-storage-s3 pack contributes.
6
+ #
7
+ # Three hops, and what each one hands on:
8
+ #
9
+ # export dhis2.data_value_set_export. The set comes back as `body`, a value in the run.
10
+ # stage storage.write, that value as one JSON object. A converter reads a URI, so a
11
+ # value the run is holding is put down before it is re-encoded.
12
+ # to_parquet convert.arrow, json to parquet. Parquet is bytes and never travels as a value:
13
+ # the step names a source URI and a target URI and carries nothing between them.
14
+ # Output: source, target, bytes_written.
15
+ #
16
+ # Which connection backs s3:// is an instance setting (storage_connections: {s3: <code>}).
17
+ #
18
+ # dg run --local examples/dhis2-values-to-parquet.yaml
19
+
20
+ format: dirigent/v1
21
+ kind: pipeline
22
+ code: dhis2-values-to-parquet
23
+ name: Export a data value set as parquet
24
+ description: Read a DHIS2 data value set into object storage, then re-encode it as parquet beside it.
25
+
26
+ tags: [dhis2, parquet, s3, cross-boundary]
27
+
28
+ requires:
29
+ blocks:
30
+ - dhis2.data_value_set_export
31
+ - storage.write
32
+ - convert.arrow
33
+
34
+ connections:
35
+ dhis2-demo:
36
+ kind: dhis2
37
+ config:
38
+ base_url: https://play.im.dhis2.org/stable-2-43-1
39
+ basic_username: admin
40
+ basic_password: district
41
+ timeout: 60s
42
+
43
+ steps:
44
+ export:
45
+ block: dhis2.data_value_set_export
46
+ config:
47
+ connection: dhis2-demo
48
+ data_set: BfMAe6Itzgt
49
+ period: "202507"
50
+ org_unit: vSbt6cezomG
51
+ stage:
52
+ block: storage.write
53
+ depends_on: [export]
54
+ config:
55
+ target: s3://dirigent-exports/dhis2/BfMAe6Itzgt-202507.json
56
+ value: ${steps.export.output.body}
57
+ to_parquet:
58
+ block: convert.arrow
59
+ depends_on: [stage]
60
+ config:
61
+ source: ${steps.stage.output.uri}
62
+ from: json
63
+ to: parquet
64
+ target: s3://dirigent-exports/dhis2/BfMAe6Itzgt-202507.parquet
@@ -0,0 +1,79 @@
1
+ # FHIR examples
2
+
3
+ Pipelines that move data between a FHIR endpoint and a DHIS2 instance. They are cross-boundary
4
+ twice over: the DHIS2 blocks come from the `dirigent-dhis2` pack, the HTTP, transform and
5
+ validate blocks from the runtime, and neither knows the other exists -- only an assembled
6
+ environment runs any of these at all.
7
+
8
+ Most of them read a `d2w fhir serve` facade, which publishes one DHIS2 instance as a FHIR
9
+ endpoint and takes captures back. Two are running on this machine at `http://localhost:8095`
10
+ and `http://localhost:8096`; `fhir_base` defaults to the first, so every read below works
11
+ against a real server. Where a general-purpose FHIR server is the point -- an Encounter, an
12
+ Observation, a Subscription, a `_lastUpdated` search, none of which the facade serves -- the
13
+ default is the public HAPI R4 sandbox at `https://hapi.fhir.org/baseR4` instead, and the
14
+ document's header says why.
15
+
16
+ No document here imports for real without being told to. Every write is a dry run:
17
+ `dryRun=true` on `/api/dataValueSets` and `importMode=VALIDATE` on `/api/tracker`. Those are
18
+ two spellings of one intention on two endpoints of the same server, and reaching for the wrong
19
+ one does not fail -- it imports.
20
+
21
+ ## The mapping vocabulary
22
+
23
+ Three words carry the whole translation, and they are worth learning once.
24
+
25
+ **Subject.** Every capture is *about* something, and which something depends on the form.
26
+ An aggregate or event form is answered for a place, so its `subject` is a literal
27
+ `Reference(Location/<orgUnitUid>)` and the DHIS2 org unit is read straight out of it. A tracker
28
+ form is answered about a person, so its `subject` is a *logical* reference -- `subject.type`
29
+ `Patient` and `subject.identifier` under `http://dhis2.org/fhir/id/tracked-entity`, with no
30
+ `reference` at all -- and the org unit rides on a `D2OrganisationUnit` extension instead. Two
31
+ shapes, one field, and mixing them up is the first thing a facade refuses.
32
+
33
+ **Capture.** The capture pair is `Questionnaire` and `QuestionnaireResponse`. A `Questionnaire`
34
+ is a form *definition* generated from DHIS2 metadata: one per aggregate data set, event program,
35
+ tracker program, program stage, or tracked entity type. A `QuestionnaireResponse` is one
36
+ *submission* against it, answering item by item on the same `linkId`s. Those linkIds are DHIS2
37
+ uids -- a plain one is a data element, and a dotted one, `<dataElement>.<categoryOptionCombo>`,
38
+ is a single disaggregated cell -- which is what makes a response readable back into DHIS2
39
+ without consulting the form. Everything DHIS2 has and FHIR has no field for rides as a named
40
+ extension: `D2Period` carries the ISO period, `D2AttributeOptionCombo` the attribute option
41
+ combo, `D2TrackerEnrollment` the enrollment. A capture reaches the facade one resource per
42
+ `POST /QuestionnaireResponse`; there is no batch, and nothing reaches DHIS2 at capture time --
43
+ the facade holds a receipt whose lifecycle moves `received` to `forwarded` or `rejected`.
44
+
45
+ **ConceptMap.** A published, versioned, addressable translation between two code systems. The
46
+ facade emits one per DHIS2 option set, with two groups distinguished by `group.target`: one to
47
+ the DHIS2 option uid, always complete, and one to the DHIS2 option code, only where an option
48
+ has one. Fetching the map at run time instead of writing a lookup into a document is the
49
+ difference between a mapping that can drift and one that cannot: an option added to a DHIS2
50
+ option set is in the regenerated map, and the pipeline that reads it needs no edit.
51
+
52
+ ## Pipelines
53
+
54
+ | File | What it teaches |
55
+ | --- | --- |
56
+ | [fhir-capture-bundle-to-data-values.yaml](fhir-capture-bundle-to-data-values.yaml) | The capture pair end to end: a page of `QuestionnaireResponse` captures pulled off the facade, translated to `/api/dataValueSets`, gated on the pack's `dhis2-uid` and `dhis2-period` formats, rehearsed and then imported. |
57
+ | [fhir-questionnaire-response-to-data-values.yaml](fhir-questionnaire-response-to-data-values.yaml) | The general case, where a `linkId` is a form designer's name rather than a DHIS2 uid: an explicit lookup in params, and every unmapped answer counted rather than dropped in silence. |
58
+ | [fhir-patient-to-tracked-entity.yaml](fhir-patient-to-tracked-entity.yaml) | The `Patient` register searched by identifier and folded into an `/api/tracker` registration -- and why the write is an `http.request` on the `dhis2` connection, because `dhis2.tracker` reads and does not write. |
59
+ | [fhir-encounter-to-event.yaml](fhir-encounter-to-event.yaml) | Cardinalities that do not line up: one `Encounter` plus every `Observation` naming it, folded into the single program stage event DHIS2 wants, with the LOINC codes mapped and the rest reported. |
60
+ | [fhir-conceptmap-driven-mapping.yaml](fhir-conceptmap-driven-mapping.yaml) | The mapping fetched rather than written: a `ConceptMap` flattened into a lookup at run time, applied to coded answers, with `equivalence` honoured so an inexact match is reported instead of imported. |
61
+ | [fhir-measure-report-to-analytics-check.yaml](fhir-measure-report-to-analytics-check.yaml) | A reconciliation that writes nothing: a `MeasureReport` held against `dhis2.analytics_query` for the same period and org unit, and the gap delivered by `webhook.post` whether or not there is one. |
62
+ | [dhis2-to-fhir-observations.yaml](dhis2-to-fhir-observations.yaml) | The outbound direction: a data value set published as a transaction `Bundle` of `Observation`s, each a conditional update keyed on the aggregate cell's natural key so republishing updates rather than duplicates. |
63
+ | [fhir-subscription-webhook-to-dhis2.yaml](fhir-subscription-webhook-to-dhis2.yaml) | The subscription contract -- `criteria`, `channel.type`, `channel.endpoint`, and what `channel.payload` decides -- landing on a webhook whose payload mapping is strict enough that no caller can reach a parameter it does not name. |
64
+ | [fhir-nightly-window-sync.yaml](fhir-nightly-window-sync.yaml) | The interval a firing covers rather than the moment it fired at: a half-open `_lastUpdated` range, a fan-out per resource type under `items: continue`, and a join under `rule: all_done` that says what it did not reach. |
65
+
66
+ ## Running them
67
+
68
+ ```bash
69
+ dg run --local examples/fhir/fhir-capture-bundle-to-data-values.yaml
70
+ dg run --local examples/fhir/fhir-conceptmap-driven-mapping.yaml \
71
+ -p target_system=http://dhis2.org/fhir/id/option-code
72
+ ```
73
+
74
+ One of them reads the window its firing covers, which no document declares and only a run
75
+ carries, so an ad hoc run has to say which interval it is for:
76
+
77
+ ```bash
78
+ dg run --local examples/fhir/fhir-nightly-window-sync.yaml --window 2026-09-01..2026-09-02
79
+ ```
@@ -0,0 +1,212 @@
1
+ # The other direction: DHIS2 aggregate values published as a FHIR Bundle of Observations.
2
+ #
3
+ # THE FLOW, END TO END. Export one data value set from DHIS2, turn every data value into an
4
+ # Observation, wrap them in a transaction Bundle, and POST that Bundle to a FHIR server.
5
+ #
6
+ # WHY IT IS HERE. Every other document on this shelf reads FHIR and writes DHIS2. This one is
7
+ # the mirror, and a reader who has both in front of them can see what is symmetric and what is
8
+ # not. What is symmetric: the value, the period, the org unit, the code. What is NOT: DHIS2
9
+ # says "this cell of this data set for this period", one row; FHIR says "this observation about
10
+ # this subject at this time", one resource. Aggregate goes to FHIR by inventing an Observation
11
+ # per cell whose subject is a place rather than a person -- which is legal, and is what
12
+ # `Observation.subject` referencing a Location means.
13
+ #
14
+ # WHAT ARRIVES. A `/api/dataValueSets` export:
15
+ #
16
+ # {"dataSet": "BfMAe6Itzgt", "period": "202608", "orgUnit": "vSbt6cezomG",
17
+ # "dataValues": [{"dataElement": "s46m5MS0hxu", "period": "202608",
18
+ # "orgUnit": "vSbt6cezomG", "categoryOptionCombo": "Prlt0C1RF0s",
19
+ # "value": "12"}]}
20
+ #
21
+ # WHAT LEAVES. A transaction Bundle, one entry per data value:
22
+ #
23
+ # {"resourceType": "Bundle", "type": "transaction",
24
+ # "entry": [{"request": {"method": "POST", "url": "Observation"},
25
+ # "resource": {
26
+ # "resourceType": "Observation", "status": "final",
27
+ # "identifier": [{"system": "http://dhis2.org/fhir/id/data-value",
28
+ # "value": "s46m5MS0hxu.Prlt0C1RF0s-202608-vSbt6cezomG"}],
29
+ # "code": {"coding": [{"system": "http://dhis2.org/fhir/id/data-element",
30
+ # "code": "s46m5MS0hxu"}]},
31
+ # "subject": {"reference": "Location/vSbt6cezomG"},
32
+ # "effectivePeriod": {"start": "2026-08-01", "end": "2026-08-31"},
33
+ # "valueQuantity": {"value": 12}}}]}
34
+ #
35
+ # WHICH FIELD BECOMES WHICH.
36
+ #
37
+ # dataElement -> Observation.code.coding, system .../id/data-element
38
+ # categoryOptionCombo -> a second coding, system .../id/category-option-combo,
39
+ # because the disaggregation is part of what was counted
40
+ # orgUnit -> Observation.subject, a Reference(Location)
41
+ # period -> Observation.effectivePeriod, resolved to real dates
42
+ # value -> valueQuantity.value, or valueString when it is not a number
43
+ # (dataElement.coc, period, ou) -> Observation.identifier, so a re-publish updates rather
44
+ # than duplicates
45
+ #
46
+ # THAT IDENTIFIER IS THE WHOLE IDEMPOTENCY STORY. An aggregate cell has a natural key and FHIR
47
+ # has a place to put it, so each entry uses `PUT` with a conditional-update url --
48
+ # `Observation?identifier=<system>|<key>` -- rather than POST. Running this document twice
49
+ # leaves one Observation per cell, not two. A server that does not implement conditional update
50
+ # refuses the Bundle rather than silently duplicating, which is the right failure.
51
+ #
52
+ # WHERE A READER CHANGES IT. `data_set`, `period` and `org_unit` pick what is exported;
53
+ # `identifier_base` is the system every identifier and code is minted under, and it must be the
54
+ # same base the FHIR side of your project already uses.
55
+ #
56
+ # dg run --local examples/fhir/dhis2-to-fhir-observations.yaml
57
+ # dg run --local examples/fhir/dhis2-to-fhir-observations.yaml -p period=202507
58
+
59
+ format: dirigent/v1
60
+ kind: pipeline
61
+ code: dhis2-to-fhir-observations
62
+ name: DHIS2 data values to FHIR Observations
63
+ description: |
64
+ Export a DHIS2 data value set and publish it as a transaction `Bundle` of `Observation`
65
+ resources on a FHIR server.
66
+
67
+ The outbound half of this shelf. Each entry is a conditional update keyed on the aggregate
68
+ cell's natural key -- `(dataElement.categoryOptionCombo, period, orgUnit)` -- so publishing
69
+ twice updates rather than duplicates.
70
+
71
+ tags: [fhir, dhis2, cross-boundary]
72
+
73
+ requires:
74
+ blocks:
75
+ - dhis2.data_value_set_export
76
+ - transform.jq
77
+ - http.request
78
+
79
+ connections:
80
+ dhis2-demo:
81
+ kind: dhis2
82
+ config:
83
+ base_url: https://play.im.dhis2.org/stable-2-43-1
84
+ basic_username: admin
85
+ basic_password: district
86
+ timeout: 60s
87
+ # A public FHIR R4 sandbox, open to anyone, which is exactly why the Bundle below is keyed:
88
+ # a shared server that everybody publishes to is where duplicate-on-republish shows up first.
89
+ fhir-target:
90
+ kind: http
91
+ config:
92
+ base_url: https://hapi.fhir.org/baseR4
93
+ timeout: 60s
94
+
95
+ params:
96
+ type: object
97
+ properties:
98
+ data_set:
99
+ type: string
100
+ default: BfMAe6Itzgt
101
+ period:
102
+ type: string
103
+ description: A DHIS2 ISO period. The Observation carries the dates it resolves to.
104
+ default: "202608"
105
+ org_unit:
106
+ type: string
107
+ default: vSbt6cezomG
108
+ identifier_base:
109
+ type: string
110
+ description: The system every minted identifier and code hangs off. Match it to whatever
111
+ the FHIR side of your project already publishes under.
112
+ default: http://dhis2.org/fhir
113
+
114
+ steps:
115
+ # The export hands the set on as `body`, a value, and the next step is a jq program that
116
+ # wants exactly that. A national export over a year is the case for putting it down with a
117
+ # storage.write first and converting from there, so nothing holds it whole.
118
+ export:
119
+ block: dhis2.data_value_set_export
120
+ config:
121
+ connection: dhis2-demo
122
+ data_set: "${params.data_set}"
123
+ period: "${params.period}"
124
+ org_unit: "${params.org_unit}"
125
+ children: false
126
+
127
+ to_observation_bundle:
128
+ block: transform.jq
129
+ depends_on: [export]
130
+ config:
131
+ input:
132
+ export: "${steps.export.output.body}"
133
+ base: "${params.identifier_base}"
134
+ period: "${params.period}"
135
+ org_unit: "${params.org_unit}"
136
+ program: |
137
+ . as {$export, $base, $period, $org_unit}
138
+ # A DHIS2 monthly period is yyyyMM and carries no dates, so the range is derived. Only
139
+ # Monthly is handled here; a quarterly or weekly set needs its own arm, which is the
140
+ # honest shape -- there are 23 DHIS2 period types and none of them is guessable.
141
+ | ($period | .[0:4]) as $year
142
+ | ($period | .[4:6]) as $month
143
+ | (if ($period | length) == 6 then
144
+ {start: "\($year)-\($month)-01",
145
+ end: ("\($year)-\($month)-01" | strptime("%Y-%m-%d") | mktime
146
+ | . + (32 * 86400) | strftime("%Y-%m-01")
147
+ | strptime("%Y-%m-%d") | mktime | . - 86400 | strftime("%Y-%m-%d"))}
148
+ else null end) as $range
149
+ | {resourceType: "Bundle",
150
+ type: "transaction",
151
+ entry: [($export.dataValues // [])[]
152
+ | . as $v
153
+ | (if .categoryOptionCombo
154
+ then "\(.dataElement).\(.categoryOptionCombo)" else .dataElement end) as $cell
155
+ # A data value repeats the envelope's period and org unit only when the export
156
+ # broke them down; a single-cell-block export leaves them off every row, and the
157
+ # envelope is then the only place they are. Both are read with that fallback.
158
+ | ($v.period // $export.period // $period) as $value_period
159
+ | ($v.orgUnit // $export.orgUnit // $org_unit) as $value_org_unit
160
+ | "\($cell)-\($value_period)-\($value_org_unit)" as $key
161
+ | {# A conditional update, not a create: the natural key decides identity, so the
162
+ # server updates the Observation that already carries it or creates the first.
163
+ request: {method: "PUT",
164
+ url: "Observation?identifier=\($base)/id/data-value|\($key)"},
165
+ resource: ({
166
+ resourceType: "Observation",
167
+ status: "final",
168
+ identifier: [{system: "\($base)/id/data-value", value: $key}],
169
+ code: {coding: (
170
+ [{system: "\($base)/id/data-element", code: $v.dataElement}]
171
+ + (if $v.categoryOptionCombo
172
+ then [{system: "\($base)/id/category-option-combo",
173
+ code: $v.categoryOptionCombo}] else [] end))},
174
+ subject: {reference: "Location/\($value_org_unit)"}}
175
+ + (if $range then {effectivePeriod: $range} else {} end)
176
+ # A DHIS2 value is always a string on the wire. A number becomes a Quantity
177
+ # and anything else stays a string, because coercing a coded answer or a
178
+ # free-text comment into a number would be inventing a measurement.
179
+ + (if ($v.value | tonumber? ) != null
180
+ then {valueQuantity: {value: ($v.value | tonumber)}}
181
+ else {valueString: $v.value} end))}]}
182
+
183
+ # A transaction is all-or-nothing: the server applies every entry or none of them, and
184
+ # answers a transaction-response Bundle with one entry per request. That is the right
185
+ # granularity for one org unit and one period -- the whole month lands or the whole month
186
+ # does not -- and the wrong one for a year, which wants a batch per period instead.
187
+ publish:
188
+ block: http.request
189
+ depends_on: [to_observation_bundle]
190
+ config:
191
+ connection: fhir-target
192
+ path: /
193
+ method: POST
194
+ headers:
195
+ Content-Type: application/fhir+json
196
+ Accept: application/fhir+json
197
+ body: "${steps.to_observation_bundle.output.value}"
198
+
199
+ # The transaction response, read as a verdict rather than a status code: each entry carries
200
+ # its own `response.status`, and a 201 and a 200 mean created and updated respectively.
201
+ receipt:
202
+ block: transform.jq
203
+ depends_on: [publish]
204
+ config:
205
+ input: "${steps.publish.output.body}"
206
+ program: |
207
+ {bundle: .type,
208
+ entries: ((.entry // []) | length),
209
+ created: [(.entry // [])[] | select(.response.status | startswith("201"))] | length,
210
+ updated: [(.entry // [])[] | select(.response.status | startswith("200"))] | length,
211
+ issues: [(.entry // [])[] | select(.response.outcome.issue != null)
212
+ | .response.outcome.issue[] | {severity, code, diagnostics}]}