messagefoundry 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. messagefoundry/__init__.py +108 -0
  2. messagefoundry/__main__.py +1155 -0
  3. messagefoundry/api/__init__.py +27 -0
  4. messagefoundry/api/app.py +1581 -0
  5. messagefoundry/api/approvals.py +184 -0
  6. messagefoundry/api/auth_models.py +211 -0
  7. messagefoundry/api/auth_routes.py +655 -0
  8. messagefoundry/api/field_authz.py +96 -0
  9. messagefoundry/api/models.py +374 -0
  10. messagefoundry/api/security.py +247 -0
  11. messagefoundry/api/tls.py +47 -0
  12. messagefoundry/auth/__init__.py +39 -0
  13. messagefoundry/auth/data/common_passwords.NOTICE +13 -0
  14. messagefoundry/auth/data/common_passwords.txt +10000 -0
  15. messagefoundry/auth/identity.py +71 -0
  16. messagefoundry/auth/ldap.py +264 -0
  17. messagefoundry/auth/notifications.py +68 -0
  18. messagefoundry/auth/passwords.py +53 -0
  19. messagefoundry/auth/permissions.py +120 -0
  20. messagefoundry/auth/policy.py +153 -0
  21. messagefoundry/auth/ratelimit.py +55 -0
  22. messagefoundry/auth/service.py +1323 -0
  23. messagefoundry/auth/tokens.py +26 -0
  24. messagefoundry/auth/totp.py +174 -0
  25. messagefoundry/checks.py +174 -0
  26. messagefoundry/config/__init__.py +30 -0
  27. messagefoundry/config/active_environment.py +80 -0
  28. messagefoundry/config/ai_policy.py +140 -0
  29. messagefoundry/config/code_sets.py +260 -0
  30. messagefoundry/config/connections_edit.py +200 -0
  31. messagefoundry/config/connections_file.py +287 -0
  32. messagefoundry/config/db_lookup.py +117 -0
  33. messagefoundry/config/environments.py +116 -0
  34. messagefoundry/config/ingest_time.py +83 -0
  35. messagefoundry/config/models.py +240 -0
  36. messagefoundry/config/reference.py +158 -0
  37. messagefoundry/config/response.py +83 -0
  38. messagefoundry/config/run_context.py +153 -0
  39. messagefoundry/config/settings.py +1311 -0
  40. messagefoundry/config/state.py +99 -0
  41. messagefoundry/config/tls_policy.py +110 -0
  42. messagefoundry/config/wiring.py +1918 -0
  43. messagefoundry/console/__init__.py +20 -0
  44. messagefoundry/console/__main__.py +274 -0
  45. messagefoundry/console/_async.py +107 -0
  46. messagefoundry/console/change_password.py +111 -0
  47. messagefoundry/console/client.py +552 -0
  48. messagefoundry/console/connections.py +324 -0
  49. messagefoundry/console/login.py +107 -0
  50. messagefoundry/console/mfa.py +205 -0
  51. messagefoundry/console/reauth.py +94 -0
  52. messagefoundry/console/search.py +57 -0
  53. messagefoundry/console/service_control.py +137 -0
  54. messagefoundry/console/sessions.py +122 -0
  55. messagefoundry/console/shell.py +410 -0
  56. messagefoundry/console/status.py +377 -0
  57. messagefoundry/console/users_page.py +282 -0
  58. messagefoundry/console/widgets.py +553 -0
  59. messagefoundry/generators/README.md +27 -0
  60. messagefoundry/generators/__init__.py +15 -0
  61. messagefoundry/generators/_core.py +589 -0
  62. messagefoundry/generators/_hl7data.py +428 -0
  63. messagefoundry/generators/adt.py +286 -0
  64. messagefoundry/generators/all_types.py +24 -0
  65. messagefoundry/generators/bar.py +28 -0
  66. messagefoundry/generators/dft.py +20 -0
  67. messagefoundry/generators/mdm.py +39 -0
  68. messagefoundry/generators/mfn.py +46 -0
  69. messagefoundry/generators/oml.py +32 -0
  70. messagefoundry/generators/orl.py +30 -0
  71. messagefoundry/generators/orm.py +23 -0
  72. messagefoundry/generators/oru.py +21 -0
  73. messagefoundry/generators/ras.py +20 -0
  74. messagefoundry/generators/rde.py +54 -0
  75. messagefoundry/generators/siu.py +64 -0
  76. messagefoundry/generators/vxu.py +20 -0
  77. messagefoundry/hl7schema.py +75 -0
  78. messagefoundry/last_resort.py +55 -0
  79. messagefoundry/logging_setup.py +332 -0
  80. messagefoundry/parsing/__init__.py +64 -0
  81. messagefoundry/parsing/consistency.py +166 -0
  82. messagefoundry/parsing/groups.py +228 -0
  83. messagefoundry/parsing/message.py +453 -0
  84. messagefoundry/parsing/peek.py +237 -0
  85. messagefoundry/parsing/split.py +120 -0
  86. messagefoundry/parsing/summary.py +46 -0
  87. messagefoundry/parsing/tree.py +128 -0
  88. messagefoundry/parsing/validate.py +95 -0
  89. messagefoundry/parsing/x12/__init__.py +46 -0
  90. messagefoundry/parsing/x12/delimiters.py +140 -0
  91. messagefoundry/parsing/x12/errors.py +30 -0
  92. messagefoundry/parsing/x12/interchange.py +232 -0
  93. messagefoundry/parsing/x12/message.py +200 -0
  94. messagefoundry/parsing/x12/peek.py +207 -0
  95. messagefoundry/pipeline/__init__.py +21 -0
  96. messagefoundry/pipeline/alert_sinks.py +486 -0
  97. messagefoundry/pipeline/alerts.py +100 -0
  98. messagefoundry/pipeline/cert_expiry.py +219 -0
  99. messagefoundry/pipeline/cluster.py +955 -0
  100. messagefoundry/pipeline/cluster_sqlserver.py +444 -0
  101. messagefoundry/pipeline/config_convergence.py +137 -0
  102. messagefoundry/pipeline/dryrun.py +450 -0
  103. messagefoundry/pipeline/engine.py +756 -0
  104. messagefoundry/pipeline/leader_tasks.py +158 -0
  105. messagefoundry/pipeline/reference_sync.py +369 -0
  106. messagefoundry/pipeline/retention.py +289 -0
  107. messagefoundry/pipeline/security_notify.py +168 -0
  108. messagefoundry/pipeline/state_convergence.py +143 -0
  109. messagefoundry/pipeline/wiring_runner.py +1722 -0
  110. messagefoundry/py.typed +0 -0
  111. messagefoundry/redaction.py +71 -0
  112. messagefoundry/scaffold.py +321 -0
  113. messagefoundry/secrets_dpapi.py +129 -0
  114. messagefoundry/store/__init__.py +46 -0
  115. messagefoundry/store/audit_tee.py +67 -0
  116. messagefoundry/store/base.py +758 -0
  117. messagefoundry/store/crypto.py +166 -0
  118. messagefoundry/store/keyprovider.py +192 -0
  119. messagefoundry/store/postgres.py +3447 -0
  120. messagefoundry/store/sqlserver.py +3014 -0
  121. messagefoundry/store/store.py +3790 -0
  122. messagefoundry/timezone.py +207 -0
  123. messagefoundry/transports/__init__.py +50 -0
  124. messagefoundry/transports/base.py +269 -0
  125. messagefoundry/transports/database.py +693 -0
  126. messagefoundry/transports/file.py +551 -0
  127. messagefoundry/transports/framing.py +164 -0
  128. messagefoundry/transports/loopback.py +53 -0
  129. messagefoundry/transports/mllp.py +644 -0
  130. messagefoundry/transports/remotefile.py +664 -0
  131. messagefoundry/transports/rest.py +281 -0
  132. messagefoundry/transports/signing.py +321 -0
  133. messagefoundry/transports/soap.py +507 -0
  134. messagefoundry/transports/tcp.py +307 -0
  135. messagefoundry/transports/timer.py +146 -0
  136. messagefoundry/transports/x12.py +323 -0
  137. messagefoundry-0.1.0.dist-info/METADATA +212 -0
  138. messagefoundry-0.1.0.dist-info/RECORD +142 -0
  139. messagefoundry-0.1.0.dist-info/WHEEL +4 -0
  140. messagefoundry-0.1.0.dist-info/entry_points.txt +2 -0
  141. messagefoundry-0.1.0.dist-info/licenses/LICENSE +662 -0
  142. messagefoundry-0.1.0.dist-info/licenses/NOTICE +27 -0
@@ -0,0 +1,323 @@
1
+ # SPDX-License-Identifier: AGPL-3.0-or-later
2
+ # Copyright (C) 2026 MessageFoundry Organization and contributors
3
+ """Raw-TCP ASC X12 EDI transport — source + destination (ADR 0012).
4
+
5
+ X12-over-TCP has **no transport sentinel**: the *frame is the interchange* (``ISA…IEA<terminator>``),
6
+ and the segment terminator is **discovered from each ISA header** (possibly a two-byte ``CR``+``LF``),
7
+ so the shared single-byte :class:`~messagefoundry.transports.framing.FrameDecoder` cannot frame it.
8
+ This connector therefore swaps that codec for the ISA/IEA assembler
9
+ (:class:`~messagefoundry.parsing.x12.interchange.X12FrameReader`, pure parsing layer) but reuses
10
+ :class:`~messagefoundry.transports.tcp.TcpSource`'s socket-plumbing *shape* (DoS guards, cooperative
11
+ stop). For partners who *do* wrap each interchange in a fixed sentinel (STX/ETX, VT/FS), the existing
12
+ ``Tcp(framing=...)`` connector already applies — use this one when the interchange itself is the frame.
13
+
14
+ The payload is relayed **opaquely**: each received interchange's raw bytes are handed to the pipeline
15
+ handler and routed as a :class:`~messagefoundry.parsing.message.RawMessage` (pair it with
16
+ ``content_type="x12"`` on the inbound, ADR 0004); a Router/Handler parses it on demand via the pure
17
+ :mod:`messagefoundry.parsing.x12` codec. There is **no X12 acknowledgment** (TA1/997/999 are deferred)
18
+ — if a Handler returns a reply it is written back verbatim, otherwise nothing is sent. Delivery is
19
+ at-least-once, so the receiving system must be **idempotent**.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import asyncio
25
+ import logging
26
+ from collections.abc import Callable
27
+
28
+ from messagefoundry.config.models import ConnectorType, Destination, Source
29
+ from messagefoundry.parsing.x12.delimiters import DEFAULT_MAX_INTERCHANGE_BYTES
30
+ from messagefoundry.parsing.x12.errors import X12FrameError, X12PeekError
31
+ from messagefoundry.parsing.x12.interchange import X12FrameReader
32
+ from messagefoundry.parsing.x12.message import X12Message
33
+ from messagefoundry.redaction import safe_exc
34
+ from messagefoundry.transports.base import (
35
+ DeliveryError,
36
+ DeliveryResponse,
37
+ DestinationConnector,
38
+ InboundHandler,
39
+ NegativeAckError,
40
+ SourceConnector,
41
+ probe_tcp_reachable,
42
+ register_destination,
43
+ register_source,
44
+ )
45
+ from messagefoundry.transports.mllp import DEFAULT_MAX_CONNECTIONS, DEFAULT_RECEIVE_TIMEOUT
46
+
47
+ __all__ = ["X12Source", "X12Destination"]
48
+
49
+ logger = logging.getLogger(__name__)
50
+
51
+ # Established clients get this long to finish an in-flight commit on stop()/reload before cancellation
52
+ # (mirrors MLLPSource/TcpSource; bounds shutdown).
53
+ _CLIENT_SHUTDOWN_GRACE = 5.0
54
+
55
+
56
+ # --- destination -------------------------------------------------------------
57
+
58
+
59
+ class X12Destination(DestinationConnector):
60
+ """Send a complete X12 interchange to a raw-TCP receiver, **verbatim** (no synthetic framing — the
61
+ interchange is its own frame). Opens a fresh connection per delivery. Any connect/IO/timeout raises
62
+ :class:`DeliveryError`, so the pipeline retries.
63
+
64
+ *Fire-and-forget* (default): with ``expect_reply`` it reads one returned interchange and treats it
65
+ as confirmation — **not** parsed. *Synchronous request/response* (``capture_response``/``reingress_to``,
66
+ ADR 0016): it blocks for the returned interchange on the same socket, classifies a **TA1** interchange
67
+ acknowledgement (``_check_ta1``: TA1*A → accepted; TA1*R → permanent reject → dead-letter; TA1*E →
68
+ accepted-with-warning, *not* retried), and **returns** the reply as a :class:`DeliveryResponse` for the
69
+ delivery worker to capture (ADR 0013). A business response (271/277/278) returned *instead of* a TA1
70
+ is itself the confirmation. ``ta1_required`` makes a no-reply a retry. At-least-once: a lost reply
71
+ re-delivers, so the receiver must be idempotent."""
72
+
73
+ def __init__(self, config: Destination) -> None:
74
+ s = config.settings
75
+ self.host: str = str(s.get("host", "127.0.0.1"))
76
+ self.port: int = int(s["port"])
77
+ self.encoding: str = str(s.get("encoding", "utf-8"))
78
+ self.timeout: float = float(s.get("timeout_seconds", 30.0))
79
+ self.connect_timeout: float = float(s.get("connect_timeout", 10.0))
80
+ self.expect_reply: bool = bool(s.get("expect_reply", False))
81
+ # ADR 0016: capture the returned interchange (the 271/TA1) as a reply; ta1_required makes a
82
+ # no-reply a retry. capture_response is forced True by reingress_to at wiring time (ADR 0013).
83
+ self.capture_response: bool = bool(s.get("capture_response", False))
84
+ self.ta1_required: bool = bool(s.get("ta1_required", False))
85
+ mib = s.get("max_interchange_bytes", DEFAULT_MAX_INTERCHANGE_BYTES)
86
+ self.max_interchange_bytes: int | None = int(mib) if mib else None
87
+
88
+ async def send(self, payload: str) -> DeliveryResponse | None:
89
+ # Read the returned interchange when capturing it, when a TA1 is contractually required, or for
90
+ # the legacy expect_reply confirmation. Fire-and-forget (none set) is byte-identical to before.
91
+ need_reply = self.expect_reply or self.capture_response or self.ta1_required
92
+ try:
93
+ reader, writer = await asyncio.wait_for(
94
+ asyncio.open_connection(self.host, self.port), self.connect_timeout
95
+ )
96
+ except (OSError, asyncio.TimeoutError) as exc:
97
+ raise DeliveryError(f"X12 connect to {self.host}:{self.port} failed: {exc}") from exc
98
+ try:
99
+ writer.write(payload.encode(self.encoding)) # verbatim — the interchange is the frame
100
+ await asyncio.wait_for(writer.drain(), self.timeout)
101
+ if need_reply:
102
+ interchange = await asyncio.wait_for(self._read_reply(reader), self.timeout)
103
+ # Classify only when capturing or a TA1 is required; a plain expect_reply stays a
104
+ # not-inspected confirmation (byte-identical legacy behavior). _check_ta1 returns the
105
+ # captured reply, raises on a TA1 reject/error, and returns None for a non-capturing
106
+ # accept (so a ta1_required-only outbound still fails fast on a reject).
107
+ if self.capture_response or self.ta1_required:
108
+ response = self._check_ta1(interchange)
109
+ if self.capture_response:
110
+ return response
111
+ except asyncio.TimeoutError as exc:
112
+ raise DeliveryError("X12 timed out") from exc
113
+ except OSError as exc:
114
+ raise DeliveryError(f"X12 I/O error: {exc}") from exc
115
+ finally:
116
+ writer.close()
117
+ try:
118
+ await writer.wait_closed()
119
+ except OSError:
120
+ pass
121
+ return None
122
+
123
+ async def test_connection(self) -> None:
124
+ # Reachability only: open + close a connection (no interchange sent) so a test never delivers.
125
+ await probe_tcp_reachable(self.host, self.port, self.connect_timeout, "X12")
126
+
127
+ async def _read_reply(self, reader: asyncio.StreamReader) -> bytes:
128
+ """Read one returned interchange. A peer-close / frame-size breach is a **read failure** — the
129
+ partner's disposition is UNKNOWN — so it raises :class:`DeliveryError` and **retries** (it is
130
+ never a captured ``no_reply``), mirroring MLLP ``_read_ack`` (ADR 0013/0016 read/parse split)."""
131
+ decoder = X12FrameReader(max_interchange_bytes=self.max_interchange_bytes)
132
+ while True:
133
+ chunk = await reader.read(4096)
134
+ if not chunk:
135
+ raise DeliveryError("X12 peer closed before returning an interchange")
136
+ try:
137
+ for interchange in decoder.feed(chunk):
138
+ return interchange
139
+ except X12FrameError as exc:
140
+ raise DeliveryError(f"reply exceeded max interchange size: {exc}") from exc
141
+
142
+ def _check_ta1(self, interchange: bytes) -> DeliveryResponse | None:
143
+ """Classify a fully-read returned interchange (ADR 0016 Q2), modelled on MLLP ``_check_ack``.
144
+
145
+ Only a **TA1** (interchange acknowledgement) is a transport-level retry gate; a 999/997 functional
146
+ ack or a 271/277/278 application response is **content** that rides re-ingress. Uses only the
147
+ existing ``parsing/x12`` codec (separators discovered from the ISA, never hardcoded — CLAUDE.md §8);
148
+ the socket read and retry decision stay in the transport, so ``parsing/x12`` gains nothing."""
149
+ text = interchange.decode(self.encoding, errors="replace")
150
+ try:
151
+ msg = X12Message.parse(interchange)
152
+ except X12PeekError as exc:
153
+ # A reply interchange WAS received but won't parse. Capturing → outcome='unparseable' (a reply
154
+ # arrived; we just can't read it — NOT "no reply"). Non-capturing → a retryable DeliveryError.
155
+ if self.capture_response:
156
+ return DeliveryResponse(
157
+ body=text,
158
+ outcome="unparseable",
159
+ detail=f"unparseable X12 reply: {safe_exc(exc)}", # scrub: a bad reply can embed a fragment (#120)
160
+ )
161
+ raise DeliveryError(f"unparseable X12 reply: {exc}") from exc
162
+ functional = [sid for sid in msg.segment_ids() if sid != "ISA"]
163
+ if functional and functional[0] == "TA1":
164
+ ta104 = (msg.get("TA1-04") or "").upper()
165
+ control = msg.get("ISA-13") or "?" # interchange control number — not PHI
166
+ if ta104 == "R":
167
+ # Interchange rejected: the partner will never accept it → permanent dead-letter (AR).
168
+ raise NegativeAckError(
169
+ f"X12 TA1 interchange rejected (TA1-04=R, ISA-13={control})",
170
+ code="AR",
171
+ permanent=True,
172
+ )
173
+ if ta104 == "E":
174
+ # Resolved (ADR 0016): accepted-with-warning — the interchange WAS accepted, so do NOT
175
+ # retry (re-sending an accepted interchange is unsafe for a state-changing 278N). Logged
176
+ # for operators (code + control id only, never the body); a structured AlertSink for
177
+ # delivered-with-warning is a follow-up.
178
+ logger.warning(
179
+ "X12 TA1 accepted-with-errors (TA1-04=E, ISA-13=%s) — delivered, not retried",
180
+ control,
181
+ )
182
+ if self.capture_response:
183
+ return DeliveryResponse(
184
+ body=text, outcome="accepted", detail="TA1*E accepted-with-errors"
185
+ )
186
+ return None
187
+ # TA1*A (or any other present code) → accepted.
188
+ if self.capture_response:
189
+ return DeliveryResponse(body=text, outcome="accepted", detail=f"TA1*{ta104 or 'A'}")
190
+ return None
191
+ # No TA1: a business response (271/277/278) returned instead — it IS the confirmation.
192
+ if self.capture_response:
193
+ return DeliveryResponse(body=text, outcome="accepted", detail="business response")
194
+ return None
195
+
196
+
197
+ # --- source ------------------------------------------------------------------
198
+
199
+
200
+ class X12Source(SourceConnector):
201
+ """Listen for inbound raw-TCP connections, reassemble each ``ISA…IEA`` interchange, and hand its
202
+ **raw bytes** to the pipeline handler. No HL7/X12 ACK: if the handler returns a non-``None`` reply,
203
+ it is written back verbatim on the same connection; otherwise nothing is sent (fire-and-forget)."""
204
+
205
+ def __init__(self, config: Source) -> None:
206
+ s = config.settings
207
+ # The bind interface is injected from [inbound].bind_host (authors never set a host on an
208
+ # inbound); fall back to loopback for a missing/None value — never bind all interfaces by
209
+ # accident, since raw TCP has no transport auth. See docs/CONNECTIONS.md.
210
+ self.host: str = str(s.get("host") or "127.0.0.1")
211
+ self.port: int = int(s["port"])
212
+ self.encoding: str = str(s.get("encoding", "utf-8"))
213
+ # Caps below: key absent → secure default; present-but-falsy (None/0) → disabled.
214
+ mc = s.get("max_connections", DEFAULT_MAX_CONNECTIONS)
215
+ self.max_connections: int | None = int(mc) if mc else None
216
+ rt = s.get("receive_timeout", DEFAULT_RECEIVE_TIMEOUT)
217
+ self.receive_timeout: float | None = float(rt) if rt else None
218
+ mib = s.get("max_interchange_bytes", DEFAULT_MAX_INTERCHANGE_BYTES)
219
+ self.max_interchange_bytes: int | None = int(mib) if mib else None
220
+ self._server: asyncio.Server | None = None
221
+ self._handler: InboundHandler | None = None
222
+ self._active = 0
223
+ # Live client writers + handler tasks so stop()/reload can actively close connections (H-2).
224
+ self._clients: set[asyncio.StreamWriter] = set()
225
+ self._client_tasks: set[asyncio.Task[None]] = set()
226
+
227
+ async def start(
228
+ self, handler: InboundHandler, *, leader_gate: Callable[[], bool] | None = None
229
+ ) -> None:
230
+ # leader_gate is ignored: a listen source runs on every node (each binds its own endpoint), so
231
+ # there is no shared-resource double-read to gate. Accepted only so the runner's call is uniform.
232
+ self._handler = handler
233
+ self._server = await asyncio.start_server(self._on_client, self.host, self.port)
234
+
235
+ @property
236
+ def sockport(self) -> int:
237
+ """The actual bound port (useful when configured with port 0 in tests)."""
238
+ assert self._server is not None
239
+ port: int = self._server.sockets[0].getsockname()[1]
240
+ return port
241
+
242
+ async def stop(self) -> None:
243
+ if self._server is not None:
244
+ self._server.close()
245
+ # Close established clients BEFORE awaiting the server (server.wait_closed() hangs on
246
+ # py3.12.1+ waiting for in-flight handlers). A message mid-handler still finishes its commit
247
+ # (the body is durably stored before any reply, so at-least-once holds). Mirrors TcpSource.
248
+ for writer in list(self._clients):
249
+ writer.close()
250
+ pending = [task for task in self._client_tasks if not task.done()]
251
+ if pending:
252
+ _done, still_running = await asyncio.wait(pending, timeout=_CLIENT_SHUTDOWN_GRACE)
253
+ for task in still_running:
254
+ task.cancel()
255
+ if still_running:
256
+ await asyncio.gather(*still_running, return_exceptions=True)
257
+ self._clients.clear()
258
+ self._client_tasks.clear()
259
+ if self._server is not None:
260
+ await self._server.wait_closed()
261
+ self._server = None
262
+
263
+ async def _on_client(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
264
+ assert self._handler is not None
265
+ task = asyncio.current_task()
266
+ self._clients.add(writer)
267
+ if task is not None:
268
+ self._client_tasks.add(task)
269
+ try:
270
+ if self.max_connections is not None and self._active >= self.max_connections:
271
+ return # at capacity — refuse the new client (closed in the outer finally)
272
+ self._active += 1
273
+ try:
274
+ decoder = X12FrameReader(max_interchange_bytes=self.max_interchange_bytes)
275
+ while True:
276
+ if self.receive_timeout:
277
+ try:
278
+ chunk = await asyncio.wait_for(reader.read(4096), self.receive_timeout)
279
+ except asyncio.TimeoutError:
280
+ break # idle past receive_timeout — close the connection
281
+ else:
282
+ chunk = await reader.read(4096)
283
+ if not chunk:
284
+ break
285
+ try:
286
+ for interchange in decoder.feed(chunk):
287
+ reply = await self._handler(interchange)
288
+ if reply is not None:
289
+ writer.write(reply.encode(self.encoding)) # verbatim reply
290
+ await writer.drain()
291
+ except X12FrameError as exc:
292
+ peer = writer.get_extra_info("peername")
293
+ logger.warning(
294
+ "X12 interchange from %s over cap; closing connection: %s", peer, exc
295
+ )
296
+ break # drop the connection rather than buffer without bound
297
+ except OSError:
298
+ raise # peer reset / write failure → handled by the outer OSError catch (quiet)
299
+ except Exception as exc:
300
+ # Last-resort (ASVS 16.5.4): an unexpected handler/codec error must not let the
301
+ # per-connection task die silently or leak detail. Log redacted; drop the conn.
302
+ peer = writer.get_extra_info("peername")
303
+ logger.error(
304
+ "X12 connection from %s failed unexpectedly: %s", peer, safe_exc(exc)
305
+ )
306
+ break
307
+ except OSError:
308
+ pass # peer reset; nothing to do but drop the connection
309
+ finally:
310
+ self._active -= 1
311
+ finally:
312
+ self._clients.discard(writer)
313
+ if task is not None:
314
+ self._client_tasks.discard(task)
315
+ writer.close()
316
+ try:
317
+ await writer.wait_closed()
318
+ except OSError:
319
+ pass
320
+
321
+
322
+ register_destination(ConnectorType.X12, X12Destination)
323
+ register_source(ConnectorType.X12, X12Source)
@@ -0,0 +1,212 @@
1
+ Metadata-Version: 2.4
2
+ Name: messagefoundry
3
+ Version: 0.1.0
4
+ Summary: Open-source healthcare integration engine — route, transform, and validate messages across many formats and connection types
5
+ License-Expression: AGPL-3.0-or-later
6
+ License-File: LICENSE
7
+ License-File: NOTICE
8
+ Requires-Python: >=3.11
9
+ Requires-Dist: aiosqlite>=0.20
10
+ Requires-Dist: argon2-cffi>=23.1
11
+ Requires-Dist: cryptography>=48.0.1
12
+ Requires-Dist: fastapi>=0.137.1
13
+ Requires-Dist: hl7>=0.4.5
14
+ Requires-Dist: hl7apy>=1.3
15
+ Requires-Dist: ldap3>=2.9
16
+ Requires-Dist: pydantic>=2.6
17
+ Requires-Dist: pyspnego>=0.10
18
+ Requires-Dist: tomlkit>=0.12
19
+ Requires-Dist: tzdata>=2024.1
20
+ Requires-Dist: uvicorn[standard]>=0.29
21
+ Provides-Extra: console
22
+ Requires-Dist: keyring>=24.0; extra == 'console'
23
+ Requires-Dist: pyside6>=6.6; extra == 'console'
24
+ Provides-Extra: dev
25
+ Requires-Dist: httpx>=0.27; extra == 'dev'
26
+ Requires-Dist: mypy>=1.10; extra == 'dev'
27
+ Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
28
+ Requires-Dist: pytest-timeout>=2.3; extra == 'dev'
29
+ Requires-Dist: pytest>=8.0; extra == 'dev'
30
+ Requires-Dist: ruff>=0.4; extra == 'dev'
31
+ Provides-Extra: postgres
32
+ Requires-Dist: asyncpg>=0.29; extra == 'postgres'
33
+ Provides-Extra: sftp
34
+ Requires-Dist: paramiko>=3.4; extra == 'sftp'
35
+ Provides-Extra: sqlserver
36
+ Requires-Dist: aioodbc>=0.5; extra == 'sqlserver'
37
+ Description-Content-Type: text/markdown
38
+
39
+ # MessageFoundry
40
+
41
+ [![CI](https://github.com/wshallwshall/MessageFoundry/actions/workflows/ci.yml/badge.svg)](https://github.com/wshallwshall/MessageFoundry/actions/workflows/ci.yml)
42
+ [![Security](https://github.com/wshallwshall/MessageFoundry/actions/workflows/security.yml/badge.svg)](https://github.com/wshallwshall/MessageFoundry/actions/workflows/security.yml)
43
+ [![License: AGPL v3](https://img.shields.io/badge/License-AGPL_v3-blue.svg)](LICENSE)
44
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](pyproject.toml)
45
+
46
+ MessageFoundry is an **open-source integration engine for healthcare**. It connects clinical and
47
+ business systems — routing, transforming, and validating messages across many formats (HL7 v2, JSON,
48
+ XML/SOAP, X12, database records) and connection types (MLLP, TCP, HTTP/REST, SOAP, database, files,
49
+ SFTP/FTP). Configure it with guided tooling or extend it in Python; it runs on SQLite or PostgreSQL
50
+ with authentication, RBAC, audit, and encryption-at-rest built in.
51
+
52
+ > Python import package: `messagefoundry`. Built with **hl7apy** + **python-hl7** (HL7 parsing/
53
+ > validation), **FastAPI** (engine API), and **PySide6** (admin console).
54
+
55
+ ## What it is
56
+
57
+ A modern alternative to engines like Mirth and Corepoint. Messages flow through a graph you
58
+ wire by name: an inbound **Connection** hands off to a **Router**, which forwards to one or more
59
+ **Handlers** (filter → transform), which deliver to outbound Connections — all backed by durable
60
+ queuing, automatic retries, and replay. Build that graph with guided wizards, or in Python for full
61
+ control; either way the configuration is version-controlled and yours.
62
+
63
+ ## Architecture
64
+
65
+ **Engine-as-library + localhost API.** The engine is an importable Python package.
66
+ The PySide6 console talks to it over a localhost HTTP + WebSocket API — the same way
67
+ whether the engine runs in-process, as a local daemon, or (later) on a remote host.
68
+ No hand-rolled IPC; the deployment split is a config choice, not an architectural fork.
69
+
70
+ See **[docs/architecture-diagram.md](docs/architecture-diagram.md)** for the rendered diagrams —
71
+ system topology, runtime message flow through the staged queue, and the config wiring graph (Mermaid,
72
+ renders on GitHub and in the VS Code preview). The prose source of truth is
73
+ [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md).
74
+
75
+ ### Key decisions
76
+ - **Reliable by default.** A durable, transactional pipeline gives at-least-once delivery,
77
+ automatic retries, replay, and dead-lettering — no separate message broker to run.
78
+ - **Async core.** asyncio with per-connection workers for listeners, pollers, retries.
79
+ - **Tolerant parsing first.** `python-hl7` for fast routing/peek; `hl7apy` for deep,
80
+ version-aware validation and profiles on demand (real-world HL7 is often non-conformant).
81
+ - **Configure visually or in code.** Author connections and routes with guided wizards, or in
82
+ Python (`inbound`/`outbound`/`@router`/`@handler`) for full control — always version-controlled.
83
+ The database holds runtime state and messages only, never configuration.
84
+ - **PHI is first-class.** Authentication, RBAC, a user-attributed audit log of message
85
+ views/replays, and **encryption-at-rest** for message bodies (AES-256-GCM) are **built**; log
86
+ redaction and MLLPS/TLS are on the roadmap. See [docs/PHI.md](docs/PHI.md) for the built-vs-planned
87
+ data-protection map.
88
+
89
+ ## Roadmap
90
+
91
+ **Phase 1 — minimum reliable engine**
92
+ - [x] Connection/Router/Handler model + config-module loader
93
+ - [x] Durable message store / queue (SQLite WAL, outbox pattern)
94
+ - [x] Parse / validate (tolerant peek + opt-in strict validation)
95
+ - [x] MLLP source + destination (correct `0x0B … 0x1C 0x0D` framing, ACK/NACK)
96
+ - [x] File source + destination
97
+ - [x] Pipeline: source → parse/validate/filter/transform → outbox → per-dest workers,
98
+ with retry/backoff, dead-letter, and replay
99
+ - [x] localhost API (connections start/stop, message track/search/detail, replay, stats,
100
+ live WebSocket feed) + `python -m messagefoundry serve`
101
+ - [x] PySide6 console: connection dashboard, message browser, HL7 parse-tree viewer,
102
+ delivery/audit trail, replay (`python -m messagefoundry.console`)
103
+
104
+ **Phase 1 complete.**
105
+
106
+ **Since Phase 1 — now built**
107
+ - [x] Staged pipeline (ingress → routed → outbound): at-least-once handoff, dead-letter, replay
108
+ - [x] Authentication, RBAC, user-attributed audit log, at-rest body encryption (AES-256-GCM)
109
+ - [x] **PostgreSQL** store backend (production, single-node)
110
+ - [x] **Microsoft SQL Server** store backend (production, single-node)
111
+ - [x] REST, SOAP, and Database destinations
112
+ - [x] Database poll source
113
+ - [x] Reference / lookup tables (`code_set`) for enrichment
114
+ - [x] Alerting — logging sink + webhook/email notifier
115
+ - [x] Connections-as-data (`connections.toml`) editable by hand or a VS Code GUI
116
+ - [x] **Active-passive high availability** — self-fencing leadership lease, leader-gated graph, and a
117
+ failover-load test harness (kill-the-primary-under-load), on **both** PostgreSQL and SQL Server
118
+ - [x] **Native transport TLS** — in-process API TLS (HTTPS/WSS) and MLLP-over-TLS, with an off-loopback
119
+ bind guard and a certificate-expiry monitor
120
+ - [x] Published throughput + active-passive failover **baseline** ([docs/benchmarks/TUNING-BASELINE.md](docs/benchmarks/TUNING-BASELINE.md))
121
+
122
+ **Later** — horizontal **active-active** scale-out (the experimental multi-node cluster path);
123
+ higher-throughput delivery (a pooled/persistent MLLP connector); a read-only **component SDK**
124
+ (fork-to-customize); a **de-identification** framework; MFA and off-box log shipping. See
125
+ [docs/EARLY-ADOPTER-GUIDE.md](docs/EARLY-ADOPTER-GUIDE.md) §2 for the current built-vs-experimental map.
126
+
127
+ ## Installing & rolling out
128
+
129
+ **The recommended way to deploy MessageFoundry is to install the published package from
130
+ [PyPI](https://pypi.org/project/messagefoundry/)** — a signed, version-pinned wheel is the
131
+ supported production artifact, with no source checkout required. Install it as a **pinned
132
+ dependency**, then scaffold your own config repo ([ADR 0017](docs/adr/0017-consumer-deployment-model.md)):
133
+
134
+ ```bash
135
+ pip install "messagefoundry==0.1.0" # pin the exact engine version (core runtime, SQLite store)
136
+ messagefoundry init ./my-config-repo # scaffold a standalone config repo
137
+ cd ./my-config-repo
138
+ messagefoundry serve --config config --env dev
139
+ ```
140
+
141
+ `0.1.0` is the current **Early Access** release on PyPI. Always **pin the exact version** so
142
+ upgrades stay deliberate. Add the extras your deployment needs (each is opt-in and lazy-imported):
143
+
144
+ ```bash
145
+ pip install "messagefoundry[postgres]==0.1.0" # PostgreSQL store backend (production server DB)
146
+ pip install "messagefoundry[sqlserver]==0.1.0" # SQL Server store backend (+ OS-level ODBC Driver 18)
147
+ pip install "messagefoundry[console]==0.1.0" # PySide6 admin console
148
+ pip install "messagefoundry[sftp]==0.1.0" # SFTP transport for the REMOTEFILE connector
149
+ ```
150
+
151
+ > **Verify before you install (supply chain).** Every release is built by a GitHub Actions workflow,
152
+ > Sigstore-signed, and carries SLSA build-provenance + PEP 740 attestations. Verify a downloaded
153
+ > wheel against its source commit with
154
+ > `gh attestation verify <wheel> --repo wshallwshall/MessageFoundry`, or pull the signed wheel + SBOM
155
+ > from the [GitHub Release assets](https://github.com/wshallwshall/MessageFoundry/releases). For an
156
+ > air-gapped site, mirror the wheel to a private index.
157
+ >
158
+ > *(Engine developers install from a checkout instead — see [Development](#development).)*
159
+
160
+ Piloting MessageFoundry? The **[Early-Adopter Installation & Rollout Guide](docs/EARLY-ADOPTER-GUIDE.md)**
161
+ takes you from first install through a staged, go/no-go-gated path to full production
162
+ (Lab → Shadow/Parallel → Limited → Full). It leads with an honest built-vs-experimental
163
+ maturity map and covers prerequisites, install, security/PHI hardening, reliability
164
+ configuration, validation, load testing, backup/DR, day-2 operations, and upgrade/rollback.
165
+
166
+ ## Development
167
+
168
+ **Working on the engine itself?** Install from a source checkout — **editable**, with the dev tools.
169
+ (This is the contributor path; deployments install the pinned wheel, [above](#installing--rolling-out).)
170
+
171
+ ```bash
172
+ python -m venv .venv && . .venv/Scripts/activate # Windows PowerShell: .venv\Scripts\Activate.ps1
173
+ pip install -e ".[dev]"
174
+ pytest
175
+ ```
176
+
177
+ Run the engine + localhost API (loads the bundled sample config, which ships only in a checkout):
178
+
179
+ ```bash
180
+ python -m messagefoundry serve --config samples/config --db messagefoundry.db --env dev
181
+ # API on http://127.0.0.1:8765 — GET /connections, /messages, /stats, WS /ws/stats
182
+ ```
183
+
184
+ Then open the admin console (needs the `console` extra: `pip install -e ".[console]"`):
185
+
186
+ ```bash
187
+ python -m messagefoundry.console --url http://127.0.0.1:8765
188
+ ```
189
+
190
+ ### VS Code extension & test harness
191
+
192
+ - **VS Code extension** ([`ide/`](ide/)) — author and test interfaces in your editor: a New Route
193
+ Wizard, validate-on-save, a Test Bench (dry-run `.hl7` files with before/after diffs), Stage →
194
+ Promote to a running engine, and an HL7-aware `@messagefoundry` chat participant. Open the `ide/`
195
+ folder in VS Code and press **F5**, or see [ide/README.md](ide/README.md).
196
+ - **Test harness** — a standalone PySide6 send/receive MLLP tool for exercising the engine with
197
+ synthetic, PHI-free traffic: `python -m harness`.
198
+
199
+ ## License
200
+
201
+ MessageFoundry is licensed under the **GNU Affero General Public License v3.0 or later**
202
+ (`AGPL-3.0-or-later`) — see [LICENSE](LICENSE). Running a modified version as a network service
203
+ triggers the AGPL's §13 source-offer obligation. A separately-licensed commercial edition is planned
204
+ by **MessageFoundry Organization** under the standard open-core model — see
205
+ [COMMERCIAL-LICENSE.md](COMMERCIAL-LICENSE.md) (terms pending legal review). See [NOTICE](NOTICE)
206
+ for copyright and attribution.
207
+
208
+ ## Contributing
209
+
210
+ Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md), our
211
+ [Code of Conduct](CODE_OF_CONDUCT.md), and how the project is governed in [GOVERNANCE.md](GOVERNANCE.md).
212
+ A signed [Contributor License Agreement](CLA.md) is required before a pull request can be merged.