messagefoundry 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- messagefoundry/__init__.py +108 -0
- messagefoundry/__main__.py +1155 -0
- messagefoundry/api/__init__.py +27 -0
- messagefoundry/api/app.py +1581 -0
- messagefoundry/api/approvals.py +184 -0
- messagefoundry/api/auth_models.py +211 -0
- messagefoundry/api/auth_routes.py +655 -0
- messagefoundry/api/field_authz.py +96 -0
- messagefoundry/api/models.py +374 -0
- messagefoundry/api/security.py +247 -0
- messagefoundry/api/tls.py +47 -0
- messagefoundry/auth/__init__.py +39 -0
- messagefoundry/auth/data/common_passwords.NOTICE +13 -0
- messagefoundry/auth/data/common_passwords.txt +10000 -0
- messagefoundry/auth/identity.py +71 -0
- messagefoundry/auth/ldap.py +264 -0
- messagefoundry/auth/notifications.py +68 -0
- messagefoundry/auth/passwords.py +53 -0
- messagefoundry/auth/permissions.py +120 -0
- messagefoundry/auth/policy.py +153 -0
- messagefoundry/auth/ratelimit.py +55 -0
- messagefoundry/auth/service.py +1323 -0
- messagefoundry/auth/tokens.py +26 -0
- messagefoundry/auth/totp.py +174 -0
- messagefoundry/checks.py +174 -0
- messagefoundry/config/__init__.py +30 -0
- messagefoundry/config/active_environment.py +80 -0
- messagefoundry/config/ai_policy.py +140 -0
- messagefoundry/config/code_sets.py +260 -0
- messagefoundry/config/connections_edit.py +200 -0
- messagefoundry/config/connections_file.py +287 -0
- messagefoundry/config/db_lookup.py +117 -0
- messagefoundry/config/environments.py +116 -0
- messagefoundry/config/ingest_time.py +83 -0
- messagefoundry/config/models.py +240 -0
- messagefoundry/config/reference.py +158 -0
- messagefoundry/config/response.py +83 -0
- messagefoundry/config/run_context.py +153 -0
- messagefoundry/config/settings.py +1311 -0
- messagefoundry/config/state.py +99 -0
- messagefoundry/config/tls_policy.py +110 -0
- messagefoundry/config/wiring.py +1918 -0
- messagefoundry/console/__init__.py +20 -0
- messagefoundry/console/__main__.py +274 -0
- messagefoundry/console/_async.py +107 -0
- messagefoundry/console/change_password.py +111 -0
- messagefoundry/console/client.py +552 -0
- messagefoundry/console/connections.py +324 -0
- messagefoundry/console/login.py +107 -0
- messagefoundry/console/mfa.py +205 -0
- messagefoundry/console/reauth.py +94 -0
- messagefoundry/console/search.py +57 -0
- messagefoundry/console/service_control.py +137 -0
- messagefoundry/console/sessions.py +122 -0
- messagefoundry/console/shell.py +410 -0
- messagefoundry/console/status.py +377 -0
- messagefoundry/console/users_page.py +282 -0
- messagefoundry/console/widgets.py +553 -0
- messagefoundry/generators/README.md +27 -0
- messagefoundry/generators/__init__.py +15 -0
- messagefoundry/generators/_core.py +589 -0
- messagefoundry/generators/_hl7data.py +428 -0
- messagefoundry/generators/adt.py +286 -0
- messagefoundry/generators/all_types.py +24 -0
- messagefoundry/generators/bar.py +28 -0
- messagefoundry/generators/dft.py +20 -0
- messagefoundry/generators/mdm.py +39 -0
- messagefoundry/generators/mfn.py +46 -0
- messagefoundry/generators/oml.py +32 -0
- messagefoundry/generators/orl.py +30 -0
- messagefoundry/generators/orm.py +23 -0
- messagefoundry/generators/oru.py +21 -0
- messagefoundry/generators/ras.py +20 -0
- messagefoundry/generators/rde.py +54 -0
- messagefoundry/generators/siu.py +64 -0
- messagefoundry/generators/vxu.py +20 -0
- messagefoundry/hl7schema.py +75 -0
- messagefoundry/last_resort.py +55 -0
- messagefoundry/logging_setup.py +332 -0
- messagefoundry/parsing/__init__.py +64 -0
- messagefoundry/parsing/consistency.py +166 -0
- messagefoundry/parsing/groups.py +228 -0
- messagefoundry/parsing/message.py +453 -0
- messagefoundry/parsing/peek.py +237 -0
- messagefoundry/parsing/split.py +120 -0
- messagefoundry/parsing/summary.py +46 -0
- messagefoundry/parsing/tree.py +128 -0
- messagefoundry/parsing/validate.py +95 -0
- messagefoundry/parsing/x12/__init__.py +46 -0
- messagefoundry/parsing/x12/delimiters.py +140 -0
- messagefoundry/parsing/x12/errors.py +30 -0
- messagefoundry/parsing/x12/interchange.py +232 -0
- messagefoundry/parsing/x12/message.py +200 -0
- messagefoundry/parsing/x12/peek.py +207 -0
- messagefoundry/pipeline/__init__.py +21 -0
- messagefoundry/pipeline/alert_sinks.py +486 -0
- messagefoundry/pipeline/alerts.py +100 -0
- messagefoundry/pipeline/cert_expiry.py +219 -0
- messagefoundry/pipeline/cluster.py +955 -0
- messagefoundry/pipeline/cluster_sqlserver.py +444 -0
- messagefoundry/pipeline/config_convergence.py +137 -0
- messagefoundry/pipeline/dryrun.py +450 -0
- messagefoundry/pipeline/engine.py +756 -0
- messagefoundry/pipeline/leader_tasks.py +158 -0
- messagefoundry/pipeline/reference_sync.py +369 -0
- messagefoundry/pipeline/retention.py +289 -0
- messagefoundry/pipeline/security_notify.py +168 -0
- messagefoundry/pipeline/state_convergence.py +143 -0
- messagefoundry/pipeline/wiring_runner.py +1722 -0
- messagefoundry/py.typed +0 -0
- messagefoundry/redaction.py +71 -0
- messagefoundry/scaffold.py +321 -0
- messagefoundry/secrets_dpapi.py +129 -0
- messagefoundry/store/__init__.py +46 -0
- messagefoundry/store/audit_tee.py +67 -0
- messagefoundry/store/base.py +758 -0
- messagefoundry/store/crypto.py +166 -0
- messagefoundry/store/keyprovider.py +192 -0
- messagefoundry/store/postgres.py +3447 -0
- messagefoundry/store/sqlserver.py +3014 -0
- messagefoundry/store/store.py +3790 -0
- messagefoundry/timezone.py +207 -0
- messagefoundry/transports/__init__.py +50 -0
- messagefoundry/transports/base.py +269 -0
- messagefoundry/transports/database.py +693 -0
- messagefoundry/transports/file.py +551 -0
- messagefoundry/transports/framing.py +164 -0
- messagefoundry/transports/loopback.py +53 -0
- messagefoundry/transports/mllp.py +644 -0
- messagefoundry/transports/remotefile.py +664 -0
- messagefoundry/transports/rest.py +281 -0
- messagefoundry/transports/signing.py +321 -0
- messagefoundry/transports/soap.py +507 -0
- messagefoundry/transports/tcp.py +307 -0
- messagefoundry/transports/timer.py +146 -0
- messagefoundry/transports/x12.py +323 -0
- messagefoundry-0.1.0.dist-info/METADATA +212 -0
- messagefoundry-0.1.0.dist-info/RECORD +142 -0
- messagefoundry-0.1.0.dist-info/WHEEL +4 -0
- messagefoundry-0.1.0.dist-info/entry_points.txt +2 -0
- messagefoundry-0.1.0.dist-info/licenses/LICENSE +662 -0
- messagefoundry-0.1.0.dist-info/licenses/NOTICE +27 -0
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
# Copyright (C) 2026 MessageFoundry Organization and contributors
|
|
3
|
+
"""Raw-TCP ASC X12 EDI transport — source + destination (ADR 0012).
|
|
4
|
+
|
|
5
|
+
X12-over-TCP has **no transport sentinel**: the *frame is the interchange* (``ISA…IEA<terminator>``),
|
|
6
|
+
and the segment terminator is **discovered from each ISA header** (possibly a two-byte ``CR``+``LF``),
|
|
7
|
+
so the shared single-byte :class:`~messagefoundry.transports.framing.FrameDecoder` cannot frame it.
|
|
8
|
+
This connector therefore swaps that codec for the ISA/IEA assembler
|
|
9
|
+
(:class:`~messagefoundry.parsing.x12.interchange.X12FrameReader`, pure parsing layer) but reuses
|
|
10
|
+
:class:`~messagefoundry.transports.tcp.TcpSource`'s socket-plumbing *shape* (DoS guards, cooperative
|
|
11
|
+
stop). For partners who *do* wrap each interchange in a fixed sentinel (STX/ETX, VT/FS), the existing
|
|
12
|
+
``Tcp(framing=...)`` connector already applies — use this one when the interchange itself is the frame.
|
|
13
|
+
|
|
14
|
+
The payload is relayed **opaquely**: each received interchange's raw bytes are handed to the pipeline
|
|
15
|
+
handler and routed as a :class:`~messagefoundry.parsing.message.RawMessage` (pair it with
|
|
16
|
+
``content_type="x12"`` on the inbound, ADR 0004); a Router/Handler parses it on demand via the pure
|
|
17
|
+
:mod:`messagefoundry.parsing.x12` codec. There is **no X12 acknowledgment** (TA1/997/999 are deferred)
|
|
18
|
+
— if a Handler returns a reply it is written back verbatim, otherwise nothing is sent. Delivery is
|
|
19
|
+
at-least-once, so the receiving system must be **idempotent**.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import asyncio
|
|
25
|
+
import logging
|
|
26
|
+
from collections.abc import Callable
|
|
27
|
+
|
|
28
|
+
from messagefoundry.config.models import ConnectorType, Destination, Source
|
|
29
|
+
from messagefoundry.parsing.x12.delimiters import DEFAULT_MAX_INTERCHANGE_BYTES
|
|
30
|
+
from messagefoundry.parsing.x12.errors import X12FrameError, X12PeekError
|
|
31
|
+
from messagefoundry.parsing.x12.interchange import X12FrameReader
|
|
32
|
+
from messagefoundry.parsing.x12.message import X12Message
|
|
33
|
+
from messagefoundry.redaction import safe_exc
|
|
34
|
+
from messagefoundry.transports.base import (
|
|
35
|
+
DeliveryError,
|
|
36
|
+
DeliveryResponse,
|
|
37
|
+
DestinationConnector,
|
|
38
|
+
InboundHandler,
|
|
39
|
+
NegativeAckError,
|
|
40
|
+
SourceConnector,
|
|
41
|
+
probe_tcp_reachable,
|
|
42
|
+
register_destination,
|
|
43
|
+
register_source,
|
|
44
|
+
)
|
|
45
|
+
from messagefoundry.transports.mllp import DEFAULT_MAX_CONNECTIONS, DEFAULT_RECEIVE_TIMEOUT
|
|
46
|
+
|
|
47
|
+
__all__ = ["X12Source", "X12Destination"]
|
|
48
|
+
|
|
49
|
+
logger = logging.getLogger(__name__)
|
|
50
|
+
|
|
51
|
+
# Established clients get this long to finish an in-flight commit on stop()/reload before cancellation
|
|
52
|
+
# (mirrors MLLPSource/TcpSource; bounds shutdown).
|
|
53
|
+
_CLIENT_SHUTDOWN_GRACE = 5.0
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# --- destination -------------------------------------------------------------
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class X12Destination(DestinationConnector):
|
|
60
|
+
"""Send a complete X12 interchange to a raw-TCP receiver, **verbatim** (no synthetic framing — the
|
|
61
|
+
interchange is its own frame). Opens a fresh connection per delivery. Any connect/IO/timeout raises
|
|
62
|
+
:class:`DeliveryError`, so the pipeline retries.
|
|
63
|
+
|
|
64
|
+
*Fire-and-forget* (default): with ``expect_reply`` it reads one returned interchange and treats it
|
|
65
|
+
as confirmation — **not** parsed. *Synchronous request/response* (``capture_response``/``reingress_to``,
|
|
66
|
+
ADR 0016): it blocks for the returned interchange on the same socket, classifies a **TA1** interchange
|
|
67
|
+
acknowledgement (``_check_ta1``: TA1*A → accepted; TA1*R → permanent reject → dead-letter; TA1*E →
|
|
68
|
+
accepted-with-warning, *not* retried), and **returns** the reply as a :class:`DeliveryResponse` for the
|
|
69
|
+
delivery worker to capture (ADR 0013). A business response (271/277/278) returned *instead of* a TA1
|
|
70
|
+
is itself the confirmation. ``ta1_required`` makes a no-reply a retry. At-least-once: a lost reply
|
|
71
|
+
re-delivers, so the receiver must be idempotent."""
|
|
72
|
+
|
|
73
|
+
def __init__(self, config: Destination) -> None:
|
|
74
|
+
s = config.settings
|
|
75
|
+
self.host: str = str(s.get("host", "127.0.0.1"))
|
|
76
|
+
self.port: int = int(s["port"])
|
|
77
|
+
self.encoding: str = str(s.get("encoding", "utf-8"))
|
|
78
|
+
self.timeout: float = float(s.get("timeout_seconds", 30.0))
|
|
79
|
+
self.connect_timeout: float = float(s.get("connect_timeout", 10.0))
|
|
80
|
+
self.expect_reply: bool = bool(s.get("expect_reply", False))
|
|
81
|
+
# ADR 0016: capture the returned interchange (the 271/TA1) as a reply; ta1_required makes a
|
|
82
|
+
# no-reply a retry. capture_response is forced True by reingress_to at wiring time (ADR 0013).
|
|
83
|
+
self.capture_response: bool = bool(s.get("capture_response", False))
|
|
84
|
+
self.ta1_required: bool = bool(s.get("ta1_required", False))
|
|
85
|
+
mib = s.get("max_interchange_bytes", DEFAULT_MAX_INTERCHANGE_BYTES)
|
|
86
|
+
self.max_interchange_bytes: int | None = int(mib) if mib else None
|
|
87
|
+
|
|
88
|
+
async def send(self, payload: str) -> DeliveryResponse | None:
|
|
89
|
+
# Read the returned interchange when capturing it, when a TA1 is contractually required, or for
|
|
90
|
+
# the legacy expect_reply confirmation. Fire-and-forget (none set) is byte-identical to before.
|
|
91
|
+
need_reply = self.expect_reply or self.capture_response or self.ta1_required
|
|
92
|
+
try:
|
|
93
|
+
reader, writer = await asyncio.wait_for(
|
|
94
|
+
asyncio.open_connection(self.host, self.port), self.connect_timeout
|
|
95
|
+
)
|
|
96
|
+
except (OSError, asyncio.TimeoutError) as exc:
|
|
97
|
+
raise DeliveryError(f"X12 connect to {self.host}:{self.port} failed: {exc}") from exc
|
|
98
|
+
try:
|
|
99
|
+
writer.write(payload.encode(self.encoding)) # verbatim — the interchange is the frame
|
|
100
|
+
await asyncio.wait_for(writer.drain(), self.timeout)
|
|
101
|
+
if need_reply:
|
|
102
|
+
interchange = await asyncio.wait_for(self._read_reply(reader), self.timeout)
|
|
103
|
+
# Classify only when capturing or a TA1 is required; a plain expect_reply stays a
|
|
104
|
+
# not-inspected confirmation (byte-identical legacy behavior). _check_ta1 returns the
|
|
105
|
+
# captured reply, raises on a TA1 reject/error, and returns None for a non-capturing
|
|
106
|
+
# accept (so a ta1_required-only outbound still fails fast on a reject).
|
|
107
|
+
if self.capture_response or self.ta1_required:
|
|
108
|
+
response = self._check_ta1(interchange)
|
|
109
|
+
if self.capture_response:
|
|
110
|
+
return response
|
|
111
|
+
except asyncio.TimeoutError as exc:
|
|
112
|
+
raise DeliveryError("X12 timed out") from exc
|
|
113
|
+
except OSError as exc:
|
|
114
|
+
raise DeliveryError(f"X12 I/O error: {exc}") from exc
|
|
115
|
+
finally:
|
|
116
|
+
writer.close()
|
|
117
|
+
try:
|
|
118
|
+
await writer.wait_closed()
|
|
119
|
+
except OSError:
|
|
120
|
+
pass
|
|
121
|
+
return None
|
|
122
|
+
|
|
123
|
+
async def test_connection(self) -> None:
|
|
124
|
+
# Reachability only: open + close a connection (no interchange sent) so a test never delivers.
|
|
125
|
+
await probe_tcp_reachable(self.host, self.port, self.connect_timeout, "X12")
|
|
126
|
+
|
|
127
|
+
async def _read_reply(self, reader: asyncio.StreamReader) -> bytes:
|
|
128
|
+
"""Read one returned interchange. A peer-close / frame-size breach is a **read failure** — the
|
|
129
|
+
partner's disposition is UNKNOWN — so it raises :class:`DeliveryError` and **retries** (it is
|
|
130
|
+
never a captured ``no_reply``), mirroring MLLP ``_read_ack`` (ADR 0013/0016 read/parse split)."""
|
|
131
|
+
decoder = X12FrameReader(max_interchange_bytes=self.max_interchange_bytes)
|
|
132
|
+
while True:
|
|
133
|
+
chunk = await reader.read(4096)
|
|
134
|
+
if not chunk:
|
|
135
|
+
raise DeliveryError("X12 peer closed before returning an interchange")
|
|
136
|
+
try:
|
|
137
|
+
for interchange in decoder.feed(chunk):
|
|
138
|
+
return interchange
|
|
139
|
+
except X12FrameError as exc:
|
|
140
|
+
raise DeliveryError(f"reply exceeded max interchange size: {exc}") from exc
|
|
141
|
+
|
|
142
|
+
def _check_ta1(self, interchange: bytes) -> DeliveryResponse | None:
|
|
143
|
+
"""Classify a fully-read returned interchange (ADR 0016 Q2), modelled on MLLP ``_check_ack``.
|
|
144
|
+
|
|
145
|
+
Only a **TA1** (interchange acknowledgement) is a transport-level retry gate; a 999/997 functional
|
|
146
|
+
ack or a 271/277/278 application response is **content** that rides re-ingress. Uses only the
|
|
147
|
+
existing ``parsing/x12`` codec (separators discovered from the ISA, never hardcoded — CLAUDE.md §8);
|
|
148
|
+
the socket read and retry decision stay in the transport, so ``parsing/x12`` gains nothing."""
|
|
149
|
+
text = interchange.decode(self.encoding, errors="replace")
|
|
150
|
+
try:
|
|
151
|
+
msg = X12Message.parse(interchange)
|
|
152
|
+
except X12PeekError as exc:
|
|
153
|
+
# A reply interchange WAS received but won't parse. Capturing → outcome='unparseable' (a reply
|
|
154
|
+
# arrived; we just can't read it — NOT "no reply"). Non-capturing → a retryable DeliveryError.
|
|
155
|
+
if self.capture_response:
|
|
156
|
+
return DeliveryResponse(
|
|
157
|
+
body=text,
|
|
158
|
+
outcome="unparseable",
|
|
159
|
+
detail=f"unparseable X12 reply: {safe_exc(exc)}", # scrub: a bad reply can embed a fragment (#120)
|
|
160
|
+
)
|
|
161
|
+
raise DeliveryError(f"unparseable X12 reply: {exc}") from exc
|
|
162
|
+
functional = [sid for sid in msg.segment_ids() if sid != "ISA"]
|
|
163
|
+
if functional and functional[0] == "TA1":
|
|
164
|
+
ta104 = (msg.get("TA1-04") or "").upper()
|
|
165
|
+
control = msg.get("ISA-13") or "?" # interchange control number — not PHI
|
|
166
|
+
if ta104 == "R":
|
|
167
|
+
# Interchange rejected: the partner will never accept it → permanent dead-letter (AR).
|
|
168
|
+
raise NegativeAckError(
|
|
169
|
+
f"X12 TA1 interchange rejected (TA1-04=R, ISA-13={control})",
|
|
170
|
+
code="AR",
|
|
171
|
+
permanent=True,
|
|
172
|
+
)
|
|
173
|
+
if ta104 == "E":
|
|
174
|
+
# Resolved (ADR 0016): accepted-with-warning — the interchange WAS accepted, so do NOT
|
|
175
|
+
# retry (re-sending an accepted interchange is unsafe for a state-changing 278N). Logged
|
|
176
|
+
# for operators (code + control id only, never the body); a structured AlertSink for
|
|
177
|
+
# delivered-with-warning is a follow-up.
|
|
178
|
+
logger.warning(
|
|
179
|
+
"X12 TA1 accepted-with-errors (TA1-04=E, ISA-13=%s) — delivered, not retried",
|
|
180
|
+
control,
|
|
181
|
+
)
|
|
182
|
+
if self.capture_response:
|
|
183
|
+
return DeliveryResponse(
|
|
184
|
+
body=text, outcome="accepted", detail="TA1*E accepted-with-errors"
|
|
185
|
+
)
|
|
186
|
+
return None
|
|
187
|
+
# TA1*A (or any other present code) → accepted.
|
|
188
|
+
if self.capture_response:
|
|
189
|
+
return DeliveryResponse(body=text, outcome="accepted", detail=f"TA1*{ta104 or 'A'}")
|
|
190
|
+
return None
|
|
191
|
+
# No TA1: a business response (271/277/278) returned instead — it IS the confirmation.
|
|
192
|
+
if self.capture_response:
|
|
193
|
+
return DeliveryResponse(body=text, outcome="accepted", detail="business response")
|
|
194
|
+
return None
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# --- source ------------------------------------------------------------------
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
class X12Source(SourceConnector):
|
|
201
|
+
"""Listen for inbound raw-TCP connections, reassemble each ``ISA…IEA`` interchange, and hand its
|
|
202
|
+
**raw bytes** to the pipeline handler. No HL7/X12 ACK: if the handler returns a non-``None`` reply,
|
|
203
|
+
it is written back verbatim on the same connection; otherwise nothing is sent (fire-and-forget)."""
|
|
204
|
+
|
|
205
|
+
def __init__(self, config: Source) -> None:
|
|
206
|
+
s = config.settings
|
|
207
|
+
# The bind interface is injected from [inbound].bind_host (authors never set a host on an
|
|
208
|
+
# inbound); fall back to loopback for a missing/None value — never bind all interfaces by
|
|
209
|
+
# accident, since raw TCP has no transport auth. See docs/CONNECTIONS.md.
|
|
210
|
+
self.host: str = str(s.get("host") or "127.0.0.1")
|
|
211
|
+
self.port: int = int(s["port"])
|
|
212
|
+
self.encoding: str = str(s.get("encoding", "utf-8"))
|
|
213
|
+
# Caps below: key absent → secure default; present-but-falsy (None/0) → disabled.
|
|
214
|
+
mc = s.get("max_connections", DEFAULT_MAX_CONNECTIONS)
|
|
215
|
+
self.max_connections: int | None = int(mc) if mc else None
|
|
216
|
+
rt = s.get("receive_timeout", DEFAULT_RECEIVE_TIMEOUT)
|
|
217
|
+
self.receive_timeout: float | None = float(rt) if rt else None
|
|
218
|
+
mib = s.get("max_interchange_bytes", DEFAULT_MAX_INTERCHANGE_BYTES)
|
|
219
|
+
self.max_interchange_bytes: int | None = int(mib) if mib else None
|
|
220
|
+
self._server: asyncio.Server | None = None
|
|
221
|
+
self._handler: InboundHandler | None = None
|
|
222
|
+
self._active = 0
|
|
223
|
+
# Live client writers + handler tasks so stop()/reload can actively close connections (H-2).
|
|
224
|
+
self._clients: set[asyncio.StreamWriter] = set()
|
|
225
|
+
self._client_tasks: set[asyncio.Task[None]] = set()
|
|
226
|
+
|
|
227
|
+
async def start(
|
|
228
|
+
self, handler: InboundHandler, *, leader_gate: Callable[[], bool] | None = None
|
|
229
|
+
) -> None:
|
|
230
|
+
# leader_gate is ignored: a listen source runs on every node (each binds its own endpoint), so
|
|
231
|
+
# there is no shared-resource double-read to gate. Accepted only so the runner's call is uniform.
|
|
232
|
+
self._handler = handler
|
|
233
|
+
self._server = await asyncio.start_server(self._on_client, self.host, self.port)
|
|
234
|
+
|
|
235
|
+
@property
|
|
236
|
+
def sockport(self) -> int:
|
|
237
|
+
"""The actual bound port (useful when configured with port 0 in tests)."""
|
|
238
|
+
assert self._server is not None
|
|
239
|
+
port: int = self._server.sockets[0].getsockname()[1]
|
|
240
|
+
return port
|
|
241
|
+
|
|
242
|
+
async def stop(self) -> None:
|
|
243
|
+
if self._server is not None:
|
|
244
|
+
self._server.close()
|
|
245
|
+
# Close established clients BEFORE awaiting the server (server.wait_closed() hangs on
|
|
246
|
+
# py3.12.1+ waiting for in-flight handlers). A message mid-handler still finishes its commit
|
|
247
|
+
# (the body is durably stored before any reply, so at-least-once holds). Mirrors TcpSource.
|
|
248
|
+
for writer in list(self._clients):
|
|
249
|
+
writer.close()
|
|
250
|
+
pending = [task for task in self._client_tasks if not task.done()]
|
|
251
|
+
if pending:
|
|
252
|
+
_done, still_running = await asyncio.wait(pending, timeout=_CLIENT_SHUTDOWN_GRACE)
|
|
253
|
+
for task in still_running:
|
|
254
|
+
task.cancel()
|
|
255
|
+
if still_running:
|
|
256
|
+
await asyncio.gather(*still_running, return_exceptions=True)
|
|
257
|
+
self._clients.clear()
|
|
258
|
+
self._client_tasks.clear()
|
|
259
|
+
if self._server is not None:
|
|
260
|
+
await self._server.wait_closed()
|
|
261
|
+
self._server = None
|
|
262
|
+
|
|
263
|
+
async def _on_client(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
|
|
264
|
+
assert self._handler is not None
|
|
265
|
+
task = asyncio.current_task()
|
|
266
|
+
self._clients.add(writer)
|
|
267
|
+
if task is not None:
|
|
268
|
+
self._client_tasks.add(task)
|
|
269
|
+
try:
|
|
270
|
+
if self.max_connections is not None and self._active >= self.max_connections:
|
|
271
|
+
return # at capacity — refuse the new client (closed in the outer finally)
|
|
272
|
+
self._active += 1
|
|
273
|
+
try:
|
|
274
|
+
decoder = X12FrameReader(max_interchange_bytes=self.max_interchange_bytes)
|
|
275
|
+
while True:
|
|
276
|
+
if self.receive_timeout:
|
|
277
|
+
try:
|
|
278
|
+
chunk = await asyncio.wait_for(reader.read(4096), self.receive_timeout)
|
|
279
|
+
except asyncio.TimeoutError:
|
|
280
|
+
break # idle past receive_timeout — close the connection
|
|
281
|
+
else:
|
|
282
|
+
chunk = await reader.read(4096)
|
|
283
|
+
if not chunk:
|
|
284
|
+
break
|
|
285
|
+
try:
|
|
286
|
+
for interchange in decoder.feed(chunk):
|
|
287
|
+
reply = await self._handler(interchange)
|
|
288
|
+
if reply is not None:
|
|
289
|
+
writer.write(reply.encode(self.encoding)) # verbatim reply
|
|
290
|
+
await writer.drain()
|
|
291
|
+
except X12FrameError as exc:
|
|
292
|
+
peer = writer.get_extra_info("peername")
|
|
293
|
+
logger.warning(
|
|
294
|
+
"X12 interchange from %s over cap; closing connection: %s", peer, exc
|
|
295
|
+
)
|
|
296
|
+
break # drop the connection rather than buffer without bound
|
|
297
|
+
except OSError:
|
|
298
|
+
raise # peer reset / write failure → handled by the outer OSError catch (quiet)
|
|
299
|
+
except Exception as exc:
|
|
300
|
+
# Last-resort (ASVS 16.5.4): an unexpected handler/codec error must not let the
|
|
301
|
+
# per-connection task die silently or leak detail. Log redacted; drop the conn.
|
|
302
|
+
peer = writer.get_extra_info("peername")
|
|
303
|
+
logger.error(
|
|
304
|
+
"X12 connection from %s failed unexpectedly: %s", peer, safe_exc(exc)
|
|
305
|
+
)
|
|
306
|
+
break
|
|
307
|
+
except OSError:
|
|
308
|
+
pass # peer reset; nothing to do but drop the connection
|
|
309
|
+
finally:
|
|
310
|
+
self._active -= 1
|
|
311
|
+
finally:
|
|
312
|
+
self._clients.discard(writer)
|
|
313
|
+
if task is not None:
|
|
314
|
+
self._client_tasks.discard(task)
|
|
315
|
+
writer.close()
|
|
316
|
+
try:
|
|
317
|
+
await writer.wait_closed()
|
|
318
|
+
except OSError:
|
|
319
|
+
pass
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
register_destination(ConnectorType.X12, X12Destination)
|
|
323
|
+
register_source(ConnectorType.X12, X12Source)
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: messagefoundry
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Open-source healthcare integration engine — route, transform, and validate messages across many formats and connection types
|
|
5
|
+
License-Expression: AGPL-3.0-or-later
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
License-File: NOTICE
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Requires-Dist: aiosqlite>=0.20
|
|
10
|
+
Requires-Dist: argon2-cffi>=23.1
|
|
11
|
+
Requires-Dist: cryptography>=48.0.1
|
|
12
|
+
Requires-Dist: fastapi>=0.137.1
|
|
13
|
+
Requires-Dist: hl7>=0.4.5
|
|
14
|
+
Requires-Dist: hl7apy>=1.3
|
|
15
|
+
Requires-Dist: ldap3>=2.9
|
|
16
|
+
Requires-Dist: pydantic>=2.6
|
|
17
|
+
Requires-Dist: pyspnego>=0.10
|
|
18
|
+
Requires-Dist: tomlkit>=0.12
|
|
19
|
+
Requires-Dist: tzdata>=2024.1
|
|
20
|
+
Requires-Dist: uvicorn[standard]>=0.29
|
|
21
|
+
Provides-Extra: console
|
|
22
|
+
Requires-Dist: keyring>=24.0; extra == 'console'
|
|
23
|
+
Requires-Dist: pyside6>=6.6; extra == 'console'
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: httpx>=0.27; extra == 'dev'
|
|
26
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
27
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest-timeout>=2.3; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
30
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
31
|
+
Provides-Extra: postgres
|
|
32
|
+
Requires-Dist: asyncpg>=0.29; extra == 'postgres'
|
|
33
|
+
Provides-Extra: sftp
|
|
34
|
+
Requires-Dist: paramiko>=3.4; extra == 'sftp'
|
|
35
|
+
Provides-Extra: sqlserver
|
|
36
|
+
Requires-Dist: aioodbc>=0.5; extra == 'sqlserver'
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
|
|
39
|
+
# MessageFoundry
|
|
40
|
+
|
|
41
|
+
[](https://github.com/wshallwshall/MessageFoundry/actions/workflows/ci.yml)
|
|
42
|
+
[](https://github.com/wshallwshall/MessageFoundry/actions/workflows/security.yml)
|
|
43
|
+
[](LICENSE)
|
|
44
|
+
[](pyproject.toml)
|
|
45
|
+
|
|
46
|
+
MessageFoundry is an **open-source integration engine for healthcare**. It connects clinical and
|
|
47
|
+
business systems — routing, transforming, and validating messages across many formats (HL7 v2, JSON,
|
|
48
|
+
XML/SOAP, X12, database records) and connection types (MLLP, TCP, HTTP/REST, SOAP, database, files,
|
|
49
|
+
SFTP/FTP). Configure it with guided tooling or extend it in Python; it runs on SQLite or PostgreSQL
|
|
50
|
+
with authentication, RBAC, audit, and encryption-at-rest built in.
|
|
51
|
+
|
|
52
|
+
> Python import package: `messagefoundry`. Built with **hl7apy** + **python-hl7** (HL7 parsing/
|
|
53
|
+
> validation), **FastAPI** (engine API), and **PySide6** (admin console).
|
|
54
|
+
|
|
55
|
+
## What it is
|
|
56
|
+
|
|
57
|
+
A modern alternative to engines like Mirth and Corepoint. Messages flow through a graph you
|
|
58
|
+
wire by name: an inbound **Connection** hands off to a **Router**, which forwards to one or more
|
|
59
|
+
**Handlers** (filter → transform), which deliver to outbound Connections — all backed by durable
|
|
60
|
+
queuing, automatic retries, and replay. Build that graph with guided wizards, or in Python for full
|
|
61
|
+
control; either way the configuration is version-controlled and yours.
|
|
62
|
+
|
|
63
|
+
## Architecture
|
|
64
|
+
|
|
65
|
+
**Engine-as-library + localhost API.** The engine is an importable Python package.
|
|
66
|
+
The PySide6 console talks to it over a localhost HTTP + WebSocket API — the same way
|
|
67
|
+
whether the engine runs in-process, as a local daemon, or (later) on a remote host.
|
|
68
|
+
No hand-rolled IPC; the deployment split is a config choice, not an architectural fork.
|
|
69
|
+
|
|
70
|
+
See **[docs/architecture-diagram.md](docs/architecture-diagram.md)** for the rendered diagrams —
|
|
71
|
+
system topology, runtime message flow through the staged queue, and the config wiring graph (Mermaid,
|
|
72
|
+
renders on GitHub and in the VS Code preview). The prose source of truth is
|
|
73
|
+
[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md).
|
|
74
|
+
|
|
75
|
+
### Key decisions
|
|
76
|
+
- **Reliable by default.** A durable, transactional pipeline gives at-least-once delivery,
|
|
77
|
+
automatic retries, replay, and dead-lettering — no separate message broker to run.
|
|
78
|
+
- **Async core.** asyncio with per-connection workers for listeners, pollers, retries.
|
|
79
|
+
- **Tolerant parsing first.** `python-hl7` for fast routing/peek; `hl7apy` for deep,
|
|
80
|
+
version-aware validation and profiles on demand (real-world HL7 is often non-conformant).
|
|
81
|
+
- **Configure visually or in code.** Author connections and routes with guided wizards, or in
|
|
82
|
+
Python (`inbound`/`outbound`/`@router`/`@handler`) for full control — always version-controlled.
|
|
83
|
+
The database holds runtime state and messages only, never configuration.
|
|
84
|
+
- **PHI is first-class.** Authentication, RBAC, a user-attributed audit log of message
|
|
85
|
+
views/replays, and **encryption-at-rest** for message bodies (AES-256-GCM) are **built**; log
|
|
86
|
+
redaction and MLLPS/TLS are on the roadmap. See [docs/PHI.md](docs/PHI.md) for the built-vs-planned
|
|
87
|
+
data-protection map.
|
|
88
|
+
|
|
89
|
+
## Roadmap
|
|
90
|
+
|
|
91
|
+
**Phase 1 — minimum reliable engine**
|
|
92
|
+
- [x] Connection/Router/Handler model + config-module loader
|
|
93
|
+
- [x] Durable message store / queue (SQLite WAL, outbox pattern)
|
|
94
|
+
- [x] Parse / validate (tolerant peek + opt-in strict validation)
|
|
95
|
+
- [x] MLLP source + destination (correct `0x0B … 0x1C 0x0D` framing, ACK/NACK)
|
|
96
|
+
- [x] File source + destination
|
|
97
|
+
- [x] Pipeline: source → parse/validate/filter/transform → outbox → per-dest workers,
|
|
98
|
+
with retry/backoff, dead-letter, and replay
|
|
99
|
+
- [x] localhost API (connections start/stop, message track/search/detail, replay, stats,
|
|
100
|
+
live WebSocket feed) + `python -m messagefoundry serve`
|
|
101
|
+
- [x] PySide6 console: connection dashboard, message browser, HL7 parse-tree viewer,
|
|
102
|
+
delivery/audit trail, replay (`python -m messagefoundry.console`)
|
|
103
|
+
|
|
104
|
+
**Phase 1 complete.**
|
|
105
|
+
|
|
106
|
+
**Since Phase 1 — now built**
|
|
107
|
+
- [x] Staged pipeline (ingress → routed → outbound): at-least-once handoff, dead-letter, replay
|
|
108
|
+
- [x] Authentication, RBAC, user-attributed audit log, at-rest body encryption (AES-256-GCM)
|
|
109
|
+
- [x] **PostgreSQL** store backend (production, single-node)
|
|
110
|
+
- [x] **Microsoft SQL Server** store backend (production, single-node)
|
|
111
|
+
- [x] REST, SOAP, and Database destinations
|
|
112
|
+
- [x] Database poll source
|
|
113
|
+
- [x] Reference / lookup tables (`code_set`) for enrichment
|
|
114
|
+
- [x] Alerting — logging sink + webhook/email notifier
|
|
115
|
+
- [x] Connections-as-data (`connections.toml`) editable by hand or a VS Code GUI
|
|
116
|
+
- [x] **Active-passive high availability** — self-fencing leadership lease, leader-gated graph, and a
|
|
117
|
+
failover-load test harness (kill-the-primary-under-load), on **both** PostgreSQL and SQL Server
|
|
118
|
+
- [x] **Native transport TLS** — in-process API TLS (HTTPS/WSS) and MLLP-over-TLS, with an off-loopback
|
|
119
|
+
bind guard and a certificate-expiry monitor
|
|
120
|
+
- [x] Published throughput + active-passive failover **baseline** ([docs/benchmarks/TUNING-BASELINE.md](docs/benchmarks/TUNING-BASELINE.md))
|
|
121
|
+
|
|
122
|
+
**Later** — horizontal **active-active** scale-out (the experimental multi-node cluster path);
|
|
123
|
+
higher-throughput delivery (a pooled/persistent MLLP connector); a read-only **component SDK**
|
|
124
|
+
(fork-to-customize); a **de-identification** framework; MFA and off-box log shipping. See
|
|
125
|
+
[docs/EARLY-ADOPTER-GUIDE.md](docs/EARLY-ADOPTER-GUIDE.md) §2 for the current built-vs-experimental map.
|
|
126
|
+
|
|
127
|
+
## Installing & rolling out
|
|
128
|
+
|
|
129
|
+
**The recommended way to deploy MessageFoundry is to install the published package from
|
|
130
|
+
[PyPI](https://pypi.org/project/messagefoundry/)** — a signed, version-pinned wheel is the
|
|
131
|
+
supported production artifact, with no source checkout required. Install it as a **pinned
|
|
132
|
+
dependency**, then scaffold your own config repo ([ADR 0017](docs/adr/0017-consumer-deployment-model.md)):
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
pip install "messagefoundry==0.1.0" # pin the exact engine version (core runtime, SQLite store)
|
|
136
|
+
messagefoundry init ./my-config-repo # scaffold a standalone config repo
|
|
137
|
+
cd ./my-config-repo
|
|
138
|
+
messagefoundry serve --config config --env dev
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
`0.1.0` is the current **Early Access** release on PyPI. Always **pin the exact version** so
|
|
142
|
+
upgrades stay deliberate. Add the extras your deployment needs (each is opt-in and lazy-imported):
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
pip install "messagefoundry[postgres]==0.1.0" # PostgreSQL store backend (production server DB)
|
|
146
|
+
pip install "messagefoundry[sqlserver]==0.1.0" # SQL Server store backend (+ OS-level ODBC Driver 18)
|
|
147
|
+
pip install "messagefoundry[console]==0.1.0" # PySide6 admin console
|
|
148
|
+
pip install "messagefoundry[sftp]==0.1.0" # SFTP transport for the REMOTEFILE connector
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
> **Verify before you install (supply chain).** Every release is built by a GitHub Actions workflow,
|
|
152
|
+
> Sigstore-signed, and carries SLSA build-provenance + PEP 740 attestations. Verify a downloaded
|
|
153
|
+
> wheel against its source commit with
|
|
154
|
+
> `gh attestation verify <wheel> --repo wshallwshall/MessageFoundry`, or pull the signed wheel + SBOM
|
|
155
|
+
> from the [GitHub Release assets](https://github.com/wshallwshall/MessageFoundry/releases). For an
|
|
156
|
+
> air-gapped site, mirror the wheel to a private index.
|
|
157
|
+
>
|
|
158
|
+
> *(Engine developers install from a checkout instead — see [Development](#development).)*
|
|
159
|
+
|
|
160
|
+
Piloting MessageFoundry? The **[Early-Adopter Installation & Rollout Guide](docs/EARLY-ADOPTER-GUIDE.md)**
|
|
161
|
+
takes you from first install through a staged, go/no-go-gated path to full production
|
|
162
|
+
(Lab → Shadow/Parallel → Limited → Full). It leads with an honest built-vs-experimental
|
|
163
|
+
maturity map and covers prerequisites, install, security/PHI hardening, reliability
|
|
164
|
+
configuration, validation, load testing, backup/DR, day-2 operations, and upgrade/rollback.
|
|
165
|
+
|
|
166
|
+
## Development
|
|
167
|
+
|
|
168
|
+
**Working on the engine itself?** Install from a source checkout — **editable**, with the dev tools.
|
|
169
|
+
(This is the contributor path; deployments install the pinned wheel, [above](#installing--rolling-out).)
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
python -m venv .venv && . .venv/Scripts/activate # Windows PowerShell: .venv\Scripts\Activate.ps1
|
|
173
|
+
pip install -e ".[dev]"
|
|
174
|
+
pytest
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
Run the engine + localhost API (loads the bundled sample config, which ships only in a checkout):
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
python -m messagefoundry serve --config samples/config --db messagefoundry.db --env dev
|
|
181
|
+
# API on http://127.0.0.1:8765 — GET /connections, /messages, /stats, WS /ws/stats
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
Then open the admin console (needs the `console` extra: `pip install -e ".[console]"`):
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
python -m messagefoundry.console --url http://127.0.0.1:8765
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
### VS Code extension & test harness
|
|
191
|
+
|
|
192
|
+
- **VS Code extension** ([`ide/`](ide/)) — author and test interfaces in your editor: a New Route
|
|
193
|
+
Wizard, validate-on-save, a Test Bench (dry-run `.hl7` files with before/after diffs), Stage →
|
|
194
|
+
Promote to a running engine, and an HL7-aware `@messagefoundry` chat participant. Open the `ide/`
|
|
195
|
+
folder in VS Code and press **F5**, or see [ide/README.md](ide/README.md).
|
|
196
|
+
- **Test harness** — a standalone PySide6 send/receive MLLP tool for exercising the engine with
|
|
197
|
+
synthetic, PHI-free traffic: `python -m harness`.
|
|
198
|
+
|
|
199
|
+
## License
|
|
200
|
+
|
|
201
|
+
MessageFoundry is licensed under the **GNU Affero General Public License v3.0 or later**
|
|
202
|
+
(`AGPL-3.0-or-later`) — see [LICENSE](LICENSE). Running a modified version as a network service
|
|
203
|
+
triggers the AGPL's §13 source-offer obligation. A separately-licensed commercial edition is planned
|
|
204
|
+
by **MessageFoundry Organization** under the standard open-core model — see
|
|
205
|
+
[COMMERCIAL-LICENSE.md](COMMERCIAL-LICENSE.md) (terms pending legal review). See [NOTICE](NOTICE)
|
|
206
|
+
for copyright and attribution.
|
|
207
|
+
|
|
208
|
+
## Contributing
|
|
209
|
+
|
|
210
|
+
Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md), our
|
|
211
|
+
[Code of Conduct](CODE_OF_CONDUCT.md), and how the project is governed in [GOVERNANCE.md](GOVERNANCE.md).
|
|
212
|
+
A signed [Contributor License Agreement](CLA.md) is required before a pull request can be merged.
|