messagefoundry 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- messagefoundry/__init__.py +108 -0
- messagefoundry/__main__.py +1155 -0
- messagefoundry/api/__init__.py +27 -0
- messagefoundry/api/app.py +1581 -0
- messagefoundry/api/approvals.py +184 -0
- messagefoundry/api/auth_models.py +211 -0
- messagefoundry/api/auth_routes.py +655 -0
- messagefoundry/api/field_authz.py +96 -0
- messagefoundry/api/models.py +374 -0
- messagefoundry/api/security.py +247 -0
- messagefoundry/api/tls.py +47 -0
- messagefoundry/auth/__init__.py +39 -0
- messagefoundry/auth/data/common_passwords.NOTICE +13 -0
- messagefoundry/auth/data/common_passwords.txt +10000 -0
- messagefoundry/auth/identity.py +71 -0
- messagefoundry/auth/ldap.py +264 -0
- messagefoundry/auth/notifications.py +68 -0
- messagefoundry/auth/passwords.py +53 -0
- messagefoundry/auth/permissions.py +120 -0
- messagefoundry/auth/policy.py +153 -0
- messagefoundry/auth/ratelimit.py +55 -0
- messagefoundry/auth/service.py +1323 -0
- messagefoundry/auth/tokens.py +26 -0
- messagefoundry/auth/totp.py +174 -0
- messagefoundry/checks.py +174 -0
- messagefoundry/config/__init__.py +30 -0
- messagefoundry/config/active_environment.py +80 -0
- messagefoundry/config/ai_policy.py +140 -0
- messagefoundry/config/code_sets.py +260 -0
- messagefoundry/config/connections_edit.py +200 -0
- messagefoundry/config/connections_file.py +287 -0
- messagefoundry/config/db_lookup.py +117 -0
- messagefoundry/config/environments.py +116 -0
- messagefoundry/config/ingest_time.py +83 -0
- messagefoundry/config/models.py +240 -0
- messagefoundry/config/reference.py +158 -0
- messagefoundry/config/response.py +83 -0
- messagefoundry/config/run_context.py +153 -0
- messagefoundry/config/settings.py +1311 -0
- messagefoundry/config/state.py +99 -0
- messagefoundry/config/tls_policy.py +110 -0
- messagefoundry/config/wiring.py +1918 -0
- messagefoundry/console/__init__.py +20 -0
- messagefoundry/console/__main__.py +274 -0
- messagefoundry/console/_async.py +107 -0
- messagefoundry/console/change_password.py +111 -0
- messagefoundry/console/client.py +552 -0
- messagefoundry/console/connections.py +324 -0
- messagefoundry/console/login.py +107 -0
- messagefoundry/console/mfa.py +205 -0
- messagefoundry/console/reauth.py +94 -0
- messagefoundry/console/search.py +57 -0
- messagefoundry/console/service_control.py +137 -0
- messagefoundry/console/sessions.py +122 -0
- messagefoundry/console/shell.py +410 -0
- messagefoundry/console/status.py +377 -0
- messagefoundry/console/users_page.py +282 -0
- messagefoundry/console/widgets.py +553 -0
- messagefoundry/generators/README.md +27 -0
- messagefoundry/generators/__init__.py +15 -0
- messagefoundry/generators/_core.py +589 -0
- messagefoundry/generators/_hl7data.py +428 -0
- messagefoundry/generators/adt.py +286 -0
- messagefoundry/generators/all_types.py +24 -0
- messagefoundry/generators/bar.py +28 -0
- messagefoundry/generators/dft.py +20 -0
- messagefoundry/generators/mdm.py +39 -0
- messagefoundry/generators/mfn.py +46 -0
- messagefoundry/generators/oml.py +32 -0
- messagefoundry/generators/orl.py +30 -0
- messagefoundry/generators/orm.py +23 -0
- messagefoundry/generators/oru.py +21 -0
- messagefoundry/generators/ras.py +20 -0
- messagefoundry/generators/rde.py +54 -0
- messagefoundry/generators/siu.py +64 -0
- messagefoundry/generators/vxu.py +20 -0
- messagefoundry/hl7schema.py +75 -0
- messagefoundry/last_resort.py +55 -0
- messagefoundry/logging_setup.py +332 -0
- messagefoundry/parsing/__init__.py +64 -0
- messagefoundry/parsing/consistency.py +166 -0
- messagefoundry/parsing/groups.py +228 -0
- messagefoundry/parsing/message.py +453 -0
- messagefoundry/parsing/peek.py +237 -0
- messagefoundry/parsing/split.py +120 -0
- messagefoundry/parsing/summary.py +46 -0
- messagefoundry/parsing/tree.py +128 -0
- messagefoundry/parsing/validate.py +95 -0
- messagefoundry/parsing/x12/__init__.py +46 -0
- messagefoundry/parsing/x12/delimiters.py +140 -0
- messagefoundry/parsing/x12/errors.py +30 -0
- messagefoundry/parsing/x12/interchange.py +232 -0
- messagefoundry/parsing/x12/message.py +200 -0
- messagefoundry/parsing/x12/peek.py +207 -0
- messagefoundry/pipeline/__init__.py +21 -0
- messagefoundry/pipeline/alert_sinks.py +486 -0
- messagefoundry/pipeline/alerts.py +100 -0
- messagefoundry/pipeline/cert_expiry.py +219 -0
- messagefoundry/pipeline/cluster.py +955 -0
- messagefoundry/pipeline/cluster_sqlserver.py +444 -0
- messagefoundry/pipeline/config_convergence.py +137 -0
- messagefoundry/pipeline/dryrun.py +450 -0
- messagefoundry/pipeline/engine.py +756 -0
- messagefoundry/pipeline/leader_tasks.py +158 -0
- messagefoundry/pipeline/reference_sync.py +369 -0
- messagefoundry/pipeline/retention.py +289 -0
- messagefoundry/pipeline/security_notify.py +168 -0
- messagefoundry/pipeline/state_convergence.py +143 -0
- messagefoundry/pipeline/wiring_runner.py +1722 -0
- messagefoundry/py.typed +0 -0
- messagefoundry/redaction.py +71 -0
- messagefoundry/scaffold.py +321 -0
- messagefoundry/secrets_dpapi.py +129 -0
- messagefoundry/store/__init__.py +46 -0
- messagefoundry/store/audit_tee.py +67 -0
- messagefoundry/store/base.py +758 -0
- messagefoundry/store/crypto.py +166 -0
- messagefoundry/store/keyprovider.py +192 -0
- messagefoundry/store/postgres.py +3447 -0
- messagefoundry/store/sqlserver.py +3014 -0
- messagefoundry/store/store.py +3790 -0
- messagefoundry/timezone.py +207 -0
- messagefoundry/transports/__init__.py +50 -0
- messagefoundry/transports/base.py +269 -0
- messagefoundry/transports/database.py +693 -0
- messagefoundry/transports/file.py +551 -0
- messagefoundry/transports/framing.py +164 -0
- messagefoundry/transports/loopback.py +53 -0
- messagefoundry/transports/mllp.py +644 -0
- messagefoundry/transports/remotefile.py +664 -0
- messagefoundry/transports/rest.py +281 -0
- messagefoundry/transports/signing.py +321 -0
- messagefoundry/transports/soap.py +507 -0
- messagefoundry/transports/tcp.py +307 -0
- messagefoundry/transports/timer.py +146 -0
- messagefoundry/transports/x12.py +323 -0
- messagefoundry-0.1.0.dist-info/METADATA +212 -0
- messagefoundry-0.1.0.dist-info/RECORD +142 -0
- messagefoundry-0.1.0.dist-info/WHEEL +4 -0
- messagefoundry-0.1.0.dist-info/entry_points.txt +2 -0
- messagefoundry-0.1.0.dist-info/licenses/LICENSE +662 -0
- messagefoundry-0.1.0.dist-info/licenses/NOTICE +27 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
# Copyright (C) 2026 MessageFoundry Organization and contributors
|
|
3
|
+
"""Generate conformant HL7 v2.5.1 **SIU** (scheduling) messages.
|
|
4
|
+
|
|
5
|
+
Every SIU_Sxx structure in the reference is covered. Required SCH + RESOURCES(→RGS) plus the
|
|
6
|
+
optional PATIENT (PID) and SERVICE (AIS) groups give a realistic appointment message. The other
|
|
7
|
+
resource groups (general/location/personnel → AIG/AIL/AIP) are left out (not in the suffix set).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import random
|
|
13
|
+
|
|
14
|
+
from hl7apy import v2_5_1 as _ref
|
|
15
|
+
|
|
16
|
+
from messagefoundry.generators import _core
|
|
17
|
+
from messagefoundry.generators import _hl7data as d
|
|
18
|
+
from messagefoundry.generators._core import Ctx, MessageSpec, next_seq, seg
|
|
19
|
+
|
|
20
|
+
# Every SIU structure: trigger "S12" -> "SIU_S12".
|
|
21
|
+
TRIGGER_TO_STRUCTURE: dict[str, str] = {
|
|
22
|
+
k.split("_", 1)[1]: k for k in _ref.MESSAGES if k.startswith("SIU_")
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _build_sch(rng: random.Random, ctx: Ctx) -> str:
|
|
27
|
+
code, text = rng.choice(d.APPT_REASONS)
|
|
28
|
+
return seg(
|
|
29
|
+
"SCH",
|
|
30
|
+
{
|
|
31
|
+
1: d.ei(str(rng.randint(100_000, 999_999))), # placer appointment id
|
|
32
|
+
2: d.ei(str(rng.randint(100_000, 999_999)), "SCHED"), # filler appointment id
|
|
33
|
+
6: d.cwe(code, text, "L"), # event reason (required)
|
|
34
|
+
16: d.xcn(*rng.choice(d.CLINICIANS)), # filler contact person (required)
|
|
35
|
+
20: d.xcn(*rng.choice(d.CLINICIANS)), # entered by person (required)
|
|
36
|
+
},
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _build_rgs(rng: random.Random, ctx: Ctx) -> str:
|
|
41
|
+
return seg("RGS", {1: str(next_seq(ctx, "RGS")), 2: "A"})
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _build_ais(rng: random.Random, ctx: Ctx) -> str:
|
|
45
|
+
code, text = rng.choice(d.SERVICES)
|
|
46
|
+
return seg(
|
|
47
|
+
"AIS",
|
|
48
|
+
{
|
|
49
|
+
1: str(next_seq(ctx, "AIS")),
|
|
50
|
+
3: d.cwe(code, text, "LN"), # universal service id (required)
|
|
51
|
+
4: d.ts(ctx.msg_dt),
|
|
52
|
+
},
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
_core.register(
|
|
57
|
+
MessageSpec(
|
|
58
|
+
code="SIU",
|
|
59
|
+
trigger_to_structure=TRIGGER_TO_STRUCTURE,
|
|
60
|
+
builders={"SCH": _build_sch, "RGS": _build_rgs, "AIS": _build_ais},
|
|
61
|
+
optional_allowlist=frozenset({"PV1", "PV2", "DG1", "OBX"}),
|
|
62
|
+
group_suffixes=frozenset({"_PATIENT", "_SERVICE"}),
|
|
63
|
+
)
|
|
64
|
+
)
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
# Copyright (C) 2026 MessageFoundry Organization and contributors
|
|
3
|
+
"""Generate conformant HL7 v2.5.1 **VXU** (unsolicited vaccination update) messages.
|
|
4
|
+
|
|
5
|
+
RXA/RXR live in the shared core (also used by RAS), so this is just the spec.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from messagefoundry.generators import _core
|
|
11
|
+
from messagefoundry.generators._core import MessageSpec
|
|
12
|
+
|
|
13
|
+
_core.register(
|
|
14
|
+
MessageSpec(
|
|
15
|
+
code="VXU",
|
|
16
|
+
trigger_to_structure={"V04": "VXU_V04"},
|
|
17
|
+
optional_allowlist=frozenset({"PD1", "PV2", "NK1", "RXR", "OBX"}),
|
|
18
|
+
group_suffixes=frozenset({"_PATIENT", "_ORDER", "_OBSERVATION"}),
|
|
19
|
+
)
|
|
20
|
+
)
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
# Copyright (C) 2026 MessageFoundry Organization and contributors
|
|
3
|
+
"""Export HL7 v2.5.1 segment/field/component metadata (from hl7apy) as plain dicts.
|
|
4
|
+
|
|
5
|
+
Drives the IDE's HL7 field-path autocomplete: the extension ships this JSON and offers paths like
|
|
6
|
+
``PID-5.1`` (Family Name) with **no per-keystroke Python**. hl7apy is already a dependency (see
|
|
7
|
+
[parsing/validate.py](messagefoundry/parsing/validate.py)); here we walk its reference tree into a
|
|
8
|
+
tool-friendly shape. hl7apy's reference format is ``('sequence', (entries...))`` where each entry is
|
|
9
|
+
``(node_id, child, cardinality, kind)`` and ``child`` is either ``('leaf', None, datatype, name,
|
|
10
|
+
table, -1)`` or a nested ``('sequence', (...))`` for composite datatypes.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from functools import lru_cache
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from hl7apy import v2_5_1 as _ref
|
|
19
|
+
|
|
20
|
+
__all__ = ["hl7_schema", "SUPPORTED_VERSION"]
|
|
21
|
+
|
|
22
|
+
SUPPORTED_VERSION = "2.5.1"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _index(node_id: str, fallback: int) -> int:
|
|
26
|
+
"""HL7 1-based position from an hl7apy node id (``PID_5`` -> 5), else the walk order."""
|
|
27
|
+
tail = node_id.rsplit("_", 1)[-1]
|
|
28
|
+
return int(tail) if tail.isdigit() else fallback
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _describe(child: Any) -> tuple[str | None, str | None, list[dict[str, Any]]]:
|
|
32
|
+
"""Return ``(name, datatype, components)`` for a field/component child node."""
|
|
33
|
+
head = child[0]
|
|
34
|
+
if head == "leaf": # ('leaf', None, datatype, long_name, table, -1)
|
|
35
|
+
return child[3], child[2], []
|
|
36
|
+
if head == "sequence": # composite datatype -> expand its components one level
|
|
37
|
+
entries = child[1]
|
|
38
|
+
components: list[dict[str, Any]] = []
|
|
39
|
+
for i, entry in enumerate(entries, start=1):
|
|
40
|
+
cid, cchild = entry[0], entry[1]
|
|
41
|
+
cname, cdt, _ = _describe(cchild)
|
|
42
|
+
components.append({"index": _index(cid, i), "name": cname, "datatype": cdt})
|
|
43
|
+
datatype = entries[0][0].rsplit("_", 1)[0] if entries else None # e.g. CX_1 -> CX
|
|
44
|
+
return None, datatype, components
|
|
45
|
+
return None, None, []
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _segment_fields(seg_def: Any) -> list[dict[str, Any]]:
|
|
49
|
+
entries = seg_def[1] # ('sequence', (field_entries...))
|
|
50
|
+
fields: list[dict[str, Any]] = []
|
|
51
|
+
for i, entry in enumerate(entries, start=1):
|
|
52
|
+
fid, child = entry[0], entry[1]
|
|
53
|
+
name, datatype, components = _describe(child)
|
|
54
|
+
fields.append(
|
|
55
|
+
{"index": _index(fid, i), "name": name, "datatype": datatype, "components": components}
|
|
56
|
+
)
|
|
57
|
+
return fields
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@lru_cache(maxsize=None)
|
|
61
|
+
def hl7_schema(version: str = SUPPORTED_VERSION) -> dict[str, Any]:
|
|
62
|
+
"""Segment → fields → components metadata for ``version`` (only 2.5.1 today).
|
|
63
|
+
|
|
64
|
+
Shape: ``{"version", "segments": {SEG: {"fields": [{"index", "name", "datatype",
|
|
65
|
+
"components": [{"index", "name", "datatype"}]}]}}}``. Cached (immutable — treat as read-only).
|
|
66
|
+
"""
|
|
67
|
+
if version != SUPPORTED_VERSION:
|
|
68
|
+
raise ValueError(
|
|
69
|
+
f"unsupported HL7 version {version!r}; only {SUPPORTED_VERSION} is available"
|
|
70
|
+
)
|
|
71
|
+
segments: dict[str, Any] = {}
|
|
72
|
+
for seg_id, seg_def in _ref.SEGMENTS.items():
|
|
73
|
+
if isinstance(seg_def, tuple) and len(seg_def) == 2 and seg_def[0] == "sequence":
|
|
74
|
+
segments[seg_id] = {"fields": _segment_fields(seg_def)}
|
|
75
|
+
return {"version": version, "segments": segments}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
# Copyright (C) 2026 MessageFoundry Organization and contributors
|
|
3
|
+
"""Process-level last-resort error handling (ASVS 16.5.4).
|
|
4
|
+
|
|
5
|
+
Per-request (the API catch-all 500) and per-lane (the pipeline workers + framed listeners) handlers
|
|
6
|
+
already exist; this adds the **process** backstop. Any asyncio task/callback exception that nothing
|
|
7
|
+
awaited, and any uncaught main-thread exception, is routed through
|
|
8
|
+
:func:`~messagefoundry.redaction.safe_exc` to the log — so a genuinely-unhandled error can never escape
|
|
9
|
+
as a raw traceback (which could quote a PHI-bearing argument) or die silently. It only fires for
|
|
10
|
+
otherwise-unhandled errors; normal flow is untouched.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
import logging
|
|
17
|
+
import sys
|
|
18
|
+
from types import TracebackType
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from messagefoundry.redaction import safe_exc
|
|
22
|
+
|
|
23
|
+
_log = logging.getLogger("messagefoundry.last_resort")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _handle_loop_exception(loop: asyncio.AbstractEventLoop, context: dict[str, Any]) -> None:
|
|
27
|
+
"""asyncio loop exception handler: log an otherwise-unhandled task/callback error, PHI-redacted."""
|
|
28
|
+
exc = context.get("exception")
|
|
29
|
+
where = str(context.get("message") or "unhandled asyncio exception")
|
|
30
|
+
if isinstance(exc, BaseException):
|
|
31
|
+
_log.error("last-resort: %s (%s)", safe_exc(exc), where)
|
|
32
|
+
else:
|
|
33
|
+
_log.error("last-resort: %s", where)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def install_loop_exception_handler(loop: asyncio.AbstractEventLoop | None = None) -> None:
|
|
37
|
+
"""Route otherwise-unhandled asyncio task/callback exceptions through ``safe_exc`` → the log.
|
|
38
|
+
Call from within the running loop (the serving lifespan does this at startup)."""
|
|
39
|
+
(loop or asyncio.get_running_loop()).set_exception_handler(_handle_loop_exception)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _excepthook(
|
|
43
|
+
exc_type: type[BaseException], exc: BaseException, tb: TracebackType | None
|
|
44
|
+
) -> None:
|
|
45
|
+
"""``sys.excepthook``: log an uncaught main-thread exception PHI-redacted, never a raw traceback."""
|
|
46
|
+
if issubclass(exc_type, KeyboardInterrupt):
|
|
47
|
+
sys.__excepthook__(exc_type, exc, tb) # Ctrl-C is a clean interrupt, not an error to redact
|
|
48
|
+
return
|
|
49
|
+
_log.critical("last-resort: uncaught exception: %s", safe_exc(exc))
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def install_excepthook() -> None:
|
|
53
|
+
"""Replace ``sys.excepthook`` so an uncaught main-thread exception is logged PHI-redacted instead of
|
|
54
|
+
printed as a raw traceback (which could quote a PHI-bearing value) to stderr."""
|
|
55
|
+
sys.excepthook = _excepthook
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
# Copyright (C) 2026 MessageFoundry Organization and contributors
|
|
3
|
+
"""Process-wide logging setup for the engine service.
|
|
4
|
+
|
|
5
|
+
Stdlib ``logging`` only (no structlog): a stdout stream handler with a timestamped text format by
|
|
6
|
+
default, optionally **structured JSON** (one object per line, ``[logging].format = "json"``), with
|
|
7
|
+
uvicorn's own loggers routed through the same handler. When the engine runs under NSSM as a Windows
|
|
8
|
+
service, NSSM captures stdout/stderr to rotating files, so we deliberately do **not** add file handlers
|
|
9
|
+
here. A copy of every record can also be **forwarded off-box** to a syslog/SIEM collector
|
|
10
|
+
(``[logging].forward_*``; sec-offbox-log, ASVS 16.x) so log evidence survives a host compromise; PHI
|
|
11
|
+
redaction + control-char scrubbing apply to the forwarded stream exactly as to stdout.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import logging
|
|
18
|
+
import logging.handlers
|
|
19
|
+
import os
|
|
20
|
+
import socket
|
|
21
|
+
import sys
|
|
22
|
+
import time
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from messagefoundry.redaction import redact
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"configure_logging",
|
|
30
|
+
"silence_phi_prone_dependency_loggers",
|
|
31
|
+
"ControlCharScrubFilter",
|
|
32
|
+
"RedactionFilter",
|
|
33
|
+
"JsonFormatter",
|
|
34
|
+
"SyslogForward",
|
|
35
|
+
"LOG_LEVELS",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
_log = logging.getLogger(__name__)
|
|
39
|
+
|
|
40
|
+
# Timestamps in UTC with a trailing 'Z' so log correlation across hosts/timezones is unambiguous
|
|
41
|
+
# (ASVS 16.2.2); the handler's formatter converter is set to time.gmtime below.
|
|
42
|
+
_LOG_FORMAT = "%(asctime)s %(levelname)-8s %(name)s: %(message)s"
|
|
43
|
+
_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
|
|
44
|
+
|
|
45
|
+
#: Accepted ``--log-level`` values (used for argparse choices too).
|
|
46
|
+
LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL")
|
|
47
|
+
|
|
48
|
+
# Logger names uvicorn configures itself; we route them through the root handler.
|
|
49
|
+
_UVICORN_LOGGERS = ("uvicorn", "uvicorn.error", "uvicorn.access")
|
|
50
|
+
|
|
51
|
+
# C0 control characters (and DEL) escaped to keep one log record on one line. CR/LF are the
|
|
52
|
+
# log-injection vector; tab (0x09) is left intact as benign whitespace.
|
|
53
|
+
_CTRL_TRANSLATION: dict[int, str] = {0x0A: "\\n", 0x0D: "\\r"}
|
|
54
|
+
for _i in range(0x20):
|
|
55
|
+
if _i not in (0x09, 0x0A, 0x0D):
|
|
56
|
+
_CTRL_TRANSLATION[_i] = f"\\x{_i:02x}"
|
|
57
|
+
_CTRL_TRANSLATION[0x7F] = "\\x7f"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class ControlCharScrubFilter(logging.Filter):
|
|
61
|
+
"""Neutralize CR/LF and other control characters in the rendered log message to prevent log
|
|
62
|
+
injection / forging (ASVS 16.4.1).
|
|
63
|
+
|
|
64
|
+
Untrusted MLLP peer data and HL7-derived exception text reach the general log; without this a
|
|
65
|
+
crafted value containing a newline could inject a forged log line into NSSM's captured stdout.
|
|
66
|
+
We render the message (applying ``%`` args) once, escape any control characters, and only then
|
|
67
|
+
replace ``record.msg`` — clean messages keep their lazy ``msg``/``args`` untouched."""
|
|
68
|
+
|
|
69
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
70
|
+
message = record.getMessage()
|
|
71
|
+
scrubbed = message.translate(_CTRL_TRANSLATION)
|
|
72
|
+
if scrubbed != message:
|
|
73
|
+
record.msg = scrubbed
|
|
74
|
+
record.args = ()
|
|
75
|
+
return True
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# A throwaway formatter used only to render a record's exception into text for redaction.
|
|
79
|
+
# ``formatException`` is independent of any format string, so one shared instance is safe.
|
|
80
|
+
_EXC_RENDERER = logging.Formatter()
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class RedactionFilter(logging.Filter):
|
|
84
|
+
"""Scrub HL7-shaped PHI from every emitted record — the rendered **message** and the formatted
|
|
85
|
+
**exception traceback** (chained ``__cause__``/``__context__`` included) — via
|
|
86
|
+
:func:`~messagefoundry.redaction.redact` (PHI.md §7, Gate #1).
|
|
87
|
+
|
|
88
|
+
Inbound HL7 is PHI-bearing and a Router/Handler is user code that can ``raise ValueError(f"…{raw}")``;
|
|
89
|
+
an outer-loop ``log.exception(...)`` / ``exc_info=`` (the delivery/router/transform catches, the
|
|
90
|
+
``_on_*_worker_done`` callbacks, the file/db/remotefile pollers, and the cluster leader-sweep /
|
|
91
|
+
heartbeat loops) renders that exception's full traceback into the general log. Installing this as a
|
|
92
|
+
**handler filter** redacts every such site *by construction* — current and future — so PHI safety
|
|
93
|
+
doesn't depend on each call site remembering to pre-redact. ``redact`` rewrites only HL7-shaped spans
|
|
94
|
+
(segment lines + runs carrying ≥2 ``|^~&`` delimiters), so ordinary operational messages pass through
|
|
95
|
+
unchanged. Pair it with :class:`ControlCharScrubFilter` (added after, so it scrubs the redacted text).
|
|
96
|
+
|
|
97
|
+
*Residual:* a bare free-text name a user invents (e.g. ``raise ValueError("DOE^JANE")`` with no
|
|
98
|
+
surrounding segment) is not HL7-shaped and is not caught — the "never put PHI in an exception
|
|
99
|
+
message" convention remains the control for that (see :mod:`messagefoundry.redaction`)."""
|
|
100
|
+
|
|
101
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
102
|
+
message = record.getMessage()
|
|
103
|
+
scrubbed = redact(message)
|
|
104
|
+
if scrubbed != message:
|
|
105
|
+
record.msg = scrubbed
|
|
106
|
+
record.args = ()
|
|
107
|
+
# The realistic PHI vector is a chained exception carrying a raw body. Render the traceback
|
|
108
|
+
# (chained causes included by default) and redact it; clear exc_info in BOTH paths so no
|
|
109
|
+
# formatter (even a custom one ignoring exc_text) can re-render the raw exception.
|
|
110
|
+
if record.exc_text:
|
|
111
|
+
record.exc_text = redact(record.exc_text)
|
|
112
|
+
record.exc_info = None
|
|
113
|
+
elif record.exc_info:
|
|
114
|
+
record.exc_text = redact(_EXC_RENDERER.formatException(record.exc_info))
|
|
115
|
+
record.exc_info = None
|
|
116
|
+
if record.stack_info:
|
|
117
|
+
record.stack_info = redact(record.stack_info)
|
|
118
|
+
return True
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class JsonFormatter(logging.Formatter):
|
|
122
|
+
"""Render each record as a single line of JSON — one object per line — for a log shipper / SIEM
|
|
123
|
+
(sec-offbox-log).
|
|
124
|
+
|
|
125
|
+
PHI redaction + control-char scrubbing run upstream as **handler filters** (see
|
|
126
|
+
:func:`configure_logging`), so by the time ``format`` runs ``record.getMessage()`` and
|
|
127
|
+
``record.exc_text`` are already redacted; ``json.dumps`` additionally escapes any residual control
|
|
128
|
+
characters, so a record can never break the one-object-per-line framing (ASVS 16.4.1). UTC ``Z``
|
|
129
|
+
timestamps match the text formatter (16.2.2). The exception/stack fields are populated **and
|
|
130
|
+
already redacted** by :class:`RedactionFilter` (which clears ``exc_info`` after rendering), so they
|
|
131
|
+
are emitted from ``exc_text``/``stack_info`` without re-rendering the raw exception."""
|
|
132
|
+
|
|
133
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
134
|
+
payload: dict[str, Any] = {
|
|
135
|
+
# UTC ``Z`` timestamp, byte-for-byte parity with the text formatter (gmtime + _DATE_FORMAT).
|
|
136
|
+
"time": time.strftime(_DATE_FORMAT, time.gmtime(record.created)),
|
|
137
|
+
"level": record.levelname,
|
|
138
|
+
"logger": record.name,
|
|
139
|
+
"message": record.getMessage(),
|
|
140
|
+
}
|
|
141
|
+
if record.exc_text:
|
|
142
|
+
payload["exception"] = record.exc_text
|
|
143
|
+
elif record.exc_info:
|
|
144
|
+
# Defensive: RedactionFilter normally pre-renders + redacts exc_text and clears exc_info,
|
|
145
|
+
# so this branch is dead on the configured handlers. If JsonFormatter is ever attached to a
|
|
146
|
+
# filter-less handler, redact here too so PHI safety doesn't depend on call-site discipline.
|
|
147
|
+
payload["exception"] = redact(self.formatException(record.exc_info))
|
|
148
|
+
if record.stack_info:
|
|
149
|
+
payload["stack"] = record.stack_info
|
|
150
|
+
return json.dumps(payload, ensure_ascii=False)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@dataclass(frozen=True)
|
|
154
|
+
class SyslogForward:
|
|
155
|
+
"""Off-box syslog forwarding target (sec-offbox-log). A primitive value object so this module stays
|
|
156
|
+
free of a config import (``config.settings`` imports ``LOG_LEVELS`` from here — the dependency must
|
|
157
|
+
not go the other way). ``protocol`` is ``"udp"`` (RFC 5426; fire-and-forget) or ``"tcp"`` (RFC 6587;
|
|
158
|
+
a down collector is tolerated — see :func:`configure_logging`); ``fmt`` is ``"json"`` or ``"text"``
|
|
159
|
+
and is independent of the stdout format."""
|
|
160
|
+
|
|
161
|
+
host: str
|
|
162
|
+
port: int = 514
|
|
163
|
+
protocol: str = "udp"
|
|
164
|
+
fmt: str = "json"
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
#: Socket timeout (seconds) pinned on a **TCP** off-box forwarder. The engine logs synchronously from
|
|
168
|
+
#: asyncio workers on the event-loop thread, so an unbounded blocking ``sendall`` to a stalled-but-
|
|
169
|
+
#: connected collector (TCP back-pressure / a wedged SIEM) would block the whole event loop. With this
|
|
170
|
+
#: timeout, ``SysLogHandler.emit`` raises ``socket.timeout``, swallows it via ``handleError``, and drops
|
|
171
|
+
#: the record — so a stalled collector costs at most this many seconds per record, never an indefinite
|
|
172
|
+
#: stall. UDP is connectionless (fire-and-forget) and needs no timeout. For a high-volume feed prefer
|
|
173
|
+
#: UDP or a local forwarding agent; a synchronous TCP forward is best-effort by design.
|
|
174
|
+
_FORWARD_TCP_TIMEOUT = 5.0
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
class _TimeoutSysLogHandler(logging.handlers.SysLogHandler):
|
|
178
|
+
""":class:`~logging.handlers.SysLogHandler` that pins a socket timeout on its socket — including on
|
|
179
|
+
any reconnect inside ``emit`` — so a runtime send to a stalled TCP collector can't block the calling
|
|
180
|
+
thread (the asyncio event loop) indefinitely."""
|
|
181
|
+
|
|
182
|
+
def __init__(self, *args: Any, timeout: float | None = None, **kwargs: Any) -> None:
|
|
183
|
+
self._sock_timeout = timeout
|
|
184
|
+
super().__init__(*args, **kwargs) # SysLogHandler.__init__ calls createSocket() (3.11+)
|
|
185
|
+
|
|
186
|
+
def createSocket(self) -> None:
|
|
187
|
+
super().createSocket()
|
|
188
|
+
# SysLogHandler.socket is set at runtime (not in typeshed); getattr keeps this mypy-clean
|
|
189
|
+
# across typeshed versions without a fragile per-version type: ignore.
|
|
190
|
+
sock = getattr(self, "socket", None)
|
|
191
|
+
if self._sock_timeout is not None and sock is not None:
|
|
192
|
+
sock.settimeout(self._sock_timeout)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _make_formatter(fmt: str) -> logging.Formatter:
|
|
196
|
+
"""A JSON formatter for ``fmt == "json"``, else the human-readable text formatter (the default)."""
|
|
197
|
+
if fmt == "json":
|
|
198
|
+
return JsonFormatter()
|
|
199
|
+
formatter = logging.Formatter(_LOG_FORMAT, datefmt=_DATE_FORMAT)
|
|
200
|
+
formatter.converter = time.gmtime # emit UTC timestamps (16.2.2)
|
|
201
|
+
return formatter
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _install_phi_filters(handler: logging.Handler) -> None:
|
|
205
|
+
"""Attach the PHI-redaction + control-char-scrub filters to ``handler``.
|
|
206
|
+
|
|
207
|
+
Order matters: redact PHI from the raw content first, then scrub control chars from the result.
|
|
208
|
+
Applied to **every** handler (stdout and the off-box forwarder) so the forwarded stream is held to
|
|
209
|
+
the same PHI-safety + log-injection guarantees as stdout. The filters are idempotent, so a record
|
|
210
|
+
dispatched to multiple filtered handlers is safely re-scrubbed."""
|
|
211
|
+
handler.addFilter(RedactionFilter()) # PHI redaction — message + exception traceback (Gate #1)
|
|
212
|
+
handler.addFilter(ControlCharScrubFilter()) # log-injection defense (16.4.1)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _build_syslog_handler(forward: SyslogForward) -> logging.handlers.SysLogHandler:
|
|
216
|
+
"""A :class:`logging.handlers.SysLogHandler` for ``forward``. For UDP the socket is created but not
|
|
217
|
+
connected (never fails on a down collector, never blocks on send). For TCP the constructor connects
|
|
218
|
+
and may raise ``OSError`` if the collector is down at startup (:func:`configure_logging` treats that
|
|
219
|
+
as best-effort), and a runtime socket timeout (``_FORWARD_TCP_TIMEOUT``) is pinned so a stalled
|
|
220
|
+
collector can't block the calling thread (the event loop) indefinitely — emit drops the record."""
|
|
221
|
+
if forward.protocol == "tcp":
|
|
222
|
+
return _TimeoutSysLogHandler(
|
|
223
|
+
address=(forward.host, forward.port),
|
|
224
|
+
socktype=socket.SOCK_STREAM,
|
|
225
|
+
timeout=_FORWARD_TCP_TIMEOUT,
|
|
226
|
+
)
|
|
227
|
+
return logging.handlers.SysLogHandler(
|
|
228
|
+
address=(forward.host, forward.port), socktype=socket.SOCK_DGRAM
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _resolve_level(level: str) -> int:
|
|
233
|
+
resolved = logging.getLevelName(level.upper())
|
|
234
|
+
if not isinstance(resolved, int):
|
|
235
|
+
raise ValueError(f"unknown log level: {level!r}")
|
|
236
|
+
return resolved
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def configure_logging(
|
|
240
|
+
level: str = "INFO",
|
|
241
|
+
*,
|
|
242
|
+
fmt: str = "text",
|
|
243
|
+
forward: SyslogForward | None = None,
|
|
244
|
+
) -> bool:
|
|
245
|
+
"""Install the stdout handler on the root logger, route uvicorn through it, and optionally forward
|
|
246
|
+
a copy of every record off-box to a syslog/SIEM collector. Returns whether the off-box forwarder
|
|
247
|
+
was actually installed (so a caller's "forwarding enabled" log only fires when it is truly live).
|
|
248
|
+
|
|
249
|
+
``fmt`` selects the stdout rendering: ``"text"`` (default, human-readable) or ``"json"`` (one JSON
|
|
250
|
+
object per line). ``forward`` adds a second handler shipping to a remote syslog collector; both
|
|
251
|
+
handlers carry the same PHI-redaction + control-char-scrub filters, so the off-box stream is held
|
|
252
|
+
to the same guarantees as stdout.
|
|
253
|
+
|
|
254
|
+
The forwarder is **best-effort, never blocking the engine indefinitely**: UDP is fire-and-forget; a
|
|
255
|
+
TCP collector that is **unreachable at startup** is skipped (the connect error is logged on stdout
|
|
256
|
+
and the service starts without it), and a TCP collector that **stalls at runtime** is bounded by a
|
|
257
|
+
socket timeout (``_FORWARD_TCP_TIMEOUT``) so a wedged SIEM costs at most that per record (the record
|
|
258
|
+
is then dropped) rather than blocking the event-loop thread the engine logs from. The send is still
|
|
259
|
+
synchronous, so for a high-volume feed prefer UDP or a local forwarding agent.
|
|
260
|
+
|
|
261
|
+
Idempotent: replaces any handlers a previous call installed, so it is safe to call from tests as
|
|
262
|
+
well as the CLI. Pair with ``uvicorn.run(..., log_config=None)`` so uvicorn's loggers propagate to
|
|
263
|
+
these handlers instead of installing their own.
|
|
264
|
+
"""
|
|
265
|
+
numeric = _resolve_level(level)
|
|
266
|
+
|
|
267
|
+
stdout_handler = logging.StreamHandler(sys.stdout)
|
|
268
|
+
stdout_handler.setFormatter(_make_formatter(fmt))
|
|
269
|
+
_install_phi_filters(stdout_handler)
|
|
270
|
+
|
|
271
|
+
root = logging.getLogger()
|
|
272
|
+
for existing in list(root.handlers):
|
|
273
|
+
root.removeHandler(existing)
|
|
274
|
+
root.addHandler(stdout_handler)
|
|
275
|
+
root.setLevel(numeric)
|
|
276
|
+
|
|
277
|
+
forwarder_installed = False
|
|
278
|
+
if forward is not None:
|
|
279
|
+
try:
|
|
280
|
+
fwd_handler = _build_syslog_handler(forward)
|
|
281
|
+
except OSError as exc:
|
|
282
|
+
# A down TCP collector would otherwise crash startup at socket-connect time. Warn (now
|
|
283
|
+
# visible on the just-installed stdout handler) and run without the forwarder.
|
|
284
|
+
_log.warning(
|
|
285
|
+
"off-box log forwarding to %s:%d (%s) is unavailable: %s; continuing without it",
|
|
286
|
+
forward.host,
|
|
287
|
+
forward.port,
|
|
288
|
+
forward.protocol,
|
|
289
|
+
exc,
|
|
290
|
+
)
|
|
291
|
+
else:
|
|
292
|
+
fwd_handler.setFormatter(_make_formatter(forward.fmt))
|
|
293
|
+
_install_phi_filters(fwd_handler)
|
|
294
|
+
root.addHandler(fwd_handler)
|
|
295
|
+
forwarder_installed = True
|
|
296
|
+
|
|
297
|
+
# Let uvicorn's loggers flow to the root handler(s) (one shared format/stream/forwarder).
|
|
298
|
+
for name in _UVICORN_LOGGERS:
|
|
299
|
+
uvicorn_logger = logging.getLogger(name)
|
|
300
|
+
uvicorn_logger.handlers.clear()
|
|
301
|
+
uvicorn_logger.propagate = True
|
|
302
|
+
uvicorn_logger.setLevel(numeric)
|
|
303
|
+
|
|
304
|
+
silence_phi_prone_dependency_loggers()
|
|
305
|
+
return forwarder_installed
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def silence_phi_prone_dependency_loggers() -> None:
|
|
309
|
+
"""Silence third-party loggers that emit raw HL7 field values (PHI) into the general log.
|
|
310
|
+
|
|
311
|
+
``python-hl7`` (0.4.5) logs the **whole field** at ERROR on benign-but-unmapped escape sequences
|
|
312
|
+
(``hl7/util.py`` ``unescape``: ``"Error decoding value [%s], field [%s]…"``; also a full segment
|
|
313
|
+
line at ``util.py:64``) — a PHI leak hit on every message via :func:`~messagefoundry.parsing.summary.summarize`,
|
|
314
|
+
landing in NSSM's captured stdout/stderr and violating the "never log full bodies at INFO+" rule
|
|
315
|
+
(review finding C-1). Those loggers are named by module ``__file__`` (``getLogger(__file__)``), so
|
|
316
|
+
``logging.getLogger("hl7")`` does **not** reach them — we match by the package directory instead.
|
|
317
|
+
|
|
318
|
+
We drop these records entirely (level ``CRITICAL``): they carry no operational signal the engine
|
|
319
|
+
doesn't already record as an ``ERROR`` disposition with non-PHI text, and they are PHI by
|
|
320
|
+
construction. Idempotent and best-effort (a missing/renamed dependency must never break logging).
|
|
321
|
+
"""
|
|
322
|
+
try:
|
|
323
|
+
import hl7
|
|
324
|
+
import hl7.containers # noqa: F401 (registers its __file__-named logger)
|
|
325
|
+
import hl7.util # noqa: F401
|
|
326
|
+
except ImportError:
|
|
327
|
+
return
|
|
328
|
+
pkg_dir = os.path.normcase(os.path.dirname(os.path.abspath(hl7.__file__)))
|
|
329
|
+
for name in list(logging.Logger.manager.loggerDict):
|
|
330
|
+
# hl7 names its loggers getLogger(__file__) → an absolute path inside the hl7 package dir.
|
|
331
|
+
if os.path.normcase(name).startswith(pkg_dir):
|
|
332
|
+
logging.getLogger(name).setLevel(logging.CRITICAL)
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
# Copyright (C) 2026 MessageFoundry Organization and contributors
|
|
3
|
+
"""HL7 v2 parsing & validation.
|
|
4
|
+
|
|
5
|
+
Two-tier strategy (see docs/ARCHITECTURE.md):
|
|
6
|
+
|
|
7
|
+
* **Tolerant tier** — :class:`~messagefoundry.parsing.peek.Peek` (``python-hl7``) parses
|
|
8
|
+
any reasonably-formed message and lets us *peek* at fields (e.g. MSH-9 trigger) for
|
|
9
|
+
routing without choking on conformance issues. This is the hot path; every message
|
|
10
|
+
goes through it.
|
|
11
|
+
* **Strict tier** — :func:`~messagefoundry.parsing.validate.validate` (``hl7apy``) runs
|
|
12
|
+
version-aware, structure-based validation only when a channel asks for it
|
|
13
|
+
(``validation.strict = true``). Slower and stricter, so it is opt-in and off the hot
|
|
14
|
+
path.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from messagefoundry.parsing.groups import SegmentGroup
|
|
20
|
+
from messagefoundry.parsing.message import Message, RawMessage
|
|
21
|
+
from messagefoundry.parsing.peek import HL7PeekError, Peek, normalize, parse_path
|
|
22
|
+
from messagefoundry.parsing.split import split_batch, split_by_obr
|
|
23
|
+
from messagefoundry.parsing.summary import summarize
|
|
24
|
+
from messagefoundry.parsing.tree import TreeNode, parse_tree
|
|
25
|
+
from messagefoundry.parsing.validate import ValidationResult, validate
|
|
26
|
+
from messagefoundry.parsing.x12 import (
|
|
27
|
+
X12FrameReader,
|
|
28
|
+
X12Group,
|
|
29
|
+
X12Message,
|
|
30
|
+
X12Peek,
|
|
31
|
+
X12PeekError,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"Peek",
|
|
36
|
+
"Message",
|
|
37
|
+
"RawMessage",
|
|
38
|
+
"SegmentGroup",
|
|
39
|
+
"HL7PeekError",
|
|
40
|
+
"normalize",
|
|
41
|
+
"parse_path",
|
|
42
|
+
"parse_tree",
|
|
43
|
+
"TreeNode",
|
|
44
|
+
"validate",
|
|
45
|
+
"ValidationResult",
|
|
46
|
+
"summarize",
|
|
47
|
+
"split_batch",
|
|
48
|
+
"split_by_obr",
|
|
49
|
+
# X12 EDI codec (ADR 0012) — full surface under messagefoundry.parsing.x12.
|
|
50
|
+
"X12Peek",
|
|
51
|
+
"X12Group",
|
|
52
|
+
"X12Message",
|
|
53
|
+
"X12FrameReader",
|
|
54
|
+
"X12PeekError",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
# Defense-in-depth for review finding C-1: python-hl7 logs raw field values at ERROR on
|
|
58
|
+
# benign-but-unmapped escape sequences (hl7/util.py unescape), a PHI leak hit on every message via
|
|
59
|
+
# summarize(). Silence its loggers the moment the parsing layer — the only thing that triggers
|
|
60
|
+
# unescape — is imported, so CLI/embedded paths that never call configure_logging() are covered too.
|
|
61
|
+
# Idempotent; configure_logging() also calls it for the serve path.
|
|
62
|
+
from messagefoundry.logging_setup import silence_phi_prone_dependency_loggers as _silence_hl7
|
|
63
|
+
|
|
64
|
+
_silence_hl7()
|