messagefoundry 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. messagefoundry/__init__.py +108 -0
  2. messagefoundry/__main__.py +1155 -0
  3. messagefoundry/api/__init__.py +27 -0
  4. messagefoundry/api/app.py +1581 -0
  5. messagefoundry/api/approvals.py +184 -0
  6. messagefoundry/api/auth_models.py +211 -0
  7. messagefoundry/api/auth_routes.py +655 -0
  8. messagefoundry/api/field_authz.py +96 -0
  9. messagefoundry/api/models.py +374 -0
  10. messagefoundry/api/security.py +247 -0
  11. messagefoundry/api/tls.py +47 -0
  12. messagefoundry/auth/__init__.py +39 -0
  13. messagefoundry/auth/data/common_passwords.NOTICE +13 -0
  14. messagefoundry/auth/data/common_passwords.txt +10000 -0
  15. messagefoundry/auth/identity.py +71 -0
  16. messagefoundry/auth/ldap.py +264 -0
  17. messagefoundry/auth/notifications.py +68 -0
  18. messagefoundry/auth/passwords.py +53 -0
  19. messagefoundry/auth/permissions.py +120 -0
  20. messagefoundry/auth/policy.py +153 -0
  21. messagefoundry/auth/ratelimit.py +55 -0
  22. messagefoundry/auth/service.py +1323 -0
  23. messagefoundry/auth/tokens.py +26 -0
  24. messagefoundry/auth/totp.py +174 -0
  25. messagefoundry/checks.py +174 -0
  26. messagefoundry/config/__init__.py +30 -0
  27. messagefoundry/config/active_environment.py +80 -0
  28. messagefoundry/config/ai_policy.py +140 -0
  29. messagefoundry/config/code_sets.py +260 -0
  30. messagefoundry/config/connections_edit.py +200 -0
  31. messagefoundry/config/connections_file.py +287 -0
  32. messagefoundry/config/db_lookup.py +117 -0
  33. messagefoundry/config/environments.py +116 -0
  34. messagefoundry/config/ingest_time.py +83 -0
  35. messagefoundry/config/models.py +240 -0
  36. messagefoundry/config/reference.py +158 -0
  37. messagefoundry/config/response.py +83 -0
  38. messagefoundry/config/run_context.py +153 -0
  39. messagefoundry/config/settings.py +1311 -0
  40. messagefoundry/config/state.py +99 -0
  41. messagefoundry/config/tls_policy.py +110 -0
  42. messagefoundry/config/wiring.py +1918 -0
  43. messagefoundry/console/__init__.py +20 -0
  44. messagefoundry/console/__main__.py +274 -0
  45. messagefoundry/console/_async.py +107 -0
  46. messagefoundry/console/change_password.py +111 -0
  47. messagefoundry/console/client.py +552 -0
  48. messagefoundry/console/connections.py +324 -0
  49. messagefoundry/console/login.py +107 -0
  50. messagefoundry/console/mfa.py +205 -0
  51. messagefoundry/console/reauth.py +94 -0
  52. messagefoundry/console/search.py +57 -0
  53. messagefoundry/console/service_control.py +137 -0
  54. messagefoundry/console/sessions.py +122 -0
  55. messagefoundry/console/shell.py +410 -0
  56. messagefoundry/console/status.py +377 -0
  57. messagefoundry/console/users_page.py +282 -0
  58. messagefoundry/console/widgets.py +553 -0
  59. messagefoundry/generators/README.md +27 -0
  60. messagefoundry/generators/__init__.py +15 -0
  61. messagefoundry/generators/_core.py +589 -0
  62. messagefoundry/generators/_hl7data.py +428 -0
  63. messagefoundry/generators/adt.py +286 -0
  64. messagefoundry/generators/all_types.py +24 -0
  65. messagefoundry/generators/bar.py +28 -0
  66. messagefoundry/generators/dft.py +20 -0
  67. messagefoundry/generators/mdm.py +39 -0
  68. messagefoundry/generators/mfn.py +46 -0
  69. messagefoundry/generators/oml.py +32 -0
  70. messagefoundry/generators/orl.py +30 -0
  71. messagefoundry/generators/orm.py +23 -0
  72. messagefoundry/generators/oru.py +21 -0
  73. messagefoundry/generators/ras.py +20 -0
  74. messagefoundry/generators/rde.py +54 -0
  75. messagefoundry/generators/siu.py +64 -0
  76. messagefoundry/generators/vxu.py +20 -0
  77. messagefoundry/hl7schema.py +75 -0
  78. messagefoundry/last_resort.py +55 -0
  79. messagefoundry/logging_setup.py +332 -0
  80. messagefoundry/parsing/__init__.py +64 -0
  81. messagefoundry/parsing/consistency.py +166 -0
  82. messagefoundry/parsing/groups.py +228 -0
  83. messagefoundry/parsing/message.py +453 -0
  84. messagefoundry/parsing/peek.py +237 -0
  85. messagefoundry/parsing/split.py +120 -0
  86. messagefoundry/parsing/summary.py +46 -0
  87. messagefoundry/parsing/tree.py +128 -0
  88. messagefoundry/parsing/validate.py +95 -0
  89. messagefoundry/parsing/x12/__init__.py +46 -0
  90. messagefoundry/parsing/x12/delimiters.py +140 -0
  91. messagefoundry/parsing/x12/errors.py +30 -0
  92. messagefoundry/parsing/x12/interchange.py +232 -0
  93. messagefoundry/parsing/x12/message.py +200 -0
  94. messagefoundry/parsing/x12/peek.py +207 -0
  95. messagefoundry/pipeline/__init__.py +21 -0
  96. messagefoundry/pipeline/alert_sinks.py +486 -0
  97. messagefoundry/pipeline/alerts.py +100 -0
  98. messagefoundry/pipeline/cert_expiry.py +219 -0
  99. messagefoundry/pipeline/cluster.py +955 -0
  100. messagefoundry/pipeline/cluster_sqlserver.py +444 -0
  101. messagefoundry/pipeline/config_convergence.py +137 -0
  102. messagefoundry/pipeline/dryrun.py +450 -0
  103. messagefoundry/pipeline/engine.py +756 -0
  104. messagefoundry/pipeline/leader_tasks.py +158 -0
  105. messagefoundry/pipeline/reference_sync.py +369 -0
  106. messagefoundry/pipeline/retention.py +289 -0
  107. messagefoundry/pipeline/security_notify.py +168 -0
  108. messagefoundry/pipeline/state_convergence.py +143 -0
  109. messagefoundry/pipeline/wiring_runner.py +1722 -0
  110. messagefoundry/py.typed +0 -0
  111. messagefoundry/redaction.py +71 -0
  112. messagefoundry/scaffold.py +321 -0
  113. messagefoundry/secrets_dpapi.py +129 -0
  114. messagefoundry/store/__init__.py +46 -0
  115. messagefoundry/store/audit_tee.py +67 -0
  116. messagefoundry/store/base.py +758 -0
  117. messagefoundry/store/crypto.py +166 -0
  118. messagefoundry/store/keyprovider.py +192 -0
  119. messagefoundry/store/postgres.py +3447 -0
  120. messagefoundry/store/sqlserver.py +3014 -0
  121. messagefoundry/store/store.py +3790 -0
  122. messagefoundry/timezone.py +207 -0
  123. messagefoundry/transports/__init__.py +50 -0
  124. messagefoundry/transports/base.py +269 -0
  125. messagefoundry/transports/database.py +693 -0
  126. messagefoundry/transports/file.py +551 -0
  127. messagefoundry/transports/framing.py +164 -0
  128. messagefoundry/transports/loopback.py +53 -0
  129. messagefoundry/transports/mllp.py +644 -0
  130. messagefoundry/transports/remotefile.py +664 -0
  131. messagefoundry/transports/rest.py +281 -0
  132. messagefoundry/transports/signing.py +321 -0
  133. messagefoundry/transports/soap.py +507 -0
  134. messagefoundry/transports/tcp.py +307 -0
  135. messagefoundry/transports/timer.py +146 -0
  136. messagefoundry/transports/x12.py +323 -0
  137. messagefoundry-0.1.0.dist-info/METADATA +212 -0
  138. messagefoundry-0.1.0.dist-info/RECORD +142 -0
  139. messagefoundry-0.1.0.dist-info/WHEEL +4 -0
  140. messagefoundry-0.1.0.dist-info/entry_points.txt +2 -0
  141. messagefoundry-0.1.0.dist-info/licenses/LICENSE +662 -0
  142. messagefoundry-0.1.0.dist-info/licenses/NOTICE +27 -0
@@ -0,0 +1,3014 @@
1
+ # SPDX-License-Identifier: AGPL-3.0-or-later
2
+ # Copyright (C) 2026 MessageFoundry Organization and contributors
3
+ """Production SQL Server implementation of the :class:`~messagefoundry.store.base.Store` protocol.
4
+
5
+ Runs the full ADR-0001 staged pipeline (ingress -> routed -> outbound) + ADR-0013 query/response on a
6
+ unified ``queue`` table, mirroring the SQLite :class:`~messagefoundry.store.store.MessageStore`
7
+ semantics (at-least-once delivery, retries, replay, dead-lettering, retention, encryption-at-rest) in
8
+ T-SQL over ``aioodbc``. Concurrency uses SQL Server row-locking — ``claim_ready`` claims due rows with
9
+ ``WITH (READPAST, UPDLOCK, ROWLOCK)`` so independent delivery workers don't block or double-claim — plus
10
+ RCSI and an ``sp_getapplock``-serialized finalizer, lifting SQLite's single-writer ceiling. Both
11
+ ``supports_ingest_stage`` and ``supports_response_capture`` are True.
12
+
13
+ ``aioodbc`` is an **optional extra** (``pip install 'messagefoundry[sqlserver]'``) and also needs the
14
+ Microsoft ODBC Driver 18 at the OS level. It's imported lazily in :meth:`SqlServerStore.open` so
15
+ SQLite-only installs never touch it. Verified against a real SQL Server by the CI service-container job
16
+ (the store suite + the SQL Server load smoke).
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import asyncio
22
+ import json
23
+ import logging
24
+ import time
25
+ from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
26
+ from contextlib import asynccontextmanager
27
+ from types import MappingProxyType
28
+ from typing import Any
29
+ from uuid import uuid4
30
+
31
+ from messagefoundry.config.models import RetryPolicy
32
+ from messagefoundry.config.settings import (
33
+ INSECURE_TLS_ESCAPE_ENV,
34
+ SqlAuth,
35
+ StoreSettings,
36
+ insecure_tls_allowed,
37
+ )
38
+ from messagefoundry.redaction import safe_text
39
+ from messagefoundry.store.audit_tee import emit_audit_tee
40
+ from messagefoundry.store.crypto import PREFIX as _ENC_PREFIX
41
+ from messagefoundry.store.crypto import AesGcmCipher, Cipher, CipherError, IdentityCipher
42
+ from messagefoundry.store.store import (
43
+ ConnectionMetrics,
44
+ DbStatus,
45
+ DestinationMetrics,
46
+ InboundMetrics,
47
+ CapturedResponse,
48
+ MessageStatus,
49
+ MessageStore,
50
+ OutboxItem,
51
+ OutboxStatus,
52
+ SessionRecord,
53
+ Stage,
54
+ UserRecord,
55
+ _append_channel_scope,
56
+ audit_row_hash,
57
+ )
58
+
59
+ log = logging.getLogger(__name__)
60
+
61
+ # Schema (T-SQL). Idempotent: guarded by OBJECT_ID / IndexProperty so re-open is a no-op. Epoch
62
+ # timestamps are FLOAT; ids are NVARCHAR(64) (uuid4 hex); bodies NVARCHAR(MAX).
63
+ _SCHEMA: list[str] = [
64
+ """IF OBJECT_ID('messages','U') IS NULL CREATE TABLE messages (
65
+ id NVARCHAR(64) NOT NULL PRIMARY KEY, channel_id NVARCHAR(256) NOT NULL,
66
+ received_at FLOAT NOT NULL, source_type NVARCHAR(64) NULL, control_id NVARCHAR(256) NULL,
67
+ message_type NVARCHAR(64) NULL, raw NVARCHAR(MAX) NOT NULL, status NVARCHAR(32) NOT NULL,
68
+ error NVARCHAR(MAX) NULL, summary NVARCHAR(MAX) NULL, metadata NVARCHAR(MAX) NULL)""",
69
+ """IF INDEXPROPERTY(OBJECT_ID('messages'),'ix_messages_channel','IndexID') IS NULL
70
+ CREATE INDEX ix_messages_channel ON messages(channel_id, received_at)""",
71
+ """IF INDEXPROPERTY(OBJECT_ID('messages'),'ix_messages_control','IndexID') IS NULL
72
+ CREATE INDEX ix_messages_control ON messages(channel_id, control_id)""",
73
+ """IF OBJECT_ID('outbox','U') IS NULL CREATE TABLE outbox (
74
+ id NVARCHAR(64) NOT NULL PRIMARY KEY, message_id NVARCHAR(64) NOT NULL,
75
+ channel_id NVARCHAR(256) NOT NULL, destination_name NVARCHAR(256) NOT NULL,
76
+ payload NVARCHAR(MAX) NOT NULL, status NVARCHAR(32) NOT NULL,
77
+ attempts INT NOT NULL DEFAULT 0, next_attempt_at FLOAT NOT NULL, last_error NVARCHAR(MAX) NULL,
78
+ created_at FLOAT NOT NULL, updated_at FLOAT NOT NULL)""",
79
+ """IF INDEXPROPERTY(OBJECT_ID('outbox'),'ix_outbox_ready','IndexID') IS NULL
80
+ CREATE INDEX ix_outbox_ready ON outbox(status, next_attempt_at)""",
81
+ """IF INDEXPROPERTY(OBJECT_ID('outbox'),'ix_outbox_message','IndexID') IS NULL
82
+ CREATE INDEX ix_outbox_message ON outbox(message_id)""",
83
+ # Unified staged queue (ADR 0001) — ingress -> routed -> outbound, one row per stage-unit with a
84
+ # `stage` discriminator. The SQL Server backend originally shipped only the flat `outbox`; the
85
+ # staged pipeline (enqueue_ingress + the handoffs) and ALL delivery-side methods now read/write
86
+ # this table (outbox is retained only for legacy read-compat). `seq` IDENTITY is the FIFO
87
+ # insertion-order tiebreak (PG uses BIGSERIAL); owner/lease_expires_at are present for parity but
88
+ # written NULL on this single-node backend (reset_stale_inflight is the recovery path).
89
+ """IF OBJECT_ID('queue','U') IS NULL CREATE TABLE queue (
90
+ id NVARCHAR(64) NOT NULL PRIMARY KEY, seq BIGINT IDENTITY(1,1) NOT NULL,
91
+ message_id NVARCHAR(64) NOT NULL, stage NVARCHAR(16) NOT NULL,
92
+ channel_id NVARCHAR(256) NOT NULL, destination_name NVARCHAR(256) NULL,
93
+ handler_name NVARCHAR(256) NULL, payload NVARCHAR(MAX) NOT NULL, status NVARCHAR(32) NOT NULL,
94
+ attempts INT NOT NULL DEFAULT 0, next_attempt_at FLOAT NOT NULL, last_error NVARCHAR(MAX) NULL,
95
+ owner NVARCHAR(256) NULL, lease_expires_at FLOAT NULL,
96
+ created_at FLOAT NOT NULL, updated_at FLOAT NOT NULL,
97
+ CONSTRAINT fk_queue_message FOREIGN KEY (message_id) REFERENCES messages(id))""",
98
+ """IF INDEXPROPERTY(OBJECT_ID('queue'),'ix_queue_ready','IndexID') IS NULL
99
+ CREATE INDEX ix_queue_ready ON queue(stage, status, next_attempt_at)""",
100
+ """IF INDEXPROPERTY(OBJECT_ID('queue'),'ix_queue_message','IndexID') IS NULL
101
+ CREATE INDEX ix_queue_message ON queue(message_id)""",
102
+ """IF INDEXPROPERTY(OBJECT_ID('queue'),'ix_queue_fifo_out','IndexID') IS NULL
103
+ CREATE INDEX ix_queue_fifo_out ON queue(stage, destination_name, status, created_at, seq)""",
104
+ """IF INDEXPROPERTY(OBJECT_ID('queue'),'ix_queue_fifo_in','IndexID') IS NULL
105
+ CREATE INDEX ix_queue_fifo_in ON queue(stage, channel_id, status, created_at, seq)""",
106
+ # LOCK_ESCALATION=DISABLE: `queue` is a hot multi-writer table; a depth-triggered escalation to a
107
+ # TABLE X lock during a deep startup orphan sweep would block ALL claim/handoff workers. Degrade a
108
+ # deep sweep to many row locks under RCSI instead. Idempotent (re-running re-sets the same option).
109
+ # IF-guarded (like the indexes) so it fires at most once — a bare ALTER on every open() takes a
110
+ # Sch-M lock on the hot queue table (review). lock_escalation 2 = DISABLE.
111
+ """IF (SELECT lock_escalation FROM sys.tables WHERE object_id=OBJECT_ID('queue')) <> 2
112
+ ALTER TABLE queue SET (LOCK_ESCALATION = DISABLE)""",
113
+ """IF OBJECT_ID('message_events','U') IS NULL CREATE TABLE message_events (
114
+ id INT IDENTITY(1,1) PRIMARY KEY, message_id NVARCHAR(64) NOT NULL, ts FLOAT NOT NULL,
115
+ event NVARCHAR(64) NOT NULL, destination NVARCHAR(256) NULL, detail NVARCHAR(MAX) NULL)""",
116
+ """IF INDEXPROPERTY(OBJECT_ID('message_events'),'ix_events_message','IndexID') IS NULL
117
+ CREATE INDEX ix_events_message ON message_events(message_id, ts)""",
118
+ # Transform-accessible state (ADR 0005). Written here via transform_handoff (parity with SQLite/
119
+ # Postgres): the read-through cache is loaded at open and refreshed post-commit, so a Handler's
120
+ # cross-message state_get(...) resolves in-process. Schema matches SQLite.
121
+ """IF OBJECT_ID('state','U') IS NULL CREATE TABLE state (
122
+ namespace NVARCHAR(256) NOT NULL, [key] NVARCHAR(256) NOT NULL, value NVARCHAR(MAX) NOT NULL,
123
+ set_at FLOAT NOT NULL, message_id NVARCHAR(64) NULL,
124
+ CONSTRAINT pk_state PRIMARY KEY (namespace, [key]))""",
125
+ """IF INDEXPROPERTY(OBJECT_ID('state'),'ix_state_set_at','IndexID') IS NULL
126
+ CREATE INDEX ix_state_set_at ON state(set_at)""",
127
+ """IF OBJECT_ID('audit_log','U') IS NULL CREATE TABLE audit_log (
128
+ id INT IDENTITY(1,1) PRIMARY KEY, ts FLOAT NOT NULL, actor NVARCHAR(256) NULL,
129
+ action NVARCHAR(128) NOT NULL, channel_id NVARCHAR(256) NULL, detail NVARCHAR(MAX) NULL,
130
+ row_hash NVARCHAR(64) NULL)""",
131
+ """IF COL_LENGTH('audit_log','row_hash') IS NULL
132
+ ALTER TABLE audit_log ADD row_hash NVARCHAR(64) NULL""",
133
+ """IF INDEXPROPERTY(OBJECT_ID('audit_log'),'ix_audit_ts','IndexID') IS NULL
134
+ CREATE INDEX ix_audit_ts ON audit_log(ts)""",
135
+ """IF OBJECT_ID('pending_approvals','U') IS NULL CREATE TABLE pending_approvals (
136
+ id NVARCHAR(64) NOT NULL PRIMARY KEY, operation NVARCHAR(128) NOT NULL,
137
+ params NVARCHAR(MAX) NOT NULL, requester NVARCHAR(256) NOT NULL,
138
+ requested_at FLOAT NOT NULL, status NVARCHAR(20) NOT NULL DEFAULT 'pending',
139
+ approver NVARCHAR(256) NULL, decided_at FLOAT NULL, expires_at FLOAT NULL)""",
140
+ """IF INDEXPROPERTY(OBJECT_ID('pending_approvals'),'ix_pending_approvals_status','IndexID') IS NULL
141
+ CREATE INDEX ix_pending_approvals_status ON pending_approvals(status, requested_at)""",
142
+ """IF OBJECT_ID('users','U') IS NULL CREATE TABLE users (
143
+ id NVARCHAR(64) NOT NULL PRIMARY KEY, username NVARCHAR(256) NOT NULL UNIQUE,
144
+ auth_provider NVARCHAR(16) NOT NULL, display_name NVARCHAR(256) NULL,
145
+ email NVARCHAR(256) NULL, disabled BIT NOT NULL DEFAULT 0, created_at FLOAT NOT NULL,
146
+ updated_at FLOAT NOT NULL, last_login_at FLOAT NULL, password_hash NVARCHAR(512) NULL,
147
+ password_changed_at FLOAT NULL, must_change_password BIT NOT NULL DEFAULT 0,
148
+ failed_attempts INT NOT NULL DEFAULT 0, locked_until FLOAT NULL,
149
+ channel_scope NVARCHAR(MAX) NULL, totp_secret NVARCHAR(MAX) NULL,
150
+ totp_enabled BIT NOT NULL DEFAULT 0, totp_enrolled_at FLOAT NULL,
151
+ totp_recovery_codes NVARCHAR(MAX) NULL, last_totp_step INT NULL)""",
152
+ """IF COL_LENGTH('users','channel_scope') IS NULL
153
+ ALTER TABLE users ADD channel_scope NVARCHAR(MAX) NULL""",
154
+ # MFA (WP-14): TOTP columns ALTER-ed in for a pre-existing users table (idempotent).
155
+ """IF COL_LENGTH('users','totp_secret') IS NULL
156
+ ALTER TABLE users ADD totp_secret NVARCHAR(MAX) NULL""",
157
+ """IF COL_LENGTH('users','totp_enabled') IS NULL
158
+ ALTER TABLE users ADD totp_enabled BIT NOT NULL DEFAULT 0""",
159
+ """IF COL_LENGTH('users','totp_enrolled_at') IS NULL
160
+ ALTER TABLE users ADD totp_enrolled_at FLOAT NULL""",
161
+ """IF COL_LENGTH('users','totp_recovery_codes') IS NULL
162
+ ALTER TABLE users ADD totp_recovery_codes NVARCHAR(MAX) NULL""",
163
+ # Single-use TOTP within the step window (ASVS 6.5.1): highest consumed time-step.
164
+ """IF COL_LENGTH('users','last_totp_step') IS NULL
165
+ ALTER TABLE users ADD last_totp_step INT NULL""",
166
+ """IF OBJECT_ID('roles','U') IS NULL CREATE TABLE roles (
167
+ id NVARCHAR(64) NOT NULL PRIMARY KEY, display_name NVARCHAR(128) NOT NULL,
168
+ description NVARCHAR(512) NULL, builtin BIT NOT NULL DEFAULT 1)""",
169
+ """IF OBJECT_ID('user_roles','U') IS NULL CREATE TABLE user_roles (
170
+ user_id NVARCHAR(64) NOT NULL, role_id NVARCHAR(64) NOT NULL, assigned_at FLOAT NOT NULL,
171
+ assigned_by NVARCHAR(256) NULL, CONSTRAINT pk_user_roles PRIMARY KEY (user_id, role_id))""",
172
+ """IF OBJECT_ID('ad_group_role_map','U') IS NULL CREATE TABLE ad_group_role_map (
173
+ ad_group NVARCHAR(256) NOT NULL, role_id NVARCHAR(64) NOT NULL,
174
+ CONSTRAINT pk_ad_group_role_map PRIMARY KEY (ad_group, role_id))""",
175
+ """IF OBJECT_ID('ad_group_scope_map','U') IS NULL CREATE TABLE ad_group_scope_map (
176
+ ad_group NVARCHAR(256) NOT NULL, channel NVARCHAR(256) NOT NULL,
177
+ CONSTRAINT pk_ad_group_scope_map PRIMARY KEY (ad_group, channel))""",
178
+ """IF OBJECT_ID('sessions','U') IS NULL CREATE TABLE sessions (
179
+ token_hash NVARCHAR(64) NOT NULL PRIMARY KEY, user_id NVARCHAR(64) NOT NULL,
180
+ created_at FLOAT NOT NULL, expires_at FLOAT NOT NULL, last_used_at FLOAT NOT NULL,
181
+ revoked_at FLOAT NULL, client NVARCHAR(256) NULL, reauth_at FLOAT NULL,
182
+ mfa_verified_at FLOAT NULL)""",
183
+ """IF COL_LENGTH('sessions','reauth_at') IS NULL
184
+ ALTER TABLE sessions ADD reauth_at FLOAT NULL""",
185
+ """IF COL_LENGTH('sessions','mfa_verified_at') IS NULL
186
+ ALTER TABLE sessions ADD mfa_verified_at FLOAT NULL""",
187
+ """IF INDEXPROPERTY(OBJECT_ID('sessions'),'ix_sessions_user','IndexID') IS NULL
188
+ CREATE INDEX ix_sessions_user ON sessions(user_id)""",
189
+ """IF INDEXPROPERTY(OBJECT_ID('sessions'),'ix_sessions_expires','IndexID') IS NULL
190
+ CREATE INDEX ix_sessions_expires ON sessions(expires_at)""",
191
+ # Captured request/response replies (ADR 0013) — an IMMUTABLE ARTIFACT table (composite PK), NOT a
192
+ # queue stage, so it is invisible to _maybe_finalize's `FROM queue` scan. response_seq is replay-
193
+ # stable (1+MAX per (message_id,destination_name)). body + detail are BOTH ciphertext at rest for
194
+ # cross-backend read-API parity with PG/SQLite (which encrypt+purge+rotate detail); outcome stays
195
+ # plaintext. This is the ONE place this backend encrypts a "detail"-class column — queue.last_error,
196
+ # messages.error and message_events.detail stay plaintext here. NOTE (#120): those columns are NOT
197
+ # assumed non-PHI — every write goes through the safe_exc/safe_text PHI chokepoint (record_received /
198
+ # mark_failed / dead_letter_now / _event) so HL7-shaped content can't land, and on read they gate on
199
+ # messages:view_summary. Encrypting them at rest on this backend too is a tracked defense-in-depth
200
+ # follow-up (docs/PHI.md); until then the read gate + the "no PHI in exceptions" convention are the
201
+ # controls for any residual invented free-text.
202
+ """IF OBJECT_ID('response','U') IS NULL CREATE TABLE response (
203
+ message_id NVARCHAR(64) NOT NULL, destination_name NVARCHAR(256) NOT NULL,
204
+ response_seq INT NOT NULL, body NVARCHAR(MAX) NULL, outcome NVARCHAR(64) NOT NULL,
205
+ detail NVARCHAR(MAX) NULL, captured_at FLOAT NOT NULL,
206
+ CONSTRAINT pk_response PRIMARY KEY (message_id, destination_name, response_seq),
207
+ CONSTRAINT fk_response_message FOREIGN KEY (message_id) REFERENCES messages(id))""",
208
+ """IF INDEXPROPERTY(OBJECT_ID('response'),'ix_response_message','IndexID') IS NULL
209
+ CREATE INDEX ix_response_message ON response(message_id)""",
210
+ ]
211
+
212
+
213
+ def _odbc_brace(value: str) -> str:
214
+ """ODBC-quote a value in braces, doubling any internal ``}`` — neutralizes ``; { } =`` inside it
215
+ so an attacker-influenced value (e.g. a password) can't inject extra connection keywords."""
216
+ return "{" + value.replace("}", "}}") + "}"
217
+
218
+
219
+ def connection_string(settings: StoreSettings) -> str:
220
+ """Build an ODBC connection string for the Microsoft ODBC Driver 18 from store settings.
221
+
222
+ Free-text values are brace-quoted to prevent connection-string injection (STORE-5), and the
223
+ ``Encrypt``/``TrustServerCertificate`` security flags are emitted **last** so — ODBC being
224
+ last-wins on duplicate keywords — nothing earlier can downgrade TLS. Identity fields are also
225
+ validated up front (see ``StoreSettings._no_odbc_injection``)."""
226
+ # A weakened TLS posture (TrustServerCertificate=yes, or Encrypt=no) is MITM-able, so it REFUSES
227
+ # unless the explicit MEFOR_ALLOW_INSECURE_TLS dev escape is set (ASVS 12.3.2) — it can't be
228
+ # silently turned on in production.
229
+ if (settings.trust_server_certificate or not settings.encrypt) and not insecure_tls_allowed():
230
+ raise ValueError(
231
+ "SQL Server TLS is weakened (trust_server_certificate=true or encrypt=false), which is "
232
+ f"MITM-able. Use a trusted server certificate, or set {INSECURE_TLS_ESCAPE_ENV}=1 to "
233
+ "explicitly allow it for a trusted-network dev/test bind."
234
+ )
235
+ parts = [
236
+ "DRIVER={ODBC Driver 18 for SQL Server}",
237
+ f"SERVER={settings.server},{settings.port}", # server validated; port is an int
238
+ f"DATABASE={_odbc_brace(settings.database or '')}",
239
+ f"Connection Timeout={settings.connect_timeout}",
240
+ f"APP={_odbc_brace(settings.application_name)}",
241
+ ]
242
+ if settings.auth is SqlAuth.SQL:
243
+ parts.append(f"UID={_odbc_brace(settings.username or '')}")
244
+ parts.append(f"PWD={_odbc_brace(settings.password or '')}")
245
+ elif settings.auth is SqlAuth.INTEGRATED:
246
+ parts.append("Trusted_Connection=yes")
247
+ elif settings.auth is SqlAuth.ENTRA:
248
+ parts.append("Authentication=ActiveDirectoryDefault")
249
+ parts.append(f"Encrypt={'yes' if settings.encrypt else 'no'}")
250
+ parts.append(f"TrustServerCertificate={'yes' if settings.trust_server_certificate else 'no'}")
251
+ return ";".join(parts) + ";"
252
+
253
+
254
+ class SqlServerStore:
255
+ """SQL Server-backed durable queue (the :class:`Store` protocol). Open with :meth:`open`."""
256
+
257
+ # The staged ingress pipeline (enqueue_ingress + the ingress->routed->outbound handoffs) is
258
+ # implemented on the unified ``queue`` table: atomic DELETE...OUTPUT claim-handoffs, an
259
+ # sp_getapplock-serialized queue-aware finalizer, and RCSI for non-blocking claim/finalize
260
+ # (ADR 0001; BACKLOG #1 closed). The engine runs the staged runner on this backend.
261
+ supports_ingest_stage = True
262
+
263
+ # Request/response capture + re-ingress (ADR 0013) IS supported on the SQL Server store: captured
264
+ # replies persist to the `response` table (body + detail ciphertext, outcome plaintext) and
265
+ # re-ingress rides a staged Stage.RESPONSE work-row. The runner may start a capturing outbound + the
266
+ # re-ingress worker on this backend.
267
+ supports_response_capture = True
268
+
269
+ def __init__(self, pool: Any, settings: StoreSettings, *, cipher: Cipher | None = None) -> None:
270
+ self._pool = pool
271
+ self._settings = settings
272
+ self._cipher: Cipher = cipher or IdentityCipher()
273
+ self.path = f"{settings.server}/{settings.database}" # descriptor for db_status
274
+ # ADR 0005 transform-state read-through cache (parity with SQLite/PG): loaded at open, updated
275
+ # post-commit by transform_handoff, surfaced via state_view() so a Handler's cross-message
276
+ # state_get(...) resolves in-process.
277
+ self._state_cache: dict[tuple[str, str], Any] = {}
278
+ # Serializes audit-chain appends in-process (the store is the single audit writer per engine
279
+ # process; active-passive = one active node) — see record_audit.
280
+ self._audit_lock = asyncio.Lock()
281
+
282
+ @classmethod
283
+ async def open(
284
+ cls, settings: StoreSettings, *, cipher: Cipher | None = None
285
+ ) -> "SqlServerStore":
286
+ try:
287
+ import aioodbc
288
+ except ImportError as exc: # pragma: no cover - exercised only without the extra
289
+ raise RuntimeError(
290
+ "SQL Server backend requires the 'sqlserver' extra: "
291
+ "pip install 'messagefoundry[sqlserver]' (plus the Microsoft ODBC Driver 18)"
292
+ ) from exc
293
+ # RCSI must be enabled BEFORE the pool exists: its one-time ALTER ... WITH ROLLBACK IMMEDIATE
294
+ # takes momentary exclusivity, and with no MEFOR pool session open yet it has nothing of ours
295
+ # to terminate (concurrency_fixes (a)).
296
+ await cls._ensure_database_options(settings)
297
+ pool = await aioodbc.create_pool(
298
+ dsn=connection_string(settings),
299
+ minsize=1,
300
+ maxsize=max(1, settings.pool_size),
301
+ autocommit=False,
302
+ )
303
+ store = cls(pool, settings, cipher=cipher)
304
+ try:
305
+ await store._ensure_schema()
306
+ await store._encrypt_existing_rows() # one-time PHI-at-rest migration when a key is set
307
+ await store._backfill_audit_chain() # chain any pre-existing (unhashed) audit rows
308
+ await store._load_state_cache() # ADR 0005 read-through cache warm-up
309
+ except Exception:
310
+ # Don't leak the pool if first-open initialization fails (M-6).
311
+ pool.close()
312
+ await pool.wait_closed()
313
+ raise
314
+ return store
315
+
316
+ async def _backfill_audit_chain(self) -> None:
317
+ """Fill ``row_hash`` for audit rows written before hash-chaining (idempotent; fills only
318
+ NULLs, chained from the prior row)."""
319
+ rows = await self._fetchall(
320
+ "SELECT id, ts, actor, action, channel_id, detail, row_hash FROM audit_log ORDER BY id"
321
+ )
322
+ prev = ""
323
+ updates: list[tuple[str, int]] = []
324
+ for r in rows:
325
+ if r["row_hash"]:
326
+ prev = r["row_hash"]
327
+ continue
328
+ prev = audit_row_hash(
329
+ prev,
330
+ ts=r["ts"],
331
+ actor=r["actor"],
332
+ action=r["action"],
333
+ channel_id=r["channel_id"],
334
+ detail=r["detail"],
335
+ )
336
+ updates.append((prev, r["id"]))
337
+ if updates:
338
+ # Runs in open() before the store is returned, so no concurrent record_audit can race it.
339
+ async with self._acquire() as conn:
340
+ cur = await conn.cursor()
341
+ try:
342
+ for row_hash, rid in updates:
343
+ await cur.execute(
344
+ "UPDATE audit_log SET row_hash=? WHERE id=?", (row_hash, rid)
345
+ )
346
+ await conn.commit()
347
+ except Exception:
348
+ await conn.rollback()
349
+ raise
350
+
351
+ async def _encrypt_existing_rows(self) -> None:
352
+ """Re-encrypt legacy plaintext bodies in place when encryption is enabled (STORE-1).
353
+
354
+ Idempotent + batched: skips rows already carrying the ciphertext prefix."""
355
+ if not self._cipher.encrypts:
356
+ return
357
+ like = f"{_ENC_PREFIX}%"
358
+ total = 0
359
+ for table, column in (
360
+ ("messages", "raw"),
361
+ ("queue", "payload"),
362
+ ("outbox", "payload"),
363
+ (
364
+ "users",
365
+ "totp_secret",
366
+ ), # MFA secret (WP-14): id-keyed, NULL rows excluded by NOT LIKE
367
+ ):
368
+ while True:
369
+ rows = await self._fetchall(
370
+ f"SELECT TOP (500) id, {column} FROM {table} WHERE {column} NOT LIKE ?", (like,)
371
+ )
372
+ if not rows:
373
+ break
374
+ async with self._acquire() as conn:
375
+ cur = await conn.cursor()
376
+ try:
377
+ for r in rows:
378
+ await cur.execute(
379
+ f"UPDATE {table} SET {column}=? WHERE id=?",
380
+ (self._cipher.encrypt(r[column]), r["id"]),
381
+ )
382
+ await conn.commit()
383
+ except Exception:
384
+ await conn.rollback()
385
+ raise
386
+ total += len(rows)
387
+ # `response` body + detail (composite PK) — a separate pass (can't ride the id-keyed loop above).
388
+ # PG/SQLite migrate these too; without it a no-key -> key -> restart leaves captured reply PHI as
389
+ # plaintext at rest. body/detail are nullable, so guard `<> '' AND IS NOT NULL`.
390
+ for rcol in ("body", "detail"):
391
+ while True:
392
+ rows = await self._fetchall(
393
+ f"SELECT TOP (500) message_id, destination_name, response_seq, {rcol} AS v"
394
+ f" FROM response WHERE {rcol} NOT LIKE ? AND {rcol} <> '' AND {rcol} IS NOT NULL",
395
+ (like,),
396
+ )
397
+ if not rows:
398
+ break
399
+ async with self._acquire() as conn:
400
+ cur = await conn.cursor()
401
+ try:
402
+ for r in rows:
403
+ await cur.execute(
404
+ f"UPDATE response SET {rcol}=?"
405
+ " WHERE message_id=? AND destination_name=? AND response_seq=?",
406
+ (
407
+ self._cipher.encrypt(r["v"]),
408
+ r["message_id"],
409
+ r["destination_name"],
410
+ r["response_seq"],
411
+ ),
412
+ )
413
+ await conn.commit()
414
+ except Exception:
415
+ await conn.rollback()
416
+ raise
417
+ total += len(rows)
418
+ if total:
419
+ log.info("encrypted %d existing message/outbox/response row(s) at rest", total)
420
+
421
+ @staticmethod
422
+ async def _ensure_database_options(settings: StoreSettings) -> None:
423
+ """Enable READ_COMMITTED_SNAPSHOT (RCSI) so the staged claim/finalize paths read on a
424
+ row-version snapshot rather than taking shared locks that deadlock writers under concurrent
425
+ load (concurrency_fixes (a)). Runs on its OWN autocommit connection BEFORE the pool is
426
+ created, so the momentary exclusivity of ``WITH ROLLBACK IMMEDIATE`` has no sibling MEFOR
427
+ session to terminate; IF-guarded on the live state, so the disruptive ALTER fires at most ONCE
428
+ (greenfield first boot) and every later open()/failover is a detect-and-skip no-op. Degrades
429
+ to a warning (never fails open()) when the principal lacks ALTER DATABASE or the lock cannot
430
+ be taken — emitting the exact statement for a DBA to run out-of-band."""
431
+ import aioodbc
432
+
433
+ db = settings.database
434
+ try:
435
+ conn = await aioodbc.connect(dsn=connection_string(settings), autocommit=True)
436
+ except Exception as exc: # noqa: BLE001 - the pool open below surfaces a real connect failure
437
+ log.warning("skipping the RCSI check on %r (could not connect): %s", db, exc)
438
+ return
439
+ try:
440
+ cur = await conn.cursor()
441
+ await cur.execute(
442
+ "SELECT is_read_committed_snapshot_on, snapshot_isolation_state "
443
+ "FROM sys.databases WHERE name = DB_NAME()"
444
+ )
445
+ row = await cur.fetchone()
446
+ # If we cannot read the state, do NOT attempt a disruptive ALTER.
447
+ rcsi_on = bool(row[0]) if row else True
448
+ snapshot_on = (row[1] in (1, 2)) if row else True
449
+ if not rcsi_on:
450
+ try:
451
+ await cur.execute(
452
+ "ALTER DATABASE CURRENT SET READ_COMMITTED_SNAPSHOT ON WITH ROLLBACK IMMEDIATE"
453
+ )
454
+ log.info("enabled READ_COMMITTED_SNAPSHOT on database %r", db)
455
+ except Exception as exc: # noqa: BLE001 - permission/lock: degrade to a DBA pointer
456
+ log.warning(
457
+ "could not enable READ_COMMITTED_SNAPSHOT on %r (%s); a DBA should run once: "
458
+ "ALTER DATABASE [%s] SET READ_COMMITTED_SNAPSHOT ON WITH ROLLBACK IMMEDIATE — "
459
+ "without it the staged claim/finalize paths are more deadlock-prone under load",
460
+ db,
461
+ exc,
462
+ db,
463
+ )
464
+ if not snapshot_on:
465
+ try:
466
+ # ALLOW_SNAPSHOT_ISOLATION is an online change (no exclusivity required).
467
+ await cur.execute("ALTER DATABASE CURRENT SET ALLOW_SNAPSHOT_ISOLATION ON")
468
+ except Exception as exc: # noqa: BLE001 - non-fatal
469
+ log.warning("could not enable ALLOW_SNAPSHOT_ISOLATION on %r: %s", db, exc)
470
+ finally:
471
+ await conn.close()
472
+
473
+ async def _ensure_schema(self) -> None:
474
+ async with self._acquire() as conn:
475
+ cur = await conn.cursor()
476
+ try:
477
+ for statement in _SCHEMA:
478
+ await cur.execute(statement)
479
+ await conn.commit()
480
+ except Exception:
481
+ await conn.rollback() # roll back the partial DDL batch (M-6)
482
+ raise
483
+
484
+ async def close(self) -> None:
485
+ self._pool.close()
486
+ await self._pool.wait_closed()
487
+
488
+ # --- helpers -------------------------------------------------------------
489
+
490
+ @asynccontextmanager
491
+ async def _acquire(self) -> AsyncIterator[Any]:
492
+ """Acquire a pooled connection with the configured command (statement) timeout applied.
493
+
494
+ ``Connection Timeout`` in the DSN is only the *login* timeout; the per-statement timeout is a
495
+ pyodbc **connection** attribute (STORE-3). aioodbc's wrapper exposes ``timeout`` read-only, so
496
+ we set it on the underlying ``pyodbc.Connection`` (``_conn``); aioodbc 0.5.0 has no creation
497
+ hook (``after_created``), so we apply it per-acquire (an idempotent int assignment). The prior
498
+ ``conn.timeout = ...`` raised AttributeError and was silently swallowed, so no statement
499
+ timeout was ever applied — a hung statement then held its queue/messages row X-locks forever."""
500
+ async with self._pool.acquire() as conn:
501
+ raw = getattr(conn, "_conn", None)
502
+ if raw is not None:
503
+ raw.timeout = self._settings.command_timeout # seconds; 0 = no limit
504
+ yield conn
505
+
506
+ async def _fetchall(self, sql: str, params: tuple[Any, ...] = ()) -> list[dict[str, Any]]:
507
+ async with self._acquire() as conn:
508
+ cur = await conn.cursor()
509
+ try:
510
+ await cur.execute(sql, params) if params else await cur.execute(sql)
511
+ columns = [c[0] for c in cur.description]
512
+ rows = await cur.fetchall()
513
+ await conn.commit()
514
+ except Exception:
515
+ # autocommit=False: a failed read otherwise leaves the pooled connection mid-txn (and
516
+ # under RCSI an open snapshot pins the version store / bloats tempdb). Roll back before
517
+ # it returns to the pool so the next borrower starts clean (M-6).
518
+ await conn.rollback()
519
+ raise
520
+ return [dict(zip(columns, row)) for row in rows]
521
+
522
+ async def _fetchone(self, sql: str, params: tuple[Any, ...] = ()) -> dict[str, Any] | None:
523
+ rows = await self._fetchall(sql, params)
524
+ return rows[0] if rows else None
525
+
526
+ async def _execute(self, sql: str, params: tuple[Any, ...] = ()) -> None:
527
+ """Run a single write statement (or T-SQL batch) in its own committed transaction."""
528
+ async with self._acquire() as conn:
529
+ cur = await conn.cursor()
530
+ try:
531
+ await cur.execute(sql, params)
532
+ await conn.commit()
533
+ except Exception:
534
+ await conn.rollback()
535
+ raise
536
+
537
+ @staticmethod
538
+ async def _event(
539
+ cur: Any,
540
+ message_id: str,
541
+ event: str,
542
+ destination: str | None,
543
+ detail: str | None,
544
+ now: float,
545
+ ) -> None:
546
+ detail = (
547
+ safe_text(detail) if detail else detail
548
+ ) # PHI chokepoint (#120); SQL Server stores plaintext
549
+ await cur.execute(
550
+ "INSERT INTO message_events (message_id, ts, event, destination, detail)"
551
+ " VALUES (?,?,?,?,?)",
552
+ (message_id, now, event, destination, detail),
553
+ )
554
+
555
+ async def _applock(self, cur: Any, resource: str) -> None:
556
+ """Take a transaction-scoped exclusive ``sp_getapplock`` — the T-SQL analog of PG's advisory
557
+ lock. A NAMED lock in its own space: it never locks ``queue``/``messages`` rows, so it cannot
558
+ invert the producers' queue->messages lock order (no AB/BA deadlock), and it is re-entrant per
559
+ (resource, transaction). ``@LockOwner='Transaction'`` auto-releases it at the caller's commit/
560
+ rollback, so the caller MUST be in an open (autocommit=False) transaction. Raises on a negative
561
+ return code (timeout/deadlock/error) rather than proceeding unserialized — a swallowed timeout
562
+ would fork the audit chain or double-finalize a message."""
563
+ ct = self._settings.command_timeout
564
+ timeout_ms = (
565
+ int(ct * 1000) if ct else -1
566
+ ) # ms; -1 = wait (the pyodbc query timeout backstops)
567
+ await cur.execute(
568
+ "SET NOCOUNT ON;"
569
+ " DECLARE @rc INT;"
570
+ " EXEC @rc = sp_getapplock @Resource=?, @LockMode='Exclusive',"
571
+ " @LockOwner='Transaction', @LockTimeout=?;"
572
+ " SELECT @rc",
573
+ (resource, timeout_ms),
574
+ )
575
+ row = await cur.fetchone()
576
+ rc = int(row[0]) if row and row[0] is not None else -999
577
+ if rc < 0: # -1 timeout, -2 cancelled, -3 deadlock victim, -999 bad param
578
+ raise RuntimeError(f"sp_getapplock({resource!r}) failed: rc={rc}")
579
+
580
+ async def _lock_finalize_batch(self, cur: Any, message_ids: Iterable[str]) -> None:
581
+ """Pre-acquire the per-message finalize applock for every id in CANONICAL sorted order, so a
582
+ multi-message finalizer (an orphan sweep / cancel_queued) can never deadlock another by taking
583
+ the per-id locks in a different order. Re-entrant: a later ``_maybe_finalize`` re-take of the
584
+ same (resource, transaction) is a no-op."""
585
+ for mid in sorted(set(message_ids)):
586
+ await self._applock(cur, f"mefor:finalize:{mid}")
587
+
588
+ async def _fifo_created_at(
589
+ self, cur: Any, stage: str, lane_col: str, lane_val: str, now: float
590
+ ) -> float:
591
+ """The ``created_at`` to stamp on a new ``stage`` row so per-lane FIFO order stays monotonic
592
+ even if the wall clock regresses: ``max(now, the lane's current max created_at)``. ``lane_col``
593
+ is a code-controlled literal (allow-listed below — never user input). One grouped MAX per lane
594
+ bounds the lock-hold window under high fan-out; under RCSI it reads the committed snapshot
595
+ without blocking writers."""
596
+ if lane_col not in (
597
+ "channel_id",
598
+ "destination_name",
599
+ ): # injection guard (survives python -O)
600
+ raise ValueError(f"invalid lane column: {lane_col!r}")
601
+ await cur.execute(
602
+ f"SELECT MAX(created_at) FROM queue WHERE stage=? AND {lane_col}=?", (stage, lane_val)
603
+ )
604
+ row = await cur.fetchone()
605
+ prior = row[0] if row and row[0] is not None else None
606
+ return max(now, prior) if prior is not None else now
607
+
608
+ async def _maybe_finalize(self, cur: Any, message_id: str, now: float) -> None:
609
+ """Recompute and persist a message's terminal disposition — the SOLE authority for it. Scans
610
+ ALL stages of ``queue`` so a delivered handler can't finalize the message while a sibling
611
+ handler's routed row is still in flight. Serialized per-message on the finalize applock so two
612
+ concurrent finalizers (a delivery + a transform handoff) can't lost-update ``messages.status``.
613
+ Precedence: any pending/inflight at any stage -> still moving (return); else any dead -> ERROR;
614
+ else any outbound row -> PROCESSED; else no rows + messages.status='routed' -> FILTERED (every
615
+ handler ran, delivered nothing); else leave (UNROUTED/ERROR/in-progress not clobbered)."""
616
+ # A per-message NAMED lock — NOT a messages-row lock, which would invert the queue->messages
617
+ # lock order the producers take and deadlock (error 1205). Re-entrant; released at commit.
618
+ await self._applock(cur, f"mefor:finalize:{message_id}")
619
+ await cur.execute(
620
+ "SELECT stage, status, COUNT(*) AS n FROM queue WHERE message_id=? GROUP BY stage, status",
621
+ (message_id,),
622
+ )
623
+ rows = await cur.fetchall()
624
+ statuses = {r[1] for r in rows}
625
+ if OutboxStatus.PENDING.value in statuses or OutboxStatus.INFLIGHT.value in statuses:
626
+ return # still moving through a stage
627
+ if OutboxStatus.DEAD.value in statuses:
628
+ status = MessageStatus.ERROR.value
629
+ elif any(r[0] == Stage.OUTBOUND.value for r in rows):
630
+ status = MessageStatus.PROCESSED.value
631
+ elif not rows:
632
+ # No queue rows remain: the router/handlers produced no delivery. FILTERED only if it was
633
+ # actually routed; never clobber UNROUTED / ERROR / a status already set terminal.
634
+ await cur.execute("SELECT status FROM messages WHERE id=?", (message_id,))
635
+ mrow = await cur.fetchone()
636
+ if not mrow or mrow[0] != MessageStatus.ROUTED.value:
637
+ return
638
+ status = MessageStatus.FILTERED.value
639
+ else:
640
+ return # rows exist but all terminal, non-dead, non-outbound — leave as-is (rare)
641
+ await cur.execute("UPDATE messages SET status=? WHERE id=?", (status, message_id))
642
+
643
+ @staticmethod
644
+ def _message_filter(
645
+ channel_id: str | None,
646
+ status: str | None,
647
+ message_type: str | None,
648
+ control_id: str | None,
649
+ allowed_channels: Sequence[str] | None = None,
650
+ ) -> tuple[str, tuple[Any, ...]]:
651
+ clauses: list[str] = []
652
+ params: list[Any] = []
653
+ for column, value in (
654
+ ("channel_id", channel_id),
655
+ ("status", status),
656
+ ("message_type", message_type),
657
+ ("control_id", control_id),
658
+ ):
659
+ if value is not None:
660
+ clauses.append(f"{column}=?")
661
+ params.append(value)
662
+ _append_channel_scope(clauses, params, "channel_id", allowed_channels)
663
+ where = f" WHERE {' AND '.join(clauses)}" if clauses else ""
664
+ return where, tuple(params)
665
+
666
+ # --- write path ----------------------------------------------------------
667
+
668
+ async def enqueue_message(
669
+ self,
670
+ *,
671
+ channel_id: str,
672
+ raw: str,
673
+ deliveries: Sequence[tuple[str, str]],
674
+ control_id: str | None = None,
675
+ message_type: str | None = None,
676
+ source_type: str | None = None,
677
+ summary: str | None = None,
678
+ metadata: str | None = None,
679
+ now: float | None = None,
680
+ ) -> str:
681
+ now = time.time() if now is None else now
682
+ mid = uuid4().hex
683
+ status = MessageStatus.RECEIVED.value if deliveries else MessageStatus.UNROUTED.value
684
+ async with self._acquire() as conn:
685
+ cur = await conn.cursor()
686
+ try:
687
+ await cur.execute(
688
+ "INSERT INTO messages (id, channel_id, received_at, source_type, control_id,"
689
+ " message_type, raw, status, error, summary, metadata)"
690
+ " VALUES (?,?,?,?,?,?,?,?,?,?,?)",
691
+ (
692
+ mid,
693
+ channel_id,
694
+ now,
695
+ source_type,
696
+ control_id,
697
+ message_type,
698
+ self._cipher.encrypt(raw),
699
+ status,
700
+ None,
701
+ summary,
702
+ metadata,
703
+ ),
704
+ )
705
+ for dest_name, payload in deliveries:
706
+ await cur.execute(
707
+ "INSERT INTO queue (id, message_id, stage, channel_id, destination_name,"
708
+ " handler_name, payload, status, attempts, next_attempt_at, owner,"
709
+ " lease_expires_at, created_at, updated_at)"
710
+ " VALUES (?,?,?,?,?,NULL,?,?,0,?,NULL,NULL,?,?)",
711
+ (
712
+ uuid4().hex,
713
+ mid,
714
+ Stage.OUTBOUND.value,
715
+ channel_id,
716
+ dest_name,
717
+ self._cipher.encrypt(payload),
718
+ OutboxStatus.PENDING.value,
719
+ now,
720
+ now,
721
+ now,
722
+ ),
723
+ )
724
+ await self._event(
725
+ cur, mid, "received", None, f"{len(deliveries)} destination(s)", now
726
+ )
727
+ await conn.commit()
728
+ except Exception:
729
+ await conn.rollback()
730
+ raise
731
+ return mid
732
+
733
+ async def _insert_outbound(
734
+ self, cur: Any, message_id: str, channel_id: str, dest_name: str, payload: str, now: float
735
+ ) -> None:
736
+ """Insert one ``stage='outbound'`` queue row (lane = destination_name), FIFO-clamped."""
737
+ created_at = await self._fifo_created_at(
738
+ cur, Stage.OUTBOUND.value, "destination_name", dest_name, now
739
+ )
740
+ await cur.execute(
741
+ "INSERT INTO queue (id, message_id, stage, channel_id, destination_name, handler_name,"
742
+ " payload, status, attempts, next_attempt_at, owner, lease_expires_at, created_at,"
743
+ " updated_at) VALUES (?,?,?,?,?,NULL,?,?,0,?,NULL,NULL,?,?)",
744
+ (
745
+ uuid4().hex,
746
+ message_id,
747
+ Stage.OUTBOUND.value,
748
+ channel_id,
749
+ dest_name,
750
+ self._cipher.encrypt(payload),
751
+ OutboxStatus.PENDING.value,
752
+ now,
753
+ created_at,
754
+ now,
755
+ ),
756
+ )
757
+
758
+ async def _insert_routed(
759
+ self,
760
+ cur: Any,
761
+ message_id: str,
762
+ channel_id: str,
763
+ handler_name: str,
764
+ payload: str,
765
+ now: float,
766
+ ) -> None:
767
+ """Insert one ``stage='routed'`` queue row (lane = channel_id), FIFO-clamped."""
768
+ created_at = await self._fifo_created_at(
769
+ cur, Stage.ROUTED.value, "channel_id", channel_id, now
770
+ )
771
+ await cur.execute(
772
+ "INSERT INTO queue (id, message_id, stage, channel_id, destination_name, handler_name,"
773
+ " payload, status, attempts, next_attempt_at, owner, lease_expires_at, created_at,"
774
+ " updated_at) VALUES (?,?,?,?,NULL,?,?,?,0,?,NULL,NULL,?,?)",
775
+ (
776
+ uuid4().hex,
777
+ message_id,
778
+ Stage.ROUTED.value,
779
+ channel_id,
780
+ handler_name,
781
+ self._cipher.encrypt(payload),
782
+ OutboxStatus.PENDING.value,
783
+ now,
784
+ created_at,
785
+ now,
786
+ ),
787
+ )
788
+
789
+ async def enqueue_ingress(
790
+ self,
791
+ *,
792
+ channel_id: str,
793
+ raw: str,
794
+ control_id: str | None = None,
795
+ message_type: str | None = None,
796
+ source_type: str | None = None,
797
+ summary: str | None = None,
798
+ metadata: str | None = None,
799
+ now: float | None = None,
800
+ ) -> str:
801
+ """Durably persist a freshly-received raw message to the ingress stage (status RECEIVED + one
802
+ ``stage='ingress'`` queue row holding the raw) in ONE transaction — the staged pipeline's
803
+ ACK-on-receipt boundary (ADR 0001). The inbound may be ACKed once this returns. Returns the
804
+ message id."""
805
+ now = time.time() if now is None else now
806
+ mid = uuid4().hex
807
+ async with self._acquire() as conn:
808
+ cur = await conn.cursor()
809
+ try:
810
+ await cur.execute(
811
+ "INSERT INTO messages (id, channel_id, received_at, source_type, control_id,"
812
+ " message_type, raw, status, error, summary, metadata)"
813
+ " VALUES (?,?,?,?,?,?,?,?,?,?,?)",
814
+ (
815
+ mid,
816
+ channel_id,
817
+ now,
818
+ source_type,
819
+ control_id,
820
+ message_type,
821
+ self._cipher.encrypt(raw),
822
+ MessageStatus.RECEIVED.value,
823
+ None,
824
+ summary,
825
+ metadata,
826
+ ),
827
+ )
828
+ created_at = await self._fifo_created_at(
829
+ cur, Stage.INGRESS.value, "channel_id", channel_id, now
830
+ )
831
+ await cur.execute(
832
+ "INSERT INTO queue (id, message_id, stage, channel_id, destination_name,"
833
+ " handler_name, payload, status, attempts, next_attempt_at, owner,"
834
+ " lease_expires_at, created_at, updated_at)"
835
+ " VALUES (?,?,?,?,NULL,NULL,?,?,0,?,NULL,NULL,?,?)",
836
+ (
837
+ uuid4().hex,
838
+ mid,
839
+ Stage.INGRESS.value,
840
+ channel_id,
841
+ self._cipher.encrypt(raw),
842
+ OutboxStatus.PENDING.value,
843
+ now,
844
+ created_at,
845
+ now,
846
+ ),
847
+ )
848
+ await self._event(cur, mid, "received", None, "ingress", now)
849
+ await conn.commit()
850
+ except Exception:
851
+ await conn.rollback()
852
+ raise
853
+ return mid
854
+
855
+ async def handoff(
856
+ self,
857
+ *,
858
+ ingress_id: str,
859
+ message_id: str,
860
+ channel_id: str,
861
+ deliveries: Sequence[tuple[str, str]],
862
+ disposition: MessageStatus,
863
+ now: float | None = None,
864
+ ) -> bool:
865
+ """Advance a message from ingress straight to outbound in ONE transaction (the Step-A combined
866
+ primitive): consume the in-flight ingress row, insert one outbound row per delivery, set the
867
+ post-router disposition under the finalize applock. Idempotent: returns False (no-op) if the
868
+ ingress row was already consumed by a committed prior run."""
869
+ now = time.time() if now is None else now
870
+ async with self._acquire() as conn:
871
+ cur = await conn.cursor()
872
+ try:
873
+ await cur.execute(
874
+ "DELETE FROM queue OUTPUT deleted.id WHERE id=? AND stage=? AND status=?",
875
+ (ingress_id, Stage.INGRESS.value, OutboxStatus.INFLIGHT.value),
876
+ )
877
+ if await cur.fetchone() is None:
878
+ await conn.rollback()
879
+ return False # already handed off (crash-restart) — idempotent no-op
880
+ for dest_name, payload in deliveries:
881
+ await self._insert_outbound(
882
+ cur, message_id, channel_id, dest_name, payload, now
883
+ )
884
+ await self._applock(cur, f"mefor:finalize:{message_id}")
885
+ await cur.execute(
886
+ "UPDATE messages SET status=? WHERE id=?", (disposition.value, message_id)
887
+ )
888
+ event = {
889
+ MessageStatus.ROUTED: "routed",
890
+ MessageStatus.FILTERED: "filtered",
891
+ MessageStatus.UNROUTED: "unrouted",
892
+ }.get(disposition, "routed")
893
+ await self._event(
894
+ cur, message_id, event, None, f"{len(deliveries)} destination(s)", now
895
+ )
896
+ await conn.commit()
897
+ except Exception:
898
+ await conn.rollback()
899
+ raise
900
+ return True
901
+
902
+ async def route_handoff(
903
+ self,
904
+ *,
905
+ ingress_id: str,
906
+ message_id: str,
907
+ channel_id: str,
908
+ handlers: Sequence[tuple[str, str]],
909
+ disposition: MessageStatus,
910
+ now: float | None = None,
911
+ ) -> bool:
912
+ """Advance a message from the ingress stage to the routed stage in ONE transaction (the router
913
+ half of the split pipeline): consume the in-flight ingress row, insert one ``stage='routed'``
914
+ row per selected handler (handler-list order; ``seq`` preserves it), set the intermediate
915
+ disposition (ROUTED with handlers, UNROUTED with none) under the finalize applock. Idempotent:
916
+ False if the ingress row was already consumed."""
917
+ now = time.time() if now is None else now
918
+ async with self._acquire() as conn:
919
+ cur = await conn.cursor()
920
+ try:
921
+ await cur.execute(
922
+ "DELETE FROM queue OUTPUT deleted.id WHERE id=? AND stage=? AND status=?",
923
+ (ingress_id, Stage.INGRESS.value, OutboxStatus.INFLIGHT.value),
924
+ )
925
+ if await cur.fetchone() is None:
926
+ await conn.rollback()
927
+ return False # already handed off (crash-restart) — idempotent no-op
928
+ for handler_name, payload in handlers:
929
+ await self._insert_routed(
930
+ cur, message_id, channel_id, handler_name, payload, now
931
+ )
932
+ await self._applock(cur, f"mefor:finalize:{message_id}")
933
+ await cur.execute(
934
+ "UPDATE messages SET status=? WHERE id=?", (disposition.value, message_id)
935
+ )
936
+ event = "routed" if disposition is MessageStatus.ROUTED else "unrouted"
937
+ await self._event(cur, message_id, event, None, f"{len(handlers)} handler(s)", now)
938
+ await conn.commit()
939
+ except Exception:
940
+ await conn.rollback()
941
+ raise
942
+ return True
943
+
944
+ async def transform_handoff(
945
+ self,
946
+ *,
947
+ routed_id: str,
948
+ message_id: str,
949
+ channel_id: str,
950
+ deliveries: Sequence[tuple[str, str]],
951
+ state_ops: Sequence[tuple[str, str, Any]] = (),
952
+ now: float | None = None,
953
+ ) -> bool:
954
+ """Advance one handler assignment from the routed stage to outbound in ONE transaction (the
955
+ transform half): consume the in-flight routed row, apply each declared state write (ADR 0005),
956
+ insert one outbound row per delivery, then let the finalizer recompute the terminal disposition
957
+ (this method NEVER writes ``messages.status`` itself). State writes are applied in sorted
958
+ (namespace, key) order under HOLDLOCK to bound MERGE range-deadlocks, and commit atomically
959
+ with the outbound rows (exactly-once per re-run); the read-through cache is updated only AFTER
960
+ commit. Idempotent: False if the routed row was already consumed."""
961
+ now = time.time() if now is None else now
962
+ applied: list[tuple[tuple[str, str], Any]] = []
963
+ async with self._acquire() as conn:
964
+ cur = await conn.cursor()
965
+ try:
966
+ await cur.execute(
967
+ "DELETE FROM queue OUTPUT deleted.id WHERE id=? AND stage=? AND status=?",
968
+ (routed_id, Stage.ROUTED.value, OutboxStatus.INFLIGHT.value),
969
+ )
970
+ if await cur.fetchone() is None:
971
+ await conn.rollback()
972
+ return False # already handed off (crash-restart) — idempotent no-op
973
+ for namespace, key, value in sorted(state_ops, key=lambda op: (op[0], op[1])):
974
+ enc = self._cipher.encrypt(json.dumps(value))
975
+ await cur.execute(
976
+ "MERGE state WITH (HOLDLOCK) AS t"
977
+ " USING (SELECT ? AS namespace, ? AS [key]) AS s"
978
+ " ON t.namespace=s.namespace AND t.[key]=s.[key]"
979
+ " WHEN MATCHED THEN UPDATE SET value=?, set_at=?, message_id=?"
980
+ " WHEN NOT MATCHED THEN INSERT (namespace, [key], value, set_at, message_id)"
981
+ " VALUES (?,?,?,?,?);",
982
+ (
983
+ namespace,
984
+ key,
985
+ enc,
986
+ now,
987
+ message_id,
988
+ namespace,
989
+ key,
990
+ enc,
991
+ now,
992
+ message_id,
993
+ ),
994
+ )
995
+ applied.append(((namespace, key), value))
996
+ for dest_name, payload in deliveries:
997
+ await self._insert_outbound(
998
+ cur, message_id, channel_id, dest_name, payload, now
999
+ )
1000
+ await self._event(
1001
+ cur, message_id, "transformed", None, f"{len(deliveries)} destination(s)", now
1002
+ )
1003
+ # Finalizer is the sole disposition authority here (no direct messages.status write).
1004
+ await self._maybe_finalize(cur, message_id, now)
1005
+ await conn.commit()
1006
+ except Exception:
1007
+ await conn.rollback()
1008
+ raise
1009
+ # Commit succeeded → publish the committed state writes to the read-through cache.
1010
+ for ck, cv in applied:
1011
+ self._state_cache[ck] = cv
1012
+ return True
1013
+
1014
+ async def complete_with_response(
1015
+ self,
1016
+ outbox_id: str,
1017
+ *,
1018
+ body: str,
1019
+ outcome: str,
1020
+ detail: str | None = None,
1021
+ reingress_to: str | None = None,
1022
+ now: float | None = None,
1023
+ ) -> None:
1024
+ """Mark one outbound row delivered AND persist the partner's captured reply in ONE transaction
1025
+ (ADR 0013). ``response_seq`` is ``1 + MAX`` per (message_id, destination_name) so it is replay-
1026
+ stable; the ``response`` table is invisible to the finalizer (it scans ``queue`` only). When
1027
+ ``reingress_to`` is set the same transaction also inserts the drainable ``Stage.RESPONSE`` work-
1028
+ row (which holds the origin non-terminal until ``ingress_handoff`` consumes it). body + detail
1029
+ are ciphertext; outcome is plaintext."""
1030
+ now = time.time() if now is None else now
1031
+ async with self._acquire() as conn:
1032
+ cur = await conn.cursor()
1033
+ try:
1034
+ # Leading SELECT (also opens the txn so _maybe_finalize's applock is never first).
1035
+ await cur.execute(
1036
+ "SELECT message_id, destination_name, attempts FROM queue WHERE id=?",
1037
+ (outbox_id,),
1038
+ )
1039
+ row = await cur.fetchone()
1040
+ if row is None:
1041
+ await conn.commit()
1042
+ return
1043
+ message_id, destination_name, attempts = row[0], row[1], row[2]
1044
+ await cur.execute(
1045
+ "UPDATE queue SET status=?, last_error=NULL, updated_at=?, owner=NULL,"
1046
+ " lease_expires_at=NULL WHERE id=?",
1047
+ (OutboxStatus.DONE.value, now, outbox_id),
1048
+ )
1049
+ await cur.execute(
1050
+ "SELECT COALESCE(MAX(response_seq), 0) + 1 FROM response"
1051
+ " WHERE message_id=? AND destination_name=?",
1052
+ (message_id, destination_name),
1053
+ )
1054
+ seq = int((await cur.fetchone())[0])
1055
+ # Inline the PG _enc empty-guard: encrypt only a truthy value (never '' / None).
1056
+ enc_body = self._cipher.encrypt(body) if body else body
1057
+ enc_detail = self._cipher.encrypt(detail) if detail else detail
1058
+ await cur.execute(
1059
+ "INSERT INTO response"
1060
+ " (message_id, destination_name, response_seq, body, outcome, detail, captured_at)"
1061
+ " VALUES (?,?,?,?,?,?,?)",
1062
+ (message_id, destination_name, seq, enc_body, outcome, enc_detail, now),
1063
+ )
1064
+ if reingress_to is not None:
1065
+ # ADR 0013 Increment 2: a drainable Stage.RESPONSE work-row in the SAME txn (orphan-
1066
+ # free) — a token referencing the immutable artifact by PK, on the loopback lane.
1067
+ artifact_ref = f"{message_id}\x1f{destination_name}\x1f{seq}"
1068
+ work_created = await self._fifo_created_at(
1069
+ cur, Stage.RESPONSE.value, "channel_id", reingress_to, now
1070
+ )
1071
+ await cur.execute(
1072
+ "INSERT INTO queue (id, message_id, stage, channel_id, destination_name,"
1073
+ " handler_name, payload, status, attempts, next_attempt_at, owner,"
1074
+ " lease_expires_at, created_at, updated_at)"
1075
+ " VALUES (?,?,?,?,NULL,NULL,?,?,0,?,NULL,NULL,?,?)",
1076
+ (
1077
+ uuid4().hex,
1078
+ message_id,
1079
+ Stage.RESPONSE.value,
1080
+ reingress_to,
1081
+ self._cipher.encrypt(artifact_ref),
1082
+ OutboxStatus.PENDING.value,
1083
+ now,
1084
+ work_created,
1085
+ now,
1086
+ ),
1087
+ )
1088
+ await self._event(
1089
+ cur,
1090
+ message_id,
1091
+ "delivered",
1092
+ destination_name,
1093
+ f"attempt {attempts} (response {outcome})",
1094
+ now,
1095
+ )
1096
+ # Finalizer last; preceded by the SELECT above so its applock is not the first statement.
1097
+ await self._maybe_finalize(cur, message_id, now)
1098
+ await conn.commit()
1099
+ except Exception:
1100
+ await conn.rollback()
1101
+ raise
1102
+
1103
+ async def correlate_response(self, message_id: str) -> list[CapturedResponse]:
1104
+ """Captured replies for a message (ADR 0013), ordered by destination then ``response_seq`` ASC
1105
+ (so the latest reply per destination is last). ``body`` + ``detail`` are both decrypted (both
1106
+ ciphertext); a NULL (never-captured or purged) body/detail returns ``None`` while an empty ``''``
1107
+ round-trips as ``''`` — parity with PG/SQLite ``_dec``; ``outcome`` is plaintext."""
1108
+ rows = await self._fetchall(
1109
+ "SELECT message_id, destination_name, response_seq, body, outcome, detail, captured_at"
1110
+ " FROM response WHERE message_id=? ORDER BY destination_name, response_seq",
1111
+ (message_id,),
1112
+ )
1113
+ return [
1114
+ CapturedResponse(
1115
+ message_id=r["message_id"],
1116
+ destination_name=r["destination_name"],
1117
+ response_seq=int(r["response_seq"]),
1118
+ outcome=r["outcome"],
1119
+ detail=self._cipher.decrypt(r["detail"]) if r["detail"] is not None else None,
1120
+ captured_at=float(r["captured_at"]),
1121
+ body=self._cipher.decrypt(r["body"]) if r["body"] is not None else None,
1122
+ )
1123
+ for r in rows
1124
+ ]
1125
+
1126
+ async def ingress_handoff(
1127
+ self,
1128
+ *,
1129
+ response_row_id: str,
1130
+ loopback_channel_id: str,
1131
+ correlation_depth_cap: int,
1132
+ control_id: str | None,
1133
+ message_type: str | None,
1134
+ summary: str | None,
1135
+ peek_failed: bool = False,
1136
+ now: float | None = None,
1137
+ ) -> bool:
1138
+ """Consume one in-flight ``Stage.RESPONSE`` work-row and re-ingress the captured reply as a new
1139
+ message on the loopback inbound (ADR 0013 Increment 2), in ONE transaction. Idempotent: returns
1140
+ ``False`` if the work-row was already consumed. A corrupt/unparseable ref or a re-ingress that
1141
+ would exceed ``correlation_depth_cap`` is dead-lettered (and the token still consumed). The
1142
+ re-ingress message id is content-addressed (deterministic), so a re-run never double-inserts the
1143
+ child."""
1144
+ now = time.time() if now is None else now
1145
+ async with self._acquire() as conn:
1146
+ cur = await conn.cursor()
1147
+ try:
1148
+ # (1) Guard-read the in-flight work-row (also opens the txn -> applock not first).
1149
+ await cur.execute(
1150
+ "SELECT message_id, payload FROM queue WHERE id=? AND stage=? AND status=?",
1151
+ (response_row_id, Stage.RESPONSE.value, OutboxStatus.INFLIGHT.value),
1152
+ )
1153
+ wr = await cur.fetchone()
1154
+ if wr is None:
1155
+ await conn.rollback()
1156
+ return False # already consumed by a committed prior run (idempotent no-op)
1157
+ origin_id = wr[0]
1158
+ # (2) Decrypt + parse the artifact ref; ANY failure -> consume-and-dead-letter.
1159
+ try:
1160
+ ref = self._cipher.decrypt(wr[1]) or ""
1161
+ origin_msg_id, dest, seq_s = ref.split("\x1f")
1162
+ seq = int(seq_s)
1163
+ except Exception: # noqa: BLE001 - any decrypt/parse failure = an unrecoverable ref
1164
+ await cur.execute(
1165
+ "UPDATE queue SET status=?, last_error=?, next_attempt_at=?, updated_at=?"
1166
+ " WHERE id=?",
1167
+ (
1168
+ OutboxStatus.DEAD.value,
1169
+ "re-ingress work-row reference is corrupt/unparseable",
1170
+ now,
1171
+ now,
1172
+ response_row_id,
1173
+ ),
1174
+ )
1175
+ await self._event(cur, origin_id, "dead", None, "re-ingress ref corrupt", now)
1176
+ await self._maybe_finalize(cur, origin_id, now) # preceded by step-1 SELECT
1177
+ await conn.commit()
1178
+ return True # CONSUME (status flipped), never re-loop
1179
+ # (3) Read the immutable artifact body.
1180
+ await cur.execute(
1181
+ "SELECT body FROM response"
1182
+ " WHERE message_id=? AND destination_name=? AND response_seq=?",
1183
+ (origin_msg_id, dest, seq),
1184
+ )
1185
+ art = await cur.fetchone()
1186
+ body = (self._cipher.decrypt(art[0]) if (art and art[0]) else "") or ""
1187
+ # (4) Correlation lineage from the origin's metadata (parse once).
1188
+ await cur.execute("SELECT metadata FROM messages WHERE id=?", (origin_id,))
1189
+ mrow = await cur.fetchone()
1190
+ loaded = json.loads(mrow[0]) if (mrow and mrow[0]) else {}
1191
+ origin_meta = loaded if isinstance(loaded, dict) else {}
1192
+ child_depth = int(origin_meta.get("correlation_depth", 0) or 0) + 1
1193
+ root = origin_meta.get("correlation_root_id") or origin_id
1194
+ # (5) Depth-cap -> consume-and-dead-letter.
1195
+ if child_depth > correlation_depth_cap:
1196
+ await cur.execute(
1197
+ "UPDATE queue SET status=?, last_error=?, next_attempt_at=?, updated_at=?"
1198
+ " WHERE id=?",
1199
+ (
1200
+ OutboxStatus.DEAD.value,
1201
+ f"re-ingress correlation depth exceeded "
1202
+ f"({child_depth} > {correlation_depth_cap})",
1203
+ now,
1204
+ now,
1205
+ response_row_id,
1206
+ ),
1207
+ )
1208
+ await self._event(
1209
+ cur, origin_id, "dead", dest, f"re-ingress depth cap ({child_depth})", now
1210
+ )
1211
+ await self._maybe_finalize(cur, origin_id, now) # preceded by step-1 SELECT
1212
+ await conn.commit()
1213
+ return True
1214
+ # (6) Deterministic child id + idempotent insert (the guarded DELETE is the real gate).
1215
+ new_mid = MessageStore._reingress_message_id(origin_id, dest, seq, body)
1216
+ await cur.execute("SELECT 1 FROM messages WHERE id=?", (new_mid,))
1217
+ if await cur.fetchone() is None:
1218
+ child_meta = json.dumps(
1219
+ {
1220
+ "correlation_id": origin_id,
1221
+ "correlation_root_id": root,
1222
+ "correlation_depth": child_depth,
1223
+ "reingress_of_seq": seq,
1224
+ }
1225
+ )
1226
+ await cur.execute(
1227
+ "INSERT INTO messages (id, channel_id, received_at, source_type, control_id,"
1228
+ " message_type, raw, status, error, summary, metadata)"
1229
+ " VALUES (?,?,?,?,?,?,?,?,?,?,?)",
1230
+ (
1231
+ new_mid,
1232
+ loopback_channel_id,
1233
+ now,
1234
+ "reingress",
1235
+ control_id,
1236
+ message_type,
1237
+ self._cipher.encrypt(body),
1238
+ MessageStatus.ERROR.value
1239
+ if peek_failed
1240
+ else MessageStatus.RECEIVED.value,
1241
+ "re-ingress body failed HL7 peek" if peek_failed else None,
1242
+ summary,
1243
+ child_meta,
1244
+ ),
1245
+ )
1246
+ if not peek_failed:
1247
+ ingress_created = await self._fifo_created_at(
1248
+ cur, Stage.INGRESS.value, "channel_id", loopback_channel_id, now
1249
+ )
1250
+ await cur.execute(
1251
+ "INSERT INTO queue (id, message_id, stage, channel_id, destination_name,"
1252
+ " handler_name, payload, status, attempts, next_attempt_at, owner,"
1253
+ " lease_expires_at, created_at, updated_at)"
1254
+ " VALUES (?,?,?,?,NULL,NULL,?,?,0,?,NULL,NULL,?,?)",
1255
+ (
1256
+ uuid4().hex,
1257
+ new_mid,
1258
+ Stage.INGRESS.value,
1259
+ loopback_channel_id,
1260
+ self._cipher.encrypt(body),
1261
+ OutboxStatus.PENDING.value,
1262
+ now,
1263
+ ingress_created,
1264
+ now,
1265
+ ),
1266
+ )
1267
+ await self._event(
1268
+ cur,
1269
+ new_mid,
1270
+ "received",
1271
+ None,
1272
+ f"reingress from {origin_id}/{dest}/seq{seq}",
1273
+ now,
1274
+ )
1275
+ await self._event(
1276
+ cur,
1277
+ origin_id,
1278
+ "reingressed",
1279
+ dest,
1280
+ f"-> {new_mid} depth {child_depth}",
1281
+ now,
1282
+ )
1283
+ # (7) Consume the token — exactly-once commit point (OUTPUT readback, never rowcount).
1284
+ await cur.execute(
1285
+ "DELETE FROM queue OUTPUT deleted.id WHERE id=? AND stage=? AND status=?",
1286
+ (response_row_id, Stage.RESPONSE.value, OutboxStatus.INFLIGHT.value),
1287
+ )
1288
+ if await cur.fetchone() is None:
1289
+ await conn.rollback()
1290
+ return False # defensive; unreachable under single-owner claim
1291
+ # (8) Finalize the origin (its last RESPONSE row is now gone).
1292
+ await self._maybe_finalize(cur, origin_id, now)
1293
+ await conn.commit()
1294
+ return True
1295
+ except Exception:
1296
+ await conn.rollback()
1297
+ raise
1298
+
1299
+ async def response_body_for_work_row(self, response_row_id: str) -> str | None:
1300
+ """The decrypted reply body behind a ``Stage.RESPONSE`` work-row (ADR 0013) — for the re-ingress
1301
+ worker's HL7 peek. ``None`` if the work-row is missing/consumed or its ref is unparseable; ``''``
1302
+ if the artifact body is missing/empty. Reads the SAME immutable artifact ``ingress_handoff``
1303
+ re-reads, so the peek and the persisted raw always agree."""
1304
+ row = await self._fetchone(
1305
+ "SELECT payload FROM queue WHERE id=? AND stage=?",
1306
+ (response_row_id, Stage.RESPONSE.value),
1307
+ )
1308
+ if row is None:
1309
+ return None
1310
+ ref = self._cipher.decrypt(row["payload"]) or ""
1311
+ try:
1312
+ mid, dest, seq_s = ref.split("\x1f")
1313
+ except ValueError:
1314
+ return None
1315
+ art = await self._fetchone(
1316
+ "SELECT body FROM response WHERE message_id=? AND destination_name=? AND response_seq=?",
1317
+ (mid, dest, int(seq_s)),
1318
+ )
1319
+ return self._cipher.decrypt(art["body"]) if (art and art["body"]) else ""
1320
+
1321
+ def state_view(self) -> Mapping[tuple[str, str], Any]:
1322
+ """Read-only view of the ADR 0005 transform-state read-through cache (parity with SQLite/PG).
1323
+ The runner publishes it around each router/transform run so a Handler's ``state_get(...)``
1324
+ resolves cross-message; ``transform_handoff`` refreshes it post-commit."""
1325
+ return MappingProxyType(self._state_cache)
1326
+
1327
+ async def _load_state_cache(self) -> None:
1328
+ """Warm the transform-state read-through cache from the ``state`` table at open (ADR 0005)."""
1329
+ rows = await self._fetchall("SELECT namespace, [key], value FROM state")
1330
+ cache: dict[tuple[str, str], Any] = {}
1331
+ for r in rows:
1332
+ try:
1333
+ cache[(r["namespace"], r["key"])] = json.loads(self._cipher.decrypt(r["value"]))
1334
+ except (CipherError, ValueError) as exc: # skip a corrupt/undecryptable state row
1335
+ log.warning(
1336
+ "skipping unreadable state row %r/%r: %s", r["namespace"], r["key"], exc
1337
+ )
1338
+ self._state_cache = cache
1339
+
1340
+ def reference_view(self) -> Mapping[str, Mapping[str, Any]]:
1341
+ """Empty reference view on the SQL Server backend (ADR 0006). Reference snapshots live in the
1342
+ SQLite store; returning an empty read-only mapping keeps a Handler's ``reference("name")``
1343
+ call shaped correctly — though it will raise ``ReferenceError`` for any name (no set synced),
1344
+ which is the honest state on this backend (reference snapshots are SQLite/Postgres-only here)."""
1345
+ return MappingProxyType({})
1346
+
1347
+ async def write_reference_snapshot(
1348
+ self, *, name: str, version: str, rows: Mapping[str, Any]
1349
+ ) -> None:
1350
+ """Not supported on the SQL Server backend — reference snapshots (ADR 0006) are SQLite/Postgres-
1351
+ only. NOTE: the engine DOES run on this backend now (staged pipeline + capture), so if a
1352
+ reference SET is configured against the SQL Server store the ReferenceSyncRunner logs a sync
1353
+ failure each interval until ADR 0006 lands here (the engine survives; the set never materializes
1354
+ and ``reference()`` raises). Present for ``Store`` protocol completeness."""
1355
+ raise NotImplementedError(
1356
+ "write_reference_snapshot is not supported on the SQL Server backend (ADR 0006 reference "
1357
+ "sets are SQLite/Postgres-only); use the SQLite or Postgres backend"
1358
+ )
1359
+
1360
+ async def converge_reference_cache(self) -> list[str]:
1361
+ """No-op on the SQL Server backend (Track B Step 6). Reference snapshots are SQLite/Postgres-only
1362
+ here (write_reference_snapshot raises), so there is no shared snapshot to read through. Present
1363
+ for ``Store`` protocol completeness; returns ``[]``."""
1364
+ return []
1365
+
1366
+ async def converge_state_cache(self) -> list[str]:
1367
+ """No-op on the SQL Server backend (Track B Step 6b): a single-node backend with no cross-node
1368
+ state convergence (transform state IS written here via transform_handoff, but never converged
1369
+ across nodes). Present for ``Store`` protocol completeness; returns ``[]``."""
1370
+ return []
1371
+
1372
+ def enable_state_convergence(self) -> None:
1373
+ """No-op on the SQL Server backend (Track B Step 6b): there is no cross-node convergence here, so
1374
+ there is no per-namespace version to bump. Present for ``Store`` protocol completeness."""
1375
+ return None
1376
+
1377
+ async def dead_letter_missing_handlers(
1378
+ self, valid_names: set[str], now: float | None = None
1379
+ ) -> int:
1380
+ """Dead-letter non-terminal routed queue rows whose handler_name is no longer in the registry
1381
+ (a removed/renamed handler) — no transform worker would drain them, so they'd strand forever.
1382
+ Call ONCE at startup, AFTER reset_stale_inflight. Per-message finalize applocks are pre-acquired
1383
+ in sorted id order to avoid multi-message deadlock; a killed routed row -> DEAD -> the finalizer
1384
+ resolves the message to ERROR."""
1385
+ now = time.time() if now is None else now
1386
+ async with self._acquire() as conn:
1387
+ cur = await conn.cursor()
1388
+ try:
1389
+ await cur.execute(
1390
+ "SELECT id, message_id, handler_name FROM queue"
1391
+ " WHERE stage=? AND status IN (?, ?)",
1392
+ (Stage.ROUTED.value, OutboxStatus.PENDING.value, OutboxStatus.INFLIGHT.value),
1393
+ )
1394
+ rows = await cur.fetchall() # positional: (id, message_id, handler_name)
1395
+ orphans = [r for r in rows if r[2] not in valid_names]
1396
+ if not orphans:
1397
+ await conn.commit()
1398
+ return 0
1399
+ error = "handler removed from registry"
1400
+ await self._lock_finalize_batch(cur, {r[1] for r in orphans})
1401
+ for row in orphans:
1402
+ await cur.execute(
1403
+ "UPDATE queue SET status=?, next_attempt_at=?, last_error=?, updated_at=?,"
1404
+ " owner=NULL, lease_expires_at=NULL WHERE id=?",
1405
+ (OutboxStatus.DEAD.value, now, error, now, row[0]),
1406
+ )
1407
+ await self._event(cur, row[1], "dead", None, error, now)
1408
+ await self._maybe_finalize(cur, row[1], now)
1409
+ await conn.commit()
1410
+ except Exception:
1411
+ await conn.rollback()
1412
+ raise
1413
+ log.warning(
1414
+ "dead-lettered %d orphaned routed row(s) at startup for missing handler(s): %s",
1415
+ len(orphans),
1416
+ ", ".join(sorted({r[2] for r in orphans})),
1417
+ )
1418
+ return len(orphans)
1419
+
1420
+ # --- retention / purge + maintenance (PHI.md §8) -------------------------
1421
+ # The RetentionRunner drives these once the staged pipeline is enabled. Bodies are blanked to ''
1422
+ # (not deleted) so cipher re-encrypt scans skip them and the FK to messages stays intact. SQL
1423
+ # Server TDE remains the at-rest baseline; this engine-side rotation/purge complements it.
1424
+
1425
+ async def reencrypt_to_active(self, *, batch: int = 500) -> int:
1426
+ """Re-encrypt body columns sitting under a RETIRED key to the active key (key rotation),
1427
+ batched per (table, column). No-op (returns 0) unless an AES-GCM keyring cipher is configured.
1428
+ Each batch's re-encrypt list is built UP FRONT so an undecryptable value raises BEFORE any
1429
+ UPDATE (all-or-nothing; PHI never dropped). Skips rows already under the active key and
1430
+ blank/purged values."""
1431
+ if not isinstance(self._cipher, AesGcmCipher):
1432
+ return 0
1433
+ active_like = f"{_ENC_PREFIX}{self._cipher.active_key_id}:%"
1434
+ total = 0
1435
+ for table, column in (("messages", "raw"), ("queue", "payload"), ("users", "totp_secret")):
1436
+ while True:
1437
+ rows = await self._fetchall(
1438
+ f"SELECT TOP (?) id, {column} AS v FROM {table}"
1439
+ f" WHERE {column} NOT LIKE ? AND {column} <> ''",
1440
+ (batch, active_like),
1441
+ )
1442
+ if not rows:
1443
+ break
1444
+ # Decrypt+re-encrypt UP FRONT: a CipherError aborts the batch before any write.
1445
+ updates = [
1446
+ (self._cipher.encrypt(self._cipher.decrypt(r["v"])), r["id"]) for r in rows
1447
+ ]
1448
+ async with self._acquire() as conn:
1449
+ cur = await conn.cursor()
1450
+ try:
1451
+ for enc, rid in updates:
1452
+ await cur.execute(
1453
+ f"UPDATE {table} SET {column}=? WHERE id=?", (enc, rid)
1454
+ )
1455
+ await conn.commit()
1456
+ except Exception:
1457
+ await conn.rollback()
1458
+ raise
1459
+ total += len(rows)
1460
+ # `state` has a composite PK (namespace, [key]) — its own pass (can't ride the id-keyed loop
1461
+ # above). transform_handoff writes state.value encrypted, so a rotation MUST rotate it too or a
1462
+ # later retired-key drop silently loses all transform cross-message state (review HIGH).
1463
+ while True:
1464
+ rows = await self._fetchall(
1465
+ "SELECT TOP (?) namespace, [key], value FROM state"
1466
+ " WHERE value NOT LIKE ? AND value <> ''",
1467
+ (batch, active_like),
1468
+ )
1469
+ if not rows:
1470
+ break
1471
+ state_updates = [
1472
+ (self._cipher.encrypt(self._cipher.decrypt(r["value"])), r["namespace"], r["key"])
1473
+ for r in rows
1474
+ ]
1475
+ async with self._acquire() as conn:
1476
+ cur = await conn.cursor()
1477
+ try:
1478
+ for enc, ns, skey in state_updates:
1479
+ await cur.execute(
1480
+ "UPDATE state SET value=? WHERE namespace=? AND [key]=?",
1481
+ (enc, ns, skey),
1482
+ )
1483
+ await conn.commit()
1484
+ except Exception:
1485
+ await conn.rollback()
1486
+ raise
1487
+ total += len(rows)
1488
+ # `response` body + detail are ciphertext with a composite PK (message_id, destination_name,
1489
+ # response_seq) — their own passes. IS NOT NULL is explicit/defensive: NOT LIKE already excludes
1490
+ # NULLs (three-valued logic) and a NULL has no ciphertext to rotate — but these columns are
1491
+ # nullable (unlike state.value/messages.raw/queue.payload), so the guard documents that intent.
1492
+ for rcol in ("body", "detail"):
1493
+ while True:
1494
+ rows = await self._fetchall(
1495
+ f"SELECT TOP (?) message_id, destination_name, response_seq, {rcol} AS v"
1496
+ f" FROM response WHERE {rcol} NOT LIKE ? AND {rcol} <> '' AND {rcol} IS NOT NULL",
1497
+ (batch, active_like),
1498
+ )
1499
+ if not rows:
1500
+ break
1501
+ resp_updates = [
1502
+ (
1503
+ self._cipher.encrypt(self._cipher.decrypt(r["v"])),
1504
+ r["message_id"],
1505
+ r["destination_name"],
1506
+ r["response_seq"],
1507
+ )
1508
+ for r in rows
1509
+ ]
1510
+ async with self._acquire() as conn:
1511
+ cur = await conn.cursor()
1512
+ try:
1513
+ for enc, rmid, rdest, rseq in resp_updates:
1514
+ await cur.execute(
1515
+ f"UPDATE response SET {rcol}=?"
1516
+ " WHERE message_id=? AND destination_name=? AND response_seq=?",
1517
+ (enc, rmid, rdest, rseq),
1518
+ )
1519
+ await conn.commit()
1520
+ except Exception:
1521
+ await conn.rollback()
1522
+ raise
1523
+ total += len(rows)
1524
+ if total:
1525
+ log.info("re-encrypted %d row(s) to the active key", total)
1526
+ return total
1527
+
1528
+ async def purge_message_bodies(self, *, older_than: float, now: float | None = None) -> int:
1529
+ """Blank message bodies (and terminal outbound payloads + event details) for messages received
1530
+ before ``older_than`` whose queue rows are all terminal — retention (PHI.md §8). Bodies are
1531
+ blanked to '' (not deleted) so the cipher re-encrypt scans skip them and the FK to messages
1532
+ stays intact. The eligible set is materialized ONCE so all three tables purge exactly the same
1533
+ messages. Returns the number of message bodies blanked."""
1534
+ async with self._acquire() as conn:
1535
+ cur = await conn.cursor()
1536
+ try:
1537
+ # CREATE (no params) so the temp table lives at CONNECTION scope; a parameterized
1538
+ # SELECT...INTO runs under sp_executesql and would scope #eligible to that proc (gone
1539
+ # before the UPDATEs). The parameterized INSERT below still populates it.
1540
+ await cur.execute("CREATE TABLE #eligible (id NVARCHAR(64) PRIMARY KEY)")
1541
+ await cur.execute(
1542
+ "INSERT INTO #eligible SELECT id FROM messages m WHERE m.received_at < ?"
1543
+ " AND NOT EXISTS (SELECT 1 FROM queue q WHERE q.message_id=m.id"
1544
+ " AND q.status IN (?, ?))",
1545
+ (older_than, OutboxStatus.PENDING.value, OutboxStatus.INFLIGHT.value),
1546
+ )
1547
+ await cur.execute(
1548
+ "UPDATE messages SET raw='', summary=NULL, error=NULL"
1549
+ " WHERE raw <> '' AND id IN (SELECT id FROM #eligible)"
1550
+ )
1551
+ purged = cur.rowcount
1552
+ await cur.execute(
1553
+ "UPDATE queue SET payload='', last_error=NULL"
1554
+ " WHERE stage=? AND status IN (?, ?) AND payload <> ''"
1555
+ " AND message_id IN (SELECT id FROM #eligible)",
1556
+ (Stage.OUTBOUND.value, OutboxStatus.DONE.value, OutboxStatus.CANCELLED.value),
1557
+ )
1558
+ await cur.execute(
1559
+ "UPDATE message_events SET detail=NULL"
1560
+ " WHERE detail IS NOT NULL AND message_id IN (SELECT id FROM #eligible)"
1561
+ )
1562
+ # NULL captured response bodies/details for eligible messages (ADR 0013 retention) — to
1563
+ # NULL (matching PG/SQLite: correlate then reads None; reencrypt's IS NOT NULL skips them).
1564
+ await cur.execute(
1565
+ "UPDATE response SET body=NULL, detail=NULL"
1566
+ " WHERE (body IS NOT NULL OR detail IS NOT NULL)"
1567
+ " AND message_id IN (SELECT id FROM #eligible)"
1568
+ )
1569
+ await cur.execute("DROP TABLE #eligible")
1570
+ await conn.commit()
1571
+ except Exception:
1572
+ await conn.rollback()
1573
+ raise
1574
+ return int(purged) if purged is not None else 0
1575
+
1576
+ async def purge_state(self, *, older_than: float, now: float | None = None) -> int:
1577
+ """Delete transform-state rows last set before ``older_than`` (ADR 0005 retention), evicting
1578
+ them from the read-through cache post-commit. Returns the number deleted."""
1579
+ async with self._acquire() as conn:
1580
+ cur = await conn.cursor()
1581
+ try:
1582
+ await cur.execute(
1583
+ "SELECT namespace, [key] FROM state WHERE set_at < ?", (older_than,)
1584
+ )
1585
+ purged_keys = [(r[0], r[1]) for r in await cur.fetchall()]
1586
+ if not purged_keys:
1587
+ await conn.commit()
1588
+ return 0
1589
+ await cur.execute("DELETE FROM state WHERE set_at < ?", (older_than,))
1590
+ await conn.commit()
1591
+ except Exception:
1592
+ await conn.rollback()
1593
+ raise
1594
+ for ck in purged_keys:
1595
+ self._state_cache.pop(ck, None)
1596
+ return len(purged_keys)
1597
+
1598
+ async def purge_dead_letters(self, *, older_than: float, now: float | None = None) -> int:
1599
+ """Blank the payload of dead outbound rows updated before ``older_than`` (retention). Keeps the
1600
+ dead row + 'dead' status (counts/disposition) but frees the body; idempotent (payload <> '')."""
1601
+ async with self._acquire() as conn:
1602
+ cur = await conn.cursor()
1603
+ try:
1604
+ await cur.execute(
1605
+ "UPDATE queue SET payload='', last_error=NULL"
1606
+ " WHERE stage=? AND status=? AND payload <> '' AND updated_at < ?",
1607
+ (Stage.OUTBOUND.value, OutboxStatus.DEAD.value, older_than),
1608
+ )
1609
+ purged = cur.rowcount
1610
+ await conn.commit()
1611
+ except Exception:
1612
+ await conn.rollback()
1613
+ raise
1614
+ return int(purged) if purged is not None else 0
1615
+
1616
+ async def wal_checkpoint(self) -> None:
1617
+ """No-op on SQL Server — there is no SQLite WAL to checkpoint (the engine never calls this on
1618
+ this backend; present for ``Store`` protocol completeness)."""
1619
+
1620
+ async def vacuum(self) -> None:
1621
+ """No-op on SQL Server — file compaction is a DBA operation here, not an engine concern (the
1622
+ engine never calls this on this backend; present for ``Store`` protocol completeness)."""
1623
+
1624
+ async def record_received(
1625
+ self,
1626
+ *,
1627
+ channel_id: str,
1628
+ raw: str,
1629
+ status: MessageStatus,
1630
+ error: str | None = None,
1631
+ control_id: str | None = None,
1632
+ message_type: str | None = None,
1633
+ source_type: str | None = None,
1634
+ summary: str | None = None,
1635
+ metadata: str | None = None,
1636
+ now: float | None = None,
1637
+ ) -> str:
1638
+ error = (
1639
+ safe_text(error) if error else error
1640
+ ) # PHI chokepoint (#120); SQL Server stores plaintext
1641
+ now = time.time() if now is None else now
1642
+ mid = uuid4().hex
1643
+ event = "error" if status is MessageStatus.ERROR else "filtered"
1644
+ async with self._acquire() as conn:
1645
+ cur = await conn.cursor()
1646
+ try:
1647
+ await cur.execute(
1648
+ "INSERT INTO messages (id, channel_id, received_at, source_type, control_id,"
1649
+ " message_type, raw, status, error, summary, metadata)"
1650
+ " VALUES (?,?,?,?,?,?,?,?,?,?,?)",
1651
+ (
1652
+ mid,
1653
+ channel_id,
1654
+ now,
1655
+ source_type,
1656
+ control_id,
1657
+ message_type,
1658
+ self._cipher.encrypt(raw),
1659
+ status.value,
1660
+ error,
1661
+ summary,
1662
+ metadata,
1663
+ ),
1664
+ )
1665
+ await self._event(cur, mid, event, None, error, now)
1666
+ await conn.commit()
1667
+ except Exception:
1668
+ await conn.rollback()
1669
+ raise
1670
+ return mid
1671
+
1672
+ # --- delivery worker path ------------------------------------------------
1673
+
1674
+ @staticmethod
1675
+ def _lane_col(stage: str) -> str:
1676
+ """The FIFO/depth lane column for a stage (code-controlled literal): ``channel_id`` for
1677
+ ingress/routed/response, ``destination_name`` for outbound."""
1678
+ return (
1679
+ "channel_id"
1680
+ if stage in (Stage.INGRESS.value, Stage.ROUTED.value, Stage.RESPONSE.value)
1681
+ else "destination_name"
1682
+ )
1683
+
1684
+ async def claim_ready(
1685
+ self,
1686
+ limit: int = 10,
1687
+ now: float | None = None,
1688
+ *,
1689
+ stage: str = Stage.OUTBOUND.value,
1690
+ channel_id: str | None = None,
1691
+ destination_name: str | None = None,
1692
+ ) -> list[OutboxItem]:
1693
+ """Atomically claim up to ``limit`` due rows at ``stage`` (UNORDERED — skips a backing-off row
1694
+ to drain others), flipping them to ``inflight`` and bumping ``attempts``. ``READPAST, UPDLOCK,
1695
+ ROWLOCK`` is the T-SQL ``FOR UPDATE SKIP LOCKED`` analog so concurrent workers skip rather than
1696
+ block or double-claim. An undecryptable payload is dead-lettered and dropped (poison-row
1697
+ containment), not raised."""
1698
+ now = time.time() if now is None else now
1699
+ where = ["stage=?", "status=?", "next_attempt_at<=?"]
1700
+ filters: list[Any] = [stage, OutboxStatus.PENDING.value, now]
1701
+ if channel_id is not None:
1702
+ where.append("channel_id=?")
1703
+ filters.append(channel_id)
1704
+ if destination_name is not None:
1705
+ where.append("destination_name=?")
1706
+ filters.append(destination_name)
1707
+ sql = (
1708
+ "WITH due AS (SELECT TOP (?) * FROM queue WITH (READPAST, UPDLOCK, ROWLOCK)"
1709
+ f" WHERE {' AND '.join(where)} ORDER BY next_attempt_at)"
1710
+ " UPDATE due SET status=?, attempts=attempts+1, updated_at=?,"
1711
+ " owner=NULL, lease_expires_at=NULL"
1712
+ " OUTPUT inserted.id, inserted.message_id, inserted.channel_id,"
1713
+ " inserted.destination_name, inserted.handler_name, inserted.payload,"
1714
+ " inserted.attempts"
1715
+ )
1716
+ args = (limit, *filters, OutboxStatus.INFLIGHT.value, now)
1717
+ async with self._acquire() as conn:
1718
+ cur = await conn.cursor()
1719
+ try:
1720
+ await cur.execute(sql, args)
1721
+ columns = [c[0] for c in cur.description]
1722
+ rows = await cur.fetchall()
1723
+ await conn.commit()
1724
+ except Exception:
1725
+ await conn.rollback()
1726
+ raise
1727
+ items = []
1728
+ for row in rows:
1729
+ d = dict(zip(columns, row))
1730
+ try:
1731
+ payload = self._cipher.decrypt(d["payload"])
1732
+ except CipherError as exc:
1733
+ log.warning("dead-lettering undecryptable queue row %s: %s", d["id"], exc)
1734
+ await self.dead_letter_now(d["id"], f"undecryptable payload: {exc}")
1735
+ continue
1736
+ items.append(
1737
+ OutboxItem(
1738
+ id=d["id"],
1739
+ message_id=d["message_id"],
1740
+ channel_id=d["channel_id"],
1741
+ destination_name=d["destination_name"],
1742
+ handler_name=d["handler_name"],
1743
+ payload=payload,
1744
+ attempts=d["attempts"],
1745
+ stage=stage,
1746
+ )
1747
+ )
1748
+ return items
1749
+
1750
+ async def claim_next_fifo(
1751
+ self,
1752
+ name: str,
1753
+ now: float | None = None,
1754
+ *,
1755
+ stage: str = Stage.OUTBOUND.value,
1756
+ owner: str | None = None,
1757
+ ) -> OutboxItem | None:
1758
+ """Claim the single oldest *due* pending row for one lane at ``stage`` (strict FIFO — the head
1759
+ blocks the lane while it backs off, via the WHERE on the UPDATE). The lane key is stage-aware
1760
+ (``destination_name`` outbound, ``channel_id`` ingress/routed); ordering is ``created_at, seq``
1761
+ (the IDENTITY tiebreak preserving same-txn insertion order — NOT the random uuid ``id``).
1762
+ ``owner`` (Track B lane ownership) is accepted for protocol uniformity and IGNORED: this backend
1763
+ is single-node (one worker per lane), so owner/lease stay NULL and the runner never passes a
1764
+ non-None owner.
1765
+
1766
+ NB: the head SELECT takes ``(UPDLOCK, ROWLOCK)`` but deliberately **NOT** ``READPAST``. With one
1767
+ serial consumer per lane, the only transaction that can hold a lock on the FIFO head is the
1768
+ *producer* (the ``transform_handoff``/finalizer that just inserted it, milliseconds from commit).
1769
+ ``READPAST`` would SKIP that locked head and claim the next-oldest row instead — delivering seq
1770
+ N+1 before N (a per-lane FIFO break; issue #285). Head-of-line *blocking* on a transiently-locked
1771
+ head is the correct FIFO semantic here — it briefly waits for the rightful head, matching Postgres'
1772
+ ``FOR UPDATE`` (no ``SKIP LOCKED`` skip of a producer-locked head). A pathologically long lock is
1773
+ bounded by ``command_timeout``. The UNORDERED :meth:`claim_ready` keeps ``READPAST`` (there,
1774
+ skipping a locked sibling to drain the lane is intended and order is explicitly not promised)."""
1775
+ now = time.time() if now is None else now
1776
+ lane_col = self._lane_col(stage) # code-controlled literal
1777
+ sql = (
1778
+ "WITH head AS (SELECT TOP (1) * FROM queue WITH (UPDLOCK, ROWLOCK)"
1779
+ f" WHERE stage=? AND {lane_col}=? AND status=? ORDER BY created_at, seq)"
1780
+ " UPDATE head SET status=?, attempts=attempts+1, updated_at=?,"
1781
+ " owner=NULL, lease_expires_at=NULL"
1782
+ " OUTPUT inserted.id, inserted.message_id, inserted.channel_id,"
1783
+ " inserted.destination_name, inserted.handler_name, inserted.payload,"
1784
+ " inserted.attempts"
1785
+ " WHERE next_attempt_at<=?"
1786
+ )
1787
+ args = (stage, name, OutboxStatus.PENDING.value, OutboxStatus.INFLIGHT.value, now, now)
1788
+ async with self._acquire() as conn:
1789
+ cur = await conn.cursor()
1790
+ try:
1791
+ await cur.execute(sql, args)
1792
+ columns = [c[0] for c in cur.description] if cur.description else []
1793
+ row = await cur.fetchone()
1794
+ await conn.commit()
1795
+ except Exception:
1796
+ await conn.rollback()
1797
+ raise
1798
+ if row is None:
1799
+ return None
1800
+ d = dict(zip(columns, row))
1801
+ try:
1802
+ payload = self._cipher.decrypt(d["payload"])
1803
+ except CipherError as exc:
1804
+ log.warning("dead-lettering undecryptable queue row %s: %s", d["id"], exc)
1805
+ await self.dead_letter_now(d["id"], f"undecryptable payload: {exc}")
1806
+ return None
1807
+ return OutboxItem(
1808
+ id=d["id"],
1809
+ message_id=d["message_id"],
1810
+ channel_id=d["channel_id"],
1811
+ destination_name=d["destination_name"],
1812
+ handler_name=d["handler_name"],
1813
+ payload=payload,
1814
+ attempts=d["attempts"],
1815
+ stage=stage,
1816
+ )
1817
+
1818
+ async def mark_done(self, outbox_id: str, now: float | None = None) -> None:
1819
+ now = time.time() if now is None else now
1820
+ async with self._acquire() as conn:
1821
+ cur = await conn.cursor()
1822
+ try:
1823
+ await cur.execute(
1824
+ "SELECT message_id, destination_name, attempts FROM queue WHERE id=?",
1825
+ (outbox_id,),
1826
+ )
1827
+ row = await cur.fetchone()
1828
+ if row is None:
1829
+ await conn.commit()
1830
+ return
1831
+ message_id, destination_name, attempts = row[0], row[1], row[2]
1832
+ await cur.execute(
1833
+ "UPDATE queue SET status=?, last_error=NULL, updated_at=? WHERE id=?",
1834
+ (OutboxStatus.DONE.value, now, outbox_id),
1835
+ )
1836
+ await self._event(
1837
+ cur, message_id, "delivered", destination_name, f"attempt {attempts}", now
1838
+ )
1839
+ await self._maybe_finalize(cur, message_id, now)
1840
+ await conn.commit()
1841
+ except Exception:
1842
+ await conn.rollback()
1843
+ raise
1844
+
1845
+ async def mark_failed(
1846
+ self, outbox_id: str, error: str, retry: RetryPolicy, now: float | None = None
1847
+ ) -> None:
1848
+ error = safe_text(error) # PHI chokepoint (#120); SQL Server stores plaintext
1849
+ now = time.time() if now is None else now
1850
+ async with self._acquire() as conn:
1851
+ cur = await conn.cursor()
1852
+ try:
1853
+ await cur.execute(
1854
+ "SELECT message_id, destination_name, attempts FROM queue WHERE id=?",
1855
+ (outbox_id,),
1856
+ )
1857
+ row = await cur.fetchone()
1858
+ if row is None:
1859
+ await conn.commit()
1860
+ return
1861
+ message_id, destination_name, attempts = row[0], row[1], row[2]
1862
+ # max_attempts None = retry forever (never dead-letter here); a finite cap dead-letters
1863
+ # once exhausted (mirrors the SQLite backend's mark_failed).
1864
+ if retry.max_attempts is not None and attempts >= retry.max_attempts:
1865
+ status, next_at, event = OutboxStatus.DEAD.value, now, "dead"
1866
+ else:
1867
+ backoff = min(
1868
+ retry.max_backoff_seconds,
1869
+ retry.backoff_seconds * (retry.backoff_multiplier ** (attempts - 1)),
1870
+ )
1871
+ status, next_at, event = OutboxStatus.PENDING.value, now + backoff, "failed"
1872
+ await cur.execute(
1873
+ "UPDATE queue SET status=?, next_attempt_at=?, last_error=?, updated_at=? WHERE id=?",
1874
+ (status, next_at, error, now, outbox_id),
1875
+ )
1876
+ await self._event(
1877
+ cur, message_id, event, destination_name, f"attempt {attempts}: {error}", now
1878
+ )
1879
+ if status == OutboxStatus.DEAD.value:
1880
+ await self._maybe_finalize(cur, message_id, now)
1881
+ await conn.commit()
1882
+ except Exception:
1883
+ await conn.rollback()
1884
+ raise
1885
+
1886
+ # --- recovery / replay ---------------------------------------------------
1887
+
1888
+ async def reset_stale_inflight(
1889
+ self, now: float | None = None, *, stage: str | None = None
1890
+ ) -> int:
1891
+ """Return in-flight rows to ``pending`` (startup crash recovery) across ALL stages by default —
1892
+ an ingress/routed row left inflight by a crash MUST be re-pended or the message hangs forever
1893
+ (count-and-log invariant). ``stage`` optionally narrows it; owner/lease are cleared (single-node
1894
+ parity)."""
1895
+ now = time.time() if now is None else now
1896
+ clauses = ["status=?"]
1897
+ params: list[Any] = [OutboxStatus.INFLIGHT.value]
1898
+ if stage is not None:
1899
+ clauses.append("stage=?")
1900
+ params.append(stage)
1901
+ sql = (
1902
+ "UPDATE queue SET status=?, next_attempt_at=?, updated_at=?, owner=NULL,"
1903
+ f" lease_expires_at=NULL WHERE {' AND '.join(clauses)}"
1904
+ )
1905
+ async with self._acquire() as conn:
1906
+ cur = await conn.cursor()
1907
+ try:
1908
+ await cur.execute(sql, (OutboxStatus.PENDING.value, now, now, *params))
1909
+ count = cur.rowcount
1910
+ await conn.commit()
1911
+ except Exception:
1912
+ await conn.rollback()
1913
+ raise
1914
+ return int(count)
1915
+
1916
+ async def dead_letter_now(self, outbox_id: str, error: str, now: float | None = None) -> None:
1917
+ """Force one row terminal (``DEAD``) immediately — fail-fast, no retry consumed. See the
1918
+ :meth:`~messagefoundry.store.base.QueueStore.dead_letter_now` contract."""
1919
+ error = safe_text(
1920
+ error
1921
+ ) # PHI chokepoint (#120) — incl. f"undecryptable payload: {exc}" callers
1922
+ now = time.time() if now is None else now
1923
+ async with self._acquire() as conn:
1924
+ cur = await conn.cursor()
1925
+ try:
1926
+ await cur.execute(
1927
+ "SELECT message_id, destination_name FROM queue WHERE id=?", (outbox_id,)
1928
+ )
1929
+ row = await cur.fetchone()
1930
+ if row is None:
1931
+ await conn.commit()
1932
+ return
1933
+ message_id, destination_name = row[0], row[1]
1934
+ await cur.execute(
1935
+ "UPDATE queue SET status=?, next_attempt_at=?, last_error=?, updated_at=?,"
1936
+ " owner=NULL, lease_expires_at=NULL WHERE id=?",
1937
+ (OutboxStatus.DEAD.value, now, error, now, outbox_id),
1938
+ )
1939
+ await self._event(cur, message_id, "dead", destination_name, error, now)
1940
+ await self._maybe_finalize(cur, message_id, now)
1941
+ await conn.commit()
1942
+ except Exception:
1943
+ await conn.rollback()
1944
+ raise
1945
+
1946
+ async def pending_depth(
1947
+ self, name: str, *, stage: str = Stage.OUTBOUND.value
1948
+ ) -> tuple[int, float | None]:
1949
+ """``(pending_count, oldest_created_at)`` for one lane at ``stage`` (see the protocol). The lane
1950
+ key is stage-aware (``destination_name`` outbound, ``channel_id`` ingress/routed)."""
1951
+ lane_col = self._lane_col(stage) # code-controlled literal
1952
+ # Route through _fetchone (which commits) so we never return the pooled connection mid read-txn
1953
+ # under RCSI (M-6 read hygiene; mirrors postgres.py).
1954
+ row = await self._fetchone(
1955
+ f"SELECT COUNT(*) AS c, MIN(created_at) AS m FROM queue"
1956
+ f" WHERE stage=? AND {lane_col}=? AND status=?",
1957
+ (stage, name, OutboxStatus.PENDING.value),
1958
+ )
1959
+ count = int(row["c"]) if row is not None and row["c"] is not None else 0
1960
+ oldest = row["m"] if row is not None else None
1961
+ return count, (float(oldest) if oldest is not None else None)
1962
+
1963
+ async def dead_letter_missing_destinations(
1964
+ self, valid_names: set[str], now: float | None = None
1965
+ ) -> int:
1966
+ """Dead-letter non-terminal outbound queue rows whose destination_name is no longer in the
1967
+ registry (a removed/renamed outbound) — they have no delivery worker and would strand forever
1968
+ (H-5). The per-message finalize applocks are pre-acquired in sorted id order so two concurrent
1969
+ multi-message finalizers can't deadlock."""
1970
+ now = time.time() if now is None else now
1971
+ async with self._acquire() as conn:
1972
+ cur = await conn.cursor()
1973
+ try:
1974
+ await cur.execute(
1975
+ "SELECT id, message_id, destination_name FROM queue"
1976
+ " WHERE stage=? AND status IN (?, ?)",
1977
+ (Stage.OUTBOUND.value, OutboxStatus.PENDING.value, OutboxStatus.INFLIGHT.value),
1978
+ )
1979
+ rows = await cur.fetchall() # positional: (id, message_id, destination_name)
1980
+ orphans = [r for r in rows if r[2] not in valid_names]
1981
+ if not orphans:
1982
+ await conn.commit() # release the read txn cleanly (M-6)
1983
+ return 0
1984
+ error = "destination removed from outbound registry"
1985
+ await self._lock_finalize_batch(cur, {r[1] for r in orphans})
1986
+ for row in orphans:
1987
+ await cur.execute(
1988
+ "UPDATE queue SET status=?, next_attempt_at=?, last_error=?, updated_at=?,"
1989
+ " owner=NULL, lease_expires_at=NULL WHERE id=?",
1990
+ (OutboxStatus.DEAD.value, now, error, now, row[0]),
1991
+ )
1992
+ await self._event(cur, row[1], "dead", row[2], error, now)
1993
+ await self._maybe_finalize(cur, row[1], now)
1994
+ await conn.commit()
1995
+ except Exception:
1996
+ await conn.rollback()
1997
+ raise
1998
+ log.warning(
1999
+ "dead-lettered %d orphaned outbound row(s) at startup for missing destination(s): %s",
2000
+ len(orphans),
2001
+ ", ".join(sorted({r[2] for r in orphans})),
2002
+ )
2003
+ return len(orphans)
2004
+
2005
+ async def replay(self, message_id: str, now: float | None = None) -> int:
2006
+ """Re-queue a message's stuck/dead deliveries — or, if none are stuck, re-send the delivered
2007
+ ones. Two-mode (M-2): if any row is dead/pending, replay ONLY those (never re-fire a DONE
2008
+ sibling); else replay the done rows. messages.status -> RECEIVED if a pending ingress/routed
2009
+ row remains (needs re-routing), else ROUTED."""
2010
+ now = time.time() if now is None else now
2011
+ async with self._acquire() as conn:
2012
+ cur = await conn.cursor()
2013
+ try:
2014
+ await cur.execute(
2015
+ "SELECT COUNT(*) FROM queue WHERE message_id=? AND status IN (?, ?)",
2016
+ (message_id, OutboxStatus.DEAD.value, OutboxStatus.PENDING.value),
2017
+ )
2018
+ row = await cur.fetchone()
2019
+ stuck = int(row[0]) if row and row[0] is not None else 0
2020
+ replay_from = (
2021
+ (OutboxStatus.DEAD.value, OutboxStatus.PENDING.value)
2022
+ if stuck
2023
+ else (OutboxStatus.DONE.value,)
2024
+ )
2025
+ placeholders = ",".join("?" * len(replay_from))
2026
+ await cur.execute(
2027
+ f"UPDATE queue SET status=?, attempts=0, next_attempt_at=?, last_error=NULL,"
2028
+ f" updated_at=? WHERE message_id=? AND status IN ({placeholders})",
2029
+ (OutboxStatus.PENDING.value, now, now, message_id, *replay_from),
2030
+ )
2031
+ count = cur.rowcount
2032
+ if (
2033
+ count
2034
+ ): # no rows => errored/filtered/unrouted: don't falsify it or strand it (M-2)
2035
+ await cur.execute(
2036
+ "SELECT 1 FROM queue WHERE message_id=? AND stage IN (?, ?) AND status=?",
2037
+ (
2038
+ message_id,
2039
+ Stage.INGRESS.value,
2040
+ Stage.ROUTED.value,
2041
+ OutboxStatus.PENDING.value,
2042
+ ),
2043
+ )
2044
+ new_status = (
2045
+ MessageStatus.RECEIVED.value
2046
+ if await cur.fetchone()
2047
+ else MessageStatus.ROUTED.value
2048
+ )
2049
+ await cur.execute(
2050
+ "UPDATE messages SET status=?, error=NULL WHERE id=?",
2051
+ (new_status, message_id),
2052
+ )
2053
+ await self._event(
2054
+ cur, message_id, "replayed", None, f"{count} destination(s)", now
2055
+ )
2056
+ await conn.commit()
2057
+ except Exception:
2058
+ await conn.rollback()
2059
+ raise
2060
+ return int(count)
2061
+
2062
+ async def replay_dead(
2063
+ self,
2064
+ *,
2065
+ channel_id: str | None = None,
2066
+ destination_name: str | None = None,
2067
+ now: float | None = None,
2068
+ ) -> int:
2069
+ now = time.time() if now is None else now
2070
+ where = ["stage=?", "status=?"]
2071
+ params: list[Any] = [Stage.OUTBOUND.value, OutboxStatus.DEAD.value]
2072
+ if channel_id is not None:
2073
+ where.append("channel_id=?")
2074
+ params.append(channel_id)
2075
+ if destination_name is not None:
2076
+ where.append("destination_name=?")
2077
+ params.append(destination_name)
2078
+ clause = " AND ".join(where)
2079
+ async with self._acquire() as conn:
2080
+ cur = await conn.cursor()
2081
+ try:
2082
+ await cur.execute(
2083
+ f"SELECT DISTINCT message_id FROM queue WHERE {clause}", tuple(params)
2084
+ )
2085
+ message_ids = [r[0] for r in await cur.fetchall()]
2086
+ if not message_ids:
2087
+ await conn.commit()
2088
+ return 0
2089
+ await cur.execute(
2090
+ f"UPDATE queue SET status=?, attempts=0, next_attempt_at=?, last_error=NULL,"
2091
+ f" updated_at=? WHERE {clause}",
2092
+ (OutboxStatus.PENDING.value, now, now, *params),
2093
+ )
2094
+ count = cur.rowcount
2095
+ for message_id in message_ids:
2096
+ await cur.execute(
2097
+ "UPDATE messages SET status=?, error=NULL WHERE id=? AND status=?",
2098
+ (MessageStatus.ROUTED.value, message_id, MessageStatus.ERROR.value),
2099
+ )
2100
+ await self._event(cur, message_id, "replayed", None, "dead-letter replay", now)
2101
+ await conn.commit()
2102
+ except Exception:
2103
+ await conn.rollback()
2104
+ raise
2105
+ return int(count)
2106
+
2107
+ async def cancel_queued(
2108
+ self,
2109
+ channel_id: str | None,
2110
+ destination_name: str,
2111
+ *,
2112
+ top_only: bool = False,
2113
+ now: float | None = None,
2114
+ ) -> int:
2115
+ now = time.time() if now is None else now
2116
+ where = ["stage=?", "destination_name=?", "status=?"]
2117
+ params: list[Any] = [Stage.OUTBOUND.value, destination_name, OutboxStatus.PENDING.value]
2118
+ if channel_id is not None:
2119
+ where.insert(1, "channel_id=?")
2120
+ params.insert(1, channel_id)
2121
+ top = "TOP (1) " if top_only else ""
2122
+ async with self._acquire() as conn:
2123
+ cur = await conn.cursor()
2124
+ try:
2125
+ await cur.execute(
2126
+ f"SELECT {top}id, message_id FROM queue WHERE {' AND '.join(where)}"
2127
+ " ORDER BY next_attempt_at, created_at",
2128
+ tuple(params),
2129
+ )
2130
+ rows = [(r[0], r[1]) for r in await cur.fetchall()]
2131
+ if not rows:
2132
+ await conn.commit()
2133
+ return 0
2134
+ ids = [r[0] for r in rows]
2135
+ placeholders = ",".join("?" * len(ids))
2136
+ await cur.execute(
2137
+ f"UPDATE queue SET status=?, updated_at=? WHERE id IN ({placeholders})",
2138
+ (OutboxStatus.CANCELLED.value, now, *ids),
2139
+ )
2140
+ for _id, message_id in rows:
2141
+ await self._event(
2142
+ cur, message_id, "cancelled", destination_name, "manual purge", now
2143
+ )
2144
+ mids = {r[1] for r in rows}
2145
+ await self._lock_finalize_batch(cur, mids)
2146
+ for message_id in sorted(mids):
2147
+ await self._maybe_finalize(cur, message_id, now)
2148
+ await conn.commit()
2149
+ except Exception:
2150
+ await conn.rollback()
2151
+ raise
2152
+ return len(ids)
2153
+
2154
+ # --- read helpers --------------------------------------------------------
2155
+
2156
+ async def get_message(self, message_id: str) -> dict[str, Any] | None:
2157
+ record = await self._fetchone("SELECT * FROM messages WHERE id=?", (message_id,))
2158
+ if record is not None:
2159
+ record["raw"] = self._cipher.decrypt(record["raw"]) # decrypt the body for display
2160
+ return record
2161
+
2162
+ async def list_messages(
2163
+ self,
2164
+ *,
2165
+ channel_id: str | None = None,
2166
+ status: str | None = None,
2167
+ message_type: str | None = None,
2168
+ control_id: str | None = None,
2169
+ limit: int = 50,
2170
+ offset: int = 0,
2171
+ allowed_channels: Sequence[str] | None = None,
2172
+ ) -> list[dict[str, Any]]:
2173
+ where, params = self._message_filter(
2174
+ channel_id, status, message_type, control_id, allowed_channels
2175
+ )
2176
+ return await self._fetchall(
2177
+ "SELECT id, channel_id, received_at, source_type, control_id, message_type,"
2178
+ " status, error, summary, metadata,"
2179
+ " (SELECT TOP 1 event FROM message_events e WHERE e.message_id = messages.id"
2180
+ " ORDER BY e.id DESC) AS last_event"
2181
+ f" FROM messages{where}"
2182
+ " ORDER BY received_at DESC, id DESC OFFSET ? ROWS FETCH NEXT ? ROWS ONLY",
2183
+ (*params, offset, limit),
2184
+ )
2185
+
2186
+ async def count_messages(
2187
+ self,
2188
+ *,
2189
+ channel_id: str | None = None,
2190
+ status: str | None = None,
2191
+ message_type: str | None = None,
2192
+ control_id: str | None = None,
2193
+ allowed_channels: Sequence[str] | None = None,
2194
+ ) -> int:
2195
+ where, params = self._message_filter(
2196
+ channel_id, status, message_type, control_id, allowed_channels
2197
+ )
2198
+ row = await self._fetchone(f"SELECT COUNT(*) AS n FROM messages{where}", params)
2199
+ return int(row["n"]) if row else 0
2200
+
2201
+ async def list_dead(
2202
+ self,
2203
+ *,
2204
+ channel_id: str | None = None,
2205
+ destination_name: str | None = None,
2206
+ limit: int = 50,
2207
+ offset: int = 0,
2208
+ allowed_channels: Sequence[str] | None = None,
2209
+ ) -> list[dict[str, Any]]:
2210
+ where, params = self._dead_filter(channel_id, destination_name, allowed_channels)
2211
+ return await self._fetchall(
2212
+ "SELECT o.id AS outbox_id, o.message_id, o.channel_id, o.destination_name,"
2213
+ " o.attempts, o.last_error, o.updated_at,"
2214
+ " m.control_id, m.message_type, m.received_at, m.summary"
2215
+ f" FROM queue o JOIN messages m ON m.id = o.message_id{where}"
2216
+ " ORDER BY o.updated_at DESC, o.id DESC OFFSET ? ROWS FETCH NEXT ? ROWS ONLY",
2217
+ (*params, offset, limit),
2218
+ )
2219
+
2220
+ async def count_dead(
2221
+ self,
2222
+ *,
2223
+ channel_id: str | None = None,
2224
+ destination_name: str | None = None,
2225
+ allowed_channels: Sequence[str] | None = None,
2226
+ ) -> int:
2227
+ where, params = self._dead_filter(channel_id, destination_name, allowed_channels)
2228
+ row = await self._fetchone(f"SELECT COUNT(*) AS n FROM queue o{where}", params)
2229
+ return int(row["n"]) if row else 0
2230
+
2231
+ @staticmethod
2232
+ def _dead_filter(
2233
+ channel_id: str | None,
2234
+ destination_name: str | None,
2235
+ allowed_channels: Sequence[str] | None = None,
2236
+ ) -> tuple[str, tuple[Any, ...]]:
2237
+ clauses = ["o.stage=?", "o.status=?"]
2238
+ params: list[Any] = [Stage.OUTBOUND.value, OutboxStatus.DEAD.value]
2239
+ if channel_id is not None:
2240
+ clauses.append("o.channel_id=?")
2241
+ params.append(channel_id)
2242
+ if destination_name is not None:
2243
+ clauses.append("o.destination_name=?")
2244
+ params.append(destination_name)
2245
+ _append_channel_scope(clauses, params, "o.channel_id", allowed_channels)
2246
+ return f" WHERE {' AND '.join(clauses)}", tuple(params)
2247
+
2248
+ async def outbox_for(self, message_id: str) -> list[dict[str, Any]]:
2249
+ return await self._fetchall(
2250
+ "SELECT * FROM queue WHERE message_id=? AND stage=? ORDER BY destination_name",
2251
+ (message_id, Stage.OUTBOUND.value),
2252
+ )
2253
+
2254
+ async def outbox_payloads_for(self, message_id: str) -> list[dict[str, Any]]:
2255
+ """Like :meth:`outbox_for`, but also decrypts the transformed ``payload`` (PHI body) per
2256
+ destination for the parity-comparison read path (#14). The outbound ``payload`` column is the
2257
+ encrypted body directly (no artifact indirection at this stage — cf. :meth:`claim_ready`); the
2258
+ API gates this on ``MESSAGES_VIEW_RAW`` and audits it."""
2259
+ rows = await self._fetchall(
2260
+ "SELECT * FROM queue WHERE message_id=? AND stage=? ORDER BY destination_name",
2261
+ (message_id, Stage.OUTBOUND.value),
2262
+ )
2263
+ for r in rows:
2264
+ r["payload"] = self._cipher.decrypt(r["payload"])
2265
+ if r["last_error"] is not None:
2266
+ r["last_error"] = self._cipher.decrypt(r["last_error"])
2267
+ return rows
2268
+
2269
+ async def events_for(self, message_id: str) -> list[dict[str, Any]]:
2270
+ return await self._fetchall(
2271
+ "SELECT * FROM message_events WHERE message_id=? ORDER BY id", (message_id,)
2272
+ )
2273
+
2274
+ async def record_view(
2275
+ self, message_id: str, *, actor: str | None = None, now: float | None = None
2276
+ ) -> None:
2277
+ now = time.time() if now is None else now
2278
+ async with self._acquire() as conn:
2279
+ cur = await conn.cursor()
2280
+ try:
2281
+ await self._event(cur, message_id, "viewed", None, actor or "", now)
2282
+ await conn.commit()
2283
+ except Exception:
2284
+ await conn.rollback()
2285
+ raise
2286
+
2287
+ async def record_audit(
2288
+ self,
2289
+ action: str,
2290
+ *,
2291
+ actor: str | None = None,
2292
+ channel_id: str | None = None,
2293
+ detail: str | None = None,
2294
+ now: float | None = None,
2295
+ ) -> None:
2296
+ now = time.time() if now is None else now
2297
+ # Serialize the read-prev-then-insert append in-process so two concurrent audited actions can't
2298
+ # read the same prev hash and FORK the hash chain (H-7). The store is the single audit writer
2299
+ # per engine process (active-passive = one active node), so an in-process lock is sufficient and
2300
+ # reliable — unlike a txn-scoped sp_getapplock taken as the connection's first statement, which
2301
+ # does not release on commit and strands under concurrent contention.
2302
+ async with self._audit_lock:
2303
+ async with self._acquire() as conn:
2304
+ cur = await conn.cursor()
2305
+ try:
2306
+ await cur.execute("SELECT TOP (1) row_hash FROM audit_log ORDER BY id DESC")
2307
+ last = await cur.fetchone()
2308
+ prev = last[0] if last and last[0] else ""
2309
+ row_hash = audit_row_hash(
2310
+ prev,
2311
+ ts=now,
2312
+ actor=actor,
2313
+ action=action,
2314
+ channel_id=channel_id,
2315
+ detail=detail,
2316
+ )
2317
+ await cur.execute(
2318
+ "INSERT INTO audit_log (ts, actor, action, channel_id, detail, row_hash)"
2319
+ " VALUES (?,?,?,?,?,?)",
2320
+ (now, actor, action, channel_id, detail, row_hash),
2321
+ )
2322
+ await conn.commit()
2323
+ except Exception:
2324
+ await conn.rollback()
2325
+ raise
2326
+ # Tee off-box AFTER commit + outside the audit lock / pooled connection (only forward what
2327
+ # truly persisted; a synchronous syslog send must never hold the lock). Shared redaction path.
2328
+ emit_audit_tee(action=action, actor=actor, channel_id=channel_id, detail=detail, ts=now)
2329
+
2330
+ async def audit_anchor(self) -> tuple[int, str]:
2331
+ """The audit log's external anchor — ``(row_count, head_hash)`` — see the SQLite store (low-1)."""
2332
+ rows = await self._fetchall(
2333
+ "SELECT COUNT(*) AS n, "
2334
+ "(SELECT TOP (1) row_hash FROM audit_log ORDER BY id DESC) AS head FROM audit_log"
2335
+ )
2336
+ if not rows:
2337
+ return 0, ""
2338
+ return int(rows[0]["n"]), (rows[0]["head"] or "")
2339
+
2340
+ async def verify_audit_chain(
2341
+ self, *, expected_anchor: tuple[int, str] | None = None
2342
+ ) -> tuple[bool, str | None]:
2343
+ """Recompute the audit hash-chain in order; returns (ok, message) — see the SQLite store.
2344
+
2345
+ Re-walking can't catch tail-truncation (the surviving prefix still verifies); pass
2346
+ ``expected_anchor`` from :meth:`audit_anchor`, held out-of-band, to detect it (review low-1)."""
2347
+ rows = await self._fetchall(
2348
+ "SELECT id, ts, actor, action, channel_id, detail, row_hash FROM audit_log ORDER BY id"
2349
+ )
2350
+ prev = ""
2351
+ for r in rows:
2352
+ expected = audit_row_hash(
2353
+ prev,
2354
+ ts=r["ts"],
2355
+ actor=r["actor"],
2356
+ action=r["action"],
2357
+ channel_id=r["channel_id"],
2358
+ detail=r["detail"],
2359
+ )
2360
+ if r["row_hash"] != expected:
2361
+ return False, f"audit chain broken at row id={r['id']}"
2362
+ prev = r["row_hash"]
2363
+ if expected_anchor is not None:
2364
+ exp_count, exp_head = expected_anchor
2365
+ if len(rows) < exp_count or prev != exp_head:
2366
+ return (
2367
+ False,
2368
+ f"audit log diverges from recorded anchor (have {len(rows)} row(s) head "
2369
+ f"{prev[:12]!r}, expected {exp_count} head {exp_head[:12]!r}) — truncated or rewritten",
2370
+ )
2371
+ return True, f"verified {len(rows)} audit row(s)"
2372
+
2373
+ # --- auth: users / roles / sessions --------------------------------------
2374
+
2375
+ async def list_audit(self, *, limit: int = 50) -> list[dict[str, Any]]:
2376
+ return await self._fetchall("SELECT TOP (?) * FROM audit_log ORDER BY id DESC", (limit,))
2377
+
2378
+ async def security_events_for_user(
2379
+ self, username: str, *, limit: int = 100
2380
+ ) -> list[dict[str, Any]]:
2381
+ """A user's own security events (``auth.*``), most-recent-first — for ``GET
2382
+ /me/security-events`` (ASVS 6.3.5/6.3.7); admin-initiated changes go out-of-band by email."""
2383
+ return await self._fetchall(
2384
+ "SELECT TOP (?) ts, action, detail FROM audit_log "
2385
+ "WHERE actor = ? AND action LIKE 'auth.%' ORDER BY id DESC",
2386
+ (limit, username),
2387
+ )
2388
+
2389
+ # --- dual-control approvals (ASVS 2.3.5) ---------------------------------
2390
+
2391
+ async def create_pending_approval(
2392
+ self,
2393
+ *,
2394
+ approval_id: str,
2395
+ operation: str,
2396
+ params: str,
2397
+ requester: str,
2398
+ requested_at: float,
2399
+ expires_at: float | None,
2400
+ ) -> None:
2401
+ """Persist a high-value action awaiting a distinct second approver (dual-control, 2.3.5)."""
2402
+ await self._execute(
2403
+ "INSERT INTO pending_approvals "
2404
+ "(id, operation, params, requester, requested_at, status, expires_at) "
2405
+ "VALUES (?,?,?,?,?,'pending',?)",
2406
+ (approval_id, operation, params, requester, requested_at, expires_at),
2407
+ )
2408
+
2409
+ async def get_pending_approval(self, approval_id: str) -> dict[str, Any] | None:
2410
+ return await self._fetchone(
2411
+ "SELECT id, operation, params, requester, requested_at, status, approver, decided_at,"
2412
+ " expires_at FROM pending_approvals WHERE id = ?",
2413
+ (approval_id,),
2414
+ )
2415
+
2416
+ async def list_pending_approvals(self, *, now: float, limit: int = 100) -> list[dict[str, Any]]:
2417
+ """Open (still-``pending``, unexpired) approval requests, newest-first."""
2418
+ return await self._fetchall(
2419
+ "SELECT TOP (?) id, operation, params, requester, requested_at, status, approver,"
2420
+ " decided_at, expires_at FROM pending_approvals"
2421
+ " WHERE status = 'pending' AND (expires_at IS NULL OR expires_at > ?)"
2422
+ " ORDER BY requested_at DESC",
2423
+ (limit, now),
2424
+ )
2425
+
2426
+ async def decide_pending_approval(
2427
+ self, approval_id: str, *, status: str, approver: str | None, decided_at: float
2428
+ ) -> bool:
2429
+ """Atomically move a still-``pending`` request to ``status`` (approved/rejected/expired).
2430
+ Returns ``True`` iff this call made the transition — guards against a double decision."""
2431
+ async with self._acquire() as conn:
2432
+ cur = await conn.cursor()
2433
+ try:
2434
+ await cur.execute(
2435
+ "UPDATE pending_approvals SET status = ?, approver = ?, decided_at = ?"
2436
+ " WHERE id = ? AND status = 'pending'",
2437
+ (status, approver, decided_at, approval_id),
2438
+ )
2439
+ count = cur.rowcount
2440
+ await conn.commit()
2441
+ except Exception:
2442
+ await conn.rollback()
2443
+ raise
2444
+ return int(count) > 0
2445
+
2446
+ async def create_user(
2447
+ self,
2448
+ *,
2449
+ user_id: str,
2450
+ username: str,
2451
+ auth_provider: str,
2452
+ display_name: str | None = None,
2453
+ email: str | None = None,
2454
+ password_hash: str | None = None,
2455
+ must_change_password: bool = False,
2456
+ now: float | None = None,
2457
+ ) -> None:
2458
+ now = time.time() if now is None else now
2459
+ await self._execute(
2460
+ "INSERT INTO users (id, username, auth_provider, display_name, email, disabled,"
2461
+ " created_at, updated_at, last_login_at, password_hash, password_changed_at,"
2462
+ " must_change_password, failed_attempts, locked_until)"
2463
+ " VALUES (?,?,?,?,?,0,?,?,NULL,?,?,?,0,NULL)",
2464
+ (
2465
+ user_id,
2466
+ username,
2467
+ auth_provider,
2468
+ display_name,
2469
+ email,
2470
+ now,
2471
+ now,
2472
+ password_hash,
2473
+ now if password_hash is not None else None,
2474
+ 1 if must_change_password else 0,
2475
+ ),
2476
+ )
2477
+
2478
+ async def get_user(self, user_id: str) -> UserRecord | None:
2479
+ d = await self._fetchone("SELECT * FROM users WHERE id=?", (user_id,))
2480
+ return UserRecord.from_mapping(d) if d else None
2481
+
2482
+ async def get_user_by_username(self, username: str) -> UserRecord | None:
2483
+ d = await self._fetchone("SELECT * FROM users WHERE username=?", (username,))
2484
+ return UserRecord.from_mapping(d) if d else None
2485
+
2486
+ async def list_users(self) -> list[UserRecord]:
2487
+ rows = await self._fetchall("SELECT * FROM users ORDER BY username")
2488
+ return [UserRecord.from_mapping(d) for d in rows]
2489
+
2490
+ async def count_users(self) -> int:
2491
+ return await self._count("users")
2492
+
2493
+ async def set_password(
2494
+ self,
2495
+ user_id: str,
2496
+ *,
2497
+ password_hash: str,
2498
+ must_change_password: bool = False,
2499
+ now: float | None = None,
2500
+ ) -> None:
2501
+ now = time.time() if now is None else now
2502
+ await self._execute(
2503
+ "UPDATE users SET password_hash=?, password_changed_at=?, must_change_password=?,"
2504
+ " failed_attempts=0, locked_until=NULL, updated_at=? WHERE id=?",
2505
+ (password_hash, now, 1 if must_change_password else 0, now, user_id),
2506
+ )
2507
+
2508
+ # --- MFA: native TOTP second factor (local accounts, WP-14) --------------
2509
+
2510
+ async def set_totp_secret(
2511
+ self, user_id: str, *, secret: str | None, now: float | None = None
2512
+ ) -> None:
2513
+ """Stage (or clear) a user's base32 TOTP secret, store-cipher encrypted. Does not enable MFA."""
2514
+ now = time.time() if now is None else now
2515
+ enc = self._cipher.encrypt(secret) if secret else None
2516
+ await self._execute(
2517
+ "UPDATE users SET totp_secret=?, updated_at=? WHERE id=?", (enc, now, user_id)
2518
+ )
2519
+
2520
+ async def get_totp_secret(self, user_id: str) -> str | None:
2521
+ d = await self._fetchone("SELECT totp_secret FROM users WHERE id=?", (user_id,))
2522
+ if not d or d["totp_secret"] is None:
2523
+ return None
2524
+ return self._cipher.decrypt(d["totp_secret"])
2525
+
2526
+ async def enable_totp(
2527
+ self, user_id: str, *, recovery_code_hashes: list[str], now: float | None = None
2528
+ ) -> None:
2529
+ now = time.time() if now is None else now
2530
+ await self._execute(
2531
+ "UPDATE users SET totp_enabled=1, totp_enrolled_at=?, totp_recovery_codes=?,"
2532
+ " updated_at=? WHERE id=?",
2533
+ (now, json.dumps(recovery_code_hashes), now, user_id),
2534
+ )
2535
+
2536
+ async def disable_totp(self, user_id: str, *, now: float | None = None) -> None:
2537
+ now = time.time() if now is None else now
2538
+ await self._execute(
2539
+ "UPDATE users SET totp_secret=NULL, totp_enabled=0, totp_enrolled_at=NULL,"
2540
+ " totp_recovery_codes=NULL, updated_at=? WHERE id=?",
2541
+ (now, user_id),
2542
+ )
2543
+
2544
+ async def get_recovery_code_hashes(self, user_id: str) -> list[str]:
2545
+ d = await self._fetchone("SELECT totp_recovery_codes FROM users WHERE id=?", (user_id,))
2546
+ if not d or d["totp_recovery_codes"] is None:
2547
+ return []
2548
+ return [str(h) for h in json.loads(d["totp_recovery_codes"])]
2549
+
2550
+ async def consume_recovery_code_hash(
2551
+ self, user_id: str, code_hash: str, *, now: float | None = None
2552
+ ) -> bool:
2553
+ """Atomically remove one recovery-code hash; ``True`` iff present. The ``UPDLOCK`` SELECT +
2554
+ UPDATE run in one transaction, so concurrent verifications can't double-spend a single-use
2555
+ recovery code (WP-14)."""
2556
+ now = time.time() if now is None else now
2557
+ async with self._acquire() as conn:
2558
+ cur = await conn.cursor()
2559
+ try:
2560
+ await cur.execute(
2561
+ "SELECT totp_recovery_codes FROM users WITH (UPDLOCK, ROWLOCK) WHERE id=?",
2562
+ (user_id,),
2563
+ )
2564
+ row = await cur.fetchone()
2565
+ raw = row[0] if row else None
2566
+ if raw is None:
2567
+ await conn.commit()
2568
+ return False
2569
+ hashes = [str(h) for h in json.loads(raw)]
2570
+ if code_hash not in hashes:
2571
+ await conn.commit()
2572
+ return False # already consumed by a concurrent caller
2573
+ hashes.remove(code_hash)
2574
+ await cur.execute(
2575
+ "UPDATE users SET totp_recovery_codes=?, updated_at=? WHERE id=?",
2576
+ (json.dumps(hashes), now, user_id),
2577
+ )
2578
+ await conn.commit()
2579
+ return True
2580
+ except Exception:
2581
+ await conn.rollback()
2582
+ raise
2583
+
2584
+ async def consume_totp_step(self, user_id: str, step: int) -> bool:
2585
+ """Atomically record ``step`` as the user's highest consumed TOTP time-step; ``True`` iff newly
2586
+ consumed (strictly greater than any prior step). A code replayed inside its ±1-step verify
2587
+ window resolves to a non-greater step and returns ``False`` — single-use per ASVS 6.5.1. The
2588
+ ``UPDLOCK`` SELECT + UPDATE run in one transaction so concurrent verifications can't both win."""
2589
+ async with self._acquire() as conn:
2590
+ cur = await conn.cursor()
2591
+ try:
2592
+ await cur.execute(
2593
+ "SELECT last_totp_step FROM users WITH (UPDLOCK, ROWLOCK) WHERE id=?",
2594
+ (user_id,),
2595
+ )
2596
+ row = await cur.fetchone()
2597
+ if row is None:
2598
+ await conn.commit()
2599
+ return False
2600
+ last = row[0]
2601
+ if last is not None and last >= step:
2602
+ await conn.commit()
2603
+ return False # already consumed (or an older step) — replay within the window
2604
+ await cur.execute("UPDATE users SET last_totp_step=? WHERE id=?", (step, user_id))
2605
+ await conn.commit()
2606
+ return True
2607
+ except Exception:
2608
+ await conn.rollback()
2609
+ raise
2610
+
2611
+ async def set_user_disabled(
2612
+ self, user_id: str, *, disabled: bool, now: float | None = None
2613
+ ) -> None:
2614
+ now = time.time() if now is None else now
2615
+ await self._execute(
2616
+ "UPDATE users SET disabled=?, updated_at=? WHERE id=?",
2617
+ (1 if disabled else 0, now, user_id),
2618
+ )
2619
+
2620
+ async def update_user_profile(
2621
+ self,
2622
+ user_id: str,
2623
+ *,
2624
+ display_name: str | None,
2625
+ email: str | None,
2626
+ now: float | None = None,
2627
+ ) -> None:
2628
+ now = time.time() if now is None else now
2629
+ await self._execute(
2630
+ "UPDATE users SET display_name=?, email=?, updated_at=? WHERE id=?",
2631
+ (display_name, email, now, user_id),
2632
+ )
2633
+
2634
+ async def delete_user(self, user_id: str) -> None:
2635
+ async with self._acquire() as conn:
2636
+ cur = await conn.cursor()
2637
+ try:
2638
+ await cur.execute("DELETE FROM user_roles WHERE user_id=?", (user_id,))
2639
+ await cur.execute("DELETE FROM sessions WHERE user_id=?", (user_id,))
2640
+ await cur.execute("DELETE FROM users WHERE id=?", (user_id,))
2641
+ await conn.commit()
2642
+ except Exception:
2643
+ await conn.rollback()
2644
+ raise
2645
+
2646
+ async def record_login_success(self, user_id: str, *, now: float | None = None) -> None:
2647
+ now = time.time() if now is None else now
2648
+ await self._execute(
2649
+ "UPDATE users SET last_login_at=?, failed_attempts=0, locked_until=NULL,"
2650
+ " updated_at=? WHERE id=?",
2651
+ (now, now, user_id),
2652
+ )
2653
+
2654
+ async def record_login_failure(
2655
+ self,
2656
+ user_id: str,
2657
+ *,
2658
+ failed_attempts: int,
2659
+ locked_until: float | None,
2660
+ now: float | None = None,
2661
+ ) -> None:
2662
+ now = time.time() if now is None else now
2663
+ await self._execute(
2664
+ "UPDATE users SET failed_attempts=?, locked_until=?, updated_at=? WHERE id=?",
2665
+ (failed_attempts, locked_until, now, user_id),
2666
+ )
2667
+
2668
+ async def upsert_role(
2669
+ self,
2670
+ *,
2671
+ role_id: str,
2672
+ display_name: str,
2673
+ description: str | None = None,
2674
+ builtin: bool = True,
2675
+ ) -> None:
2676
+ # Single atomic MERGE under HOLDLOCK (range-locks the key) so two concurrent seeders can't both
2677
+ # find the row absent and both INSERT the same PK -> violation (the UPDATE-then-INSERT race).
2678
+ await self._execute(
2679
+ "MERGE roles WITH (HOLDLOCK) AS t"
2680
+ " USING (SELECT ? AS id, ? AS display_name, ? AS description, ? AS builtin) AS s"
2681
+ " ON t.id=s.id"
2682
+ " WHEN MATCHED THEN UPDATE SET display_name=s.display_name,"
2683
+ " description=s.description, builtin=s.builtin"
2684
+ " WHEN NOT MATCHED THEN INSERT (id, display_name, description, builtin)"
2685
+ " VALUES (s.id, s.display_name, s.description, s.builtin);",
2686
+ (role_id, display_name, description, 1 if builtin else 0),
2687
+ )
2688
+
2689
+ async def list_roles(self) -> list[dict[str, Any]]:
2690
+ return await self._fetchall("SELECT * FROM roles ORDER BY id")
2691
+
2692
+ async def get_user_role_ids(self, user_id: str) -> list[str]:
2693
+ rows = await self._fetchall(
2694
+ "SELECT role_id FROM user_roles WHERE user_id=? ORDER BY role_id", (user_id,)
2695
+ )
2696
+ return [str(r["role_id"]) for r in rows]
2697
+
2698
+ async def set_user_roles(
2699
+ self,
2700
+ user_id: str,
2701
+ role_ids: Sequence[str],
2702
+ *,
2703
+ assigned_by: str | None = None,
2704
+ now: float | None = None,
2705
+ ) -> None:
2706
+ now = time.time() if now is None else now
2707
+ async with self._acquire() as conn:
2708
+ cur = await conn.cursor()
2709
+ try:
2710
+ await cur.execute("DELETE FROM user_roles WHERE user_id=?", (user_id,))
2711
+ for role_id in role_ids:
2712
+ await cur.execute(
2713
+ "INSERT INTO user_roles (user_id, role_id, assigned_at, assigned_by)"
2714
+ " VALUES (?,?,?,?)",
2715
+ (user_id, role_id, now, assigned_by),
2716
+ )
2717
+ await conn.commit()
2718
+ except Exception:
2719
+ await conn.rollback()
2720
+ raise
2721
+
2722
+ async def set_user_channel_scope(
2723
+ self, user_id: str, scope_json: str | None, *, now: float | None = None
2724
+ ) -> None:
2725
+ now = time.time() if now is None else now
2726
+ await self._execute(
2727
+ "UPDATE users SET channel_scope=?, updated_at=? WHERE id=?",
2728
+ (scope_json, now, user_id),
2729
+ )
2730
+
2731
+ async def roles_for_ad_groups(self, groups: Iterable[str]) -> set[str]:
2732
+ normalized = sorted({g.strip().lower() for g in groups if g.strip()})
2733
+ if not normalized:
2734
+ return set()
2735
+ placeholders = ",".join("?" * len(normalized)) # count-bound, not user text
2736
+ rows = await self._fetchall(
2737
+ f"SELECT DISTINCT role_id FROM ad_group_role_map WHERE ad_group IN ({placeholders})",
2738
+ tuple(normalized),
2739
+ )
2740
+ return {str(r["role_id"]) for r in rows}
2741
+
2742
+ async def list_ad_group_role_map(self) -> list[dict[str, Any]]:
2743
+ return await self._fetchall(
2744
+ "SELECT ad_group, role_id FROM ad_group_role_map ORDER BY ad_group, role_id"
2745
+ )
2746
+
2747
+ async def set_ad_group_role_map(self, entries: Iterable[tuple[str, str]]) -> None:
2748
+ pairs = sorted({(g.strip().lower(), r) for g, r in entries if g.strip()})
2749
+ async with self._acquire() as conn:
2750
+ cur = await conn.cursor()
2751
+ try:
2752
+ await cur.execute("DELETE FROM ad_group_role_map")
2753
+ for ad_group, role_id in pairs:
2754
+ await cur.execute(
2755
+ "INSERT INTO ad_group_role_map (ad_group, role_id) VALUES (?,?)",
2756
+ (ad_group, role_id),
2757
+ )
2758
+ await conn.commit()
2759
+ except Exception:
2760
+ await conn.rollback()
2761
+ raise
2762
+
2763
+ async def channels_for_ad_groups(self, groups: Iterable[str]) -> set[str]:
2764
+ normalized = sorted({g.strip().lower() for g in groups if g.strip()})
2765
+ if not normalized:
2766
+ return set()
2767
+ placeholders = ",".join("?" * len(normalized)) # count-bound, not user text
2768
+ rows = await self._fetchall(
2769
+ f"SELECT DISTINCT channel FROM ad_group_scope_map WHERE ad_group IN ({placeholders})",
2770
+ tuple(normalized),
2771
+ )
2772
+ return {str(r["channel"]) for r in rows}
2773
+
2774
+ async def list_ad_group_scope_map(self) -> list[dict[str, Any]]:
2775
+ return await self._fetchall(
2776
+ "SELECT ad_group, channel FROM ad_group_scope_map ORDER BY ad_group, channel"
2777
+ )
2778
+
2779
+ async def set_ad_group_scope_map(self, entries: Iterable[tuple[str, str]]) -> None:
2780
+ pairs = sorted(
2781
+ {(g.strip().lower(), c.strip()) for g, c in entries if g.strip() and c.strip()}
2782
+ )
2783
+ async with self._acquire() as conn:
2784
+ cur = await conn.cursor()
2785
+ try:
2786
+ await cur.execute("DELETE FROM ad_group_scope_map")
2787
+ for ad_group, channel in pairs:
2788
+ await cur.execute(
2789
+ "INSERT INTO ad_group_scope_map (ad_group, channel) VALUES (?,?)",
2790
+ (ad_group, channel),
2791
+ )
2792
+ await conn.commit()
2793
+ except Exception:
2794
+ await conn.rollback()
2795
+ raise
2796
+
2797
+ async def create_session(
2798
+ self,
2799
+ *,
2800
+ token_hash: str,
2801
+ user_id: str,
2802
+ expires_at: float,
2803
+ client: str | None = None,
2804
+ seed_reauth: bool = True,
2805
+ now: float | None = None,
2806
+ ) -> None:
2807
+ now = time.time() if now is None else now
2808
+ await self._execute(
2809
+ # reauth_at seeds the step-up window from login (ASVS 7.5.3); seed_reauth=False leaves it
2810
+ # NULL for an MFA-PENDING session (WP-14) so a stolen pre-MFA token can't enroll/step-up.
2811
+ "INSERT INTO sessions (token_hash, user_id, created_at, expires_at, last_used_at,"
2812
+ " revoked_at, client, reauth_at) VALUES (?,?,?,?,?,NULL,?,?)",
2813
+ (token_hash, user_id, now, expires_at, now, client, now if seed_reauth else None),
2814
+ )
2815
+
2816
+ async def get_session(self, token_hash: str) -> SessionRecord | None:
2817
+ d = await self._fetchone("SELECT * FROM sessions WHERE token_hash=?", (token_hash,))
2818
+ return SessionRecord.from_mapping(d) if d else None
2819
+
2820
+ async def list_sessions(self, user_id: str, *, now: float | None = None) -> list[SessionRecord]:
2821
+ """A user's active (not revoked/expired) sessions, most-recently-used first (WP-10)."""
2822
+ now = time.time() if now is None else now
2823
+ rows = await self._fetchall(
2824
+ "SELECT * FROM sessions WHERE user_id=? AND revoked_at IS NULL AND expires_at > ?"
2825
+ " ORDER BY last_used_at DESC",
2826
+ (user_id, now),
2827
+ )
2828
+ return [SessionRecord.from_mapping(r) for r in rows]
2829
+
2830
+ async def touch_session(self, token_hash: str, *, now: float | None = None) -> None:
2831
+ now = time.time() if now is None else now
2832
+ await self._execute(
2833
+ "UPDATE sessions SET last_used_at=? WHERE token_hash=?", (now, token_hash)
2834
+ )
2835
+
2836
+ async def mark_session_reauthed(
2837
+ self, token_hash: str, *, now: float | None = None, client: str | None = None
2838
+ ) -> None:
2839
+ now = time.time() if now is None else now
2840
+ # COALESCE keeps the stored client when none is supplied; a re-verify carrying the current
2841
+ # address re-anchors the session to it (WP-L3-13 new-client-IP step-up).
2842
+ await self._execute(
2843
+ "UPDATE sessions SET reauth_at=?, client=COALESCE(?, client) WHERE token_hash=?",
2844
+ (now, client, token_hash),
2845
+ )
2846
+
2847
+ async def mark_session_mfa_verified(self, token_hash: str, *, now: float | None = None) -> None:
2848
+ now = time.time() if now is None else now
2849
+ await self._execute(
2850
+ "UPDATE sessions SET mfa_verified_at=? WHERE token_hash=?", (now, token_hash)
2851
+ )
2852
+
2853
+ async def revoke_session(self, token_hash: str, *, now: float | None = None) -> None:
2854
+ now = time.time() if now is None else now
2855
+ await self._execute(
2856
+ "UPDATE sessions SET revoked_at=? WHERE token_hash=? AND revoked_at IS NULL",
2857
+ (now, token_hash),
2858
+ )
2859
+
2860
+ async def revoke_user_sessions(
2861
+ self, user_id: str, *, except_token_hash: str | None = None, now: float | None = None
2862
+ ) -> int:
2863
+ """Revoke a user's active sessions (all, or all but ``except_token_hash``). Returns the count."""
2864
+ now = time.time() if now is None else now
2865
+ sql = "UPDATE sessions SET revoked_at=? WHERE user_id=? AND revoked_at IS NULL"
2866
+ params: list[Any] = [now, user_id]
2867
+ if except_token_hash is not None:
2868
+ sql += " AND token_hash != ?"
2869
+ params.append(except_token_hash)
2870
+ async with self._acquire() as conn:
2871
+ cur = await conn.cursor()
2872
+ try:
2873
+ await cur.execute(sql, tuple(params))
2874
+ count = cur.rowcount
2875
+ await conn.commit()
2876
+ except Exception:
2877
+ await conn.rollback()
2878
+ raise
2879
+ return int(count) if count is not None else 0
2880
+
2881
+ async def enforce_session_cap(
2882
+ self, user_id: str, *, keep: int, now: float | None = None
2883
+ ) -> None:
2884
+ """Revoke a user's active sessions beyond the ``keep`` most recently created (AUTH-SESS-CAP)."""
2885
+ if keep <= 0:
2886
+ return
2887
+ now = time.time() if now is None else now
2888
+ await self._execute(
2889
+ "UPDATE sessions SET revoked_at=? WHERE user_id=? AND revoked_at IS NULL"
2890
+ " AND token_hash NOT IN ("
2891
+ " SELECT TOP (?) token_hash FROM sessions WHERE user_id=? AND revoked_at IS NULL"
2892
+ " ORDER BY created_at DESC, token_hash DESC"
2893
+ ")",
2894
+ (now, user_id, keep, user_id),
2895
+ )
2896
+
2897
+ async def purge_expired_sessions(self, *, now: float | None = None) -> int:
2898
+ now = time.time() if now is None else now
2899
+ async with self._acquire() as conn:
2900
+ cur = await conn.cursor()
2901
+ try:
2902
+ await cur.execute("DELETE FROM sessions WHERE expires_at < ?", (now,))
2903
+ count = cur.rowcount
2904
+ await conn.commit()
2905
+ except Exception:
2906
+ await conn.rollback()
2907
+ raise
2908
+ return int(count) if count is not None else 0
2909
+
2910
+ async def stats(self) -> dict[str, int]:
2911
+ rows = await self._fetchall(
2912
+ "SELECT status, COUNT(*) AS n FROM queue WHERE stage=? GROUP BY status",
2913
+ (Stage.OUTBOUND.value,),
2914
+ )
2915
+ return {r["status"]: int(r["n"]) for r in rows}
2916
+
2917
+ async def in_pipeline_depth(self) -> int:
2918
+ rows = await self._fetchall(
2919
+ "SELECT COUNT(*) AS n FROM queue WHERE stage IN (?,?,?) AND status IN (?,?)",
2920
+ (
2921
+ Stage.INGRESS.value,
2922
+ Stage.ROUTED.value,
2923
+ Stage.OUTBOUND.value,
2924
+ OutboxStatus.PENDING.value,
2925
+ OutboxStatus.INFLIGHT.value,
2926
+ ),
2927
+ )
2928
+ return int(rows[0]["n"]) if rows else 0
2929
+
2930
+ async def db_status(self) -> DbStatus:
2931
+ recovery = await self._fetchone(
2932
+ "SELECT recovery_model_desc AS m FROM sys.databases WHERE name = DB_NAME()"
2933
+ )
2934
+ size = await self._fetchone(
2935
+ "SELECT CAST(SUM(size) AS BIGINT) * 8 * 1024 AS b FROM sys.database_files"
2936
+ )
2937
+ return DbStatus(
2938
+ path=self.path,
2939
+ size_bytes=int(size["b"]) if size and size["b"] is not None else 0,
2940
+ disk_free_bytes=0, # not readily available for a remote SQL Server
2941
+ journal_mode=str(recovery["m"]) if recovery and recovery["m"] else "",
2942
+ messages=await self._count("messages"),
2943
+ events=await self._count("message_events"),
2944
+ audit=await self._count("audit_log"),
2945
+ )
2946
+
2947
+ async def integrity_check(self) -> tuple[bool, str]:
2948
+ # A connectivity/consistency probe; deep checks (DBCC CHECKDB) are an out-of-band DBA task.
2949
+ await self._fetchone("SELECT 1 AS ok")
2950
+ return True, "ok (SQL Server: run DBCC CHECKDB out-of-band for deep checks)"
2951
+
2952
+ async def _count(self, table: str) -> int:
2953
+ row = await self._fetchone(f"SELECT COUNT(*) AS n FROM {table}") # table is a constant
2954
+ return int(row["n"]) if row else 0
2955
+
2956
+ async def connection_metrics(
2957
+ self, *, since: float, now: float | None = None, rate_window: float = 60.0
2958
+ ) -> ConnectionMetrics:
2959
+ now = time.time() if now is None else now
2960
+ rate_since = now - rate_window
2961
+
2962
+ count_rows = await self._fetchall(
2963
+ "SELECT channel_id, COUNT(*) AS [read],"
2964
+ " SUM(CASE WHEN status=? THEN 1 ELSE 0 END) AS errored"
2965
+ " FROM messages WHERE received_at>=? GROUP BY channel_id",
2966
+ (MessageStatus.ERROR.value, since),
2967
+ )
2968
+ counts = {r["channel_id"]: (r["read"], r["errored"]) for r in count_rows}
2969
+ last_rows = await self._fetchall(
2970
+ "SELECT channel_id, MAX(received_at) AS last_at FROM messages GROUP BY channel_id"
2971
+ )
2972
+ inbound: dict[str, InboundMetrics] = {}
2973
+ for r in last_rows:
2974
+ read, errored = counts.pop(r["channel_id"], (0, 0))
2975
+ inbound[r["channel_id"]] = InboundMetrics(
2976
+ read=int(read), errored=int(errored or 0), last_at=r["last_at"]
2977
+ )
2978
+ for cid, (read, errored) in counts.items():
2979
+ inbound[cid] = InboundMetrics(read=int(read), errored=int(errored or 0), last_at=None)
2980
+
2981
+ dest_rows = await self._fetchall(
2982
+ "SELECT channel_id, destination_name,"
2983
+ " SUM(CASE WHEN status IN (?,?) THEN 1 ELSE 0 END) AS queue_depth,"
2984
+ " SUM(CASE WHEN status=? AND updated_at>=? THEN 1 ELSE 0 END) AS written,"
2985
+ " SUM(CASE WHEN status=? AND updated_at>=? THEN 1 ELSE 0 END) AS dead,"
2986
+ " MIN(CASE WHEN status=? THEN created_at END) AS oldest_pending_at,"
2987
+ " SUM(CASE WHEN status=? AND updated_at>=? THEN 1 ELSE 0 END) AS recent_done,"
2988
+ " MAX(CASE WHEN status=? THEN updated_at END) AS last_done_at"
2989
+ " FROM queue WHERE stage=? GROUP BY channel_id, destination_name",
2990
+ (
2991
+ OutboxStatus.PENDING.value,
2992
+ OutboxStatus.INFLIGHT.value,
2993
+ OutboxStatus.DONE.value,
2994
+ since,
2995
+ OutboxStatus.DEAD.value,
2996
+ since,
2997
+ OutboxStatus.PENDING.value,
2998
+ OutboxStatus.DONE.value,
2999
+ rate_since,
3000
+ OutboxStatus.DONE.value,
3001
+ Stage.OUTBOUND.value,
3002
+ ),
3003
+ )
3004
+ destinations: dict[tuple[str, str], DestinationMetrics] = {}
3005
+ for r in dest_rows:
3006
+ destinations[(r["channel_id"], r["destination_name"])] = DestinationMetrics(
3007
+ queue_depth=int(r["queue_depth"] or 0),
3008
+ written=int(r["written"] or 0),
3009
+ dead=int(r["dead"] or 0),
3010
+ oldest_pending_at=r["oldest_pending_at"],
3011
+ recent_done=int(r["recent_done"] or 0),
3012
+ last_done_at=r["last_done_at"],
3013
+ )
3014
+ return ConnectionMetrics(inbound=inbound, destinations=destinations)