messagefoundry 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- messagefoundry/__init__.py +108 -0
- messagefoundry/__main__.py +1155 -0
- messagefoundry/api/__init__.py +27 -0
- messagefoundry/api/app.py +1581 -0
- messagefoundry/api/approvals.py +184 -0
- messagefoundry/api/auth_models.py +211 -0
- messagefoundry/api/auth_routes.py +655 -0
- messagefoundry/api/field_authz.py +96 -0
- messagefoundry/api/models.py +374 -0
- messagefoundry/api/security.py +247 -0
- messagefoundry/api/tls.py +47 -0
- messagefoundry/auth/__init__.py +39 -0
- messagefoundry/auth/data/common_passwords.NOTICE +13 -0
- messagefoundry/auth/data/common_passwords.txt +10000 -0
- messagefoundry/auth/identity.py +71 -0
- messagefoundry/auth/ldap.py +264 -0
- messagefoundry/auth/notifications.py +68 -0
- messagefoundry/auth/passwords.py +53 -0
- messagefoundry/auth/permissions.py +120 -0
- messagefoundry/auth/policy.py +153 -0
- messagefoundry/auth/ratelimit.py +55 -0
- messagefoundry/auth/service.py +1323 -0
- messagefoundry/auth/tokens.py +26 -0
- messagefoundry/auth/totp.py +174 -0
- messagefoundry/checks.py +174 -0
- messagefoundry/config/__init__.py +30 -0
- messagefoundry/config/active_environment.py +80 -0
- messagefoundry/config/ai_policy.py +140 -0
- messagefoundry/config/code_sets.py +260 -0
- messagefoundry/config/connections_edit.py +200 -0
- messagefoundry/config/connections_file.py +287 -0
- messagefoundry/config/db_lookup.py +117 -0
- messagefoundry/config/environments.py +116 -0
- messagefoundry/config/ingest_time.py +83 -0
- messagefoundry/config/models.py +240 -0
- messagefoundry/config/reference.py +158 -0
- messagefoundry/config/response.py +83 -0
- messagefoundry/config/run_context.py +153 -0
- messagefoundry/config/settings.py +1311 -0
- messagefoundry/config/state.py +99 -0
- messagefoundry/config/tls_policy.py +110 -0
- messagefoundry/config/wiring.py +1918 -0
- messagefoundry/console/__init__.py +20 -0
- messagefoundry/console/__main__.py +274 -0
- messagefoundry/console/_async.py +107 -0
- messagefoundry/console/change_password.py +111 -0
- messagefoundry/console/client.py +552 -0
- messagefoundry/console/connections.py +324 -0
- messagefoundry/console/login.py +107 -0
- messagefoundry/console/mfa.py +205 -0
- messagefoundry/console/reauth.py +94 -0
- messagefoundry/console/search.py +57 -0
- messagefoundry/console/service_control.py +137 -0
- messagefoundry/console/sessions.py +122 -0
- messagefoundry/console/shell.py +410 -0
- messagefoundry/console/status.py +377 -0
- messagefoundry/console/users_page.py +282 -0
- messagefoundry/console/widgets.py +553 -0
- messagefoundry/generators/README.md +27 -0
- messagefoundry/generators/__init__.py +15 -0
- messagefoundry/generators/_core.py +589 -0
- messagefoundry/generators/_hl7data.py +428 -0
- messagefoundry/generators/adt.py +286 -0
- messagefoundry/generators/all_types.py +24 -0
- messagefoundry/generators/bar.py +28 -0
- messagefoundry/generators/dft.py +20 -0
- messagefoundry/generators/mdm.py +39 -0
- messagefoundry/generators/mfn.py +46 -0
- messagefoundry/generators/oml.py +32 -0
- messagefoundry/generators/orl.py +30 -0
- messagefoundry/generators/orm.py +23 -0
- messagefoundry/generators/oru.py +21 -0
- messagefoundry/generators/ras.py +20 -0
- messagefoundry/generators/rde.py +54 -0
- messagefoundry/generators/siu.py +64 -0
- messagefoundry/generators/vxu.py +20 -0
- messagefoundry/hl7schema.py +75 -0
- messagefoundry/last_resort.py +55 -0
- messagefoundry/logging_setup.py +332 -0
- messagefoundry/parsing/__init__.py +64 -0
- messagefoundry/parsing/consistency.py +166 -0
- messagefoundry/parsing/groups.py +228 -0
- messagefoundry/parsing/message.py +453 -0
- messagefoundry/parsing/peek.py +237 -0
- messagefoundry/parsing/split.py +120 -0
- messagefoundry/parsing/summary.py +46 -0
- messagefoundry/parsing/tree.py +128 -0
- messagefoundry/parsing/validate.py +95 -0
- messagefoundry/parsing/x12/__init__.py +46 -0
- messagefoundry/parsing/x12/delimiters.py +140 -0
- messagefoundry/parsing/x12/errors.py +30 -0
- messagefoundry/parsing/x12/interchange.py +232 -0
- messagefoundry/parsing/x12/message.py +200 -0
- messagefoundry/parsing/x12/peek.py +207 -0
- messagefoundry/pipeline/__init__.py +21 -0
- messagefoundry/pipeline/alert_sinks.py +486 -0
- messagefoundry/pipeline/alerts.py +100 -0
- messagefoundry/pipeline/cert_expiry.py +219 -0
- messagefoundry/pipeline/cluster.py +955 -0
- messagefoundry/pipeline/cluster_sqlserver.py +444 -0
- messagefoundry/pipeline/config_convergence.py +137 -0
- messagefoundry/pipeline/dryrun.py +450 -0
- messagefoundry/pipeline/engine.py +756 -0
- messagefoundry/pipeline/leader_tasks.py +158 -0
- messagefoundry/pipeline/reference_sync.py +369 -0
- messagefoundry/pipeline/retention.py +289 -0
- messagefoundry/pipeline/security_notify.py +168 -0
- messagefoundry/pipeline/state_convergence.py +143 -0
- messagefoundry/pipeline/wiring_runner.py +1722 -0
- messagefoundry/py.typed +0 -0
- messagefoundry/redaction.py +71 -0
- messagefoundry/scaffold.py +321 -0
- messagefoundry/secrets_dpapi.py +129 -0
- messagefoundry/store/__init__.py +46 -0
- messagefoundry/store/audit_tee.py +67 -0
- messagefoundry/store/base.py +758 -0
- messagefoundry/store/crypto.py +166 -0
- messagefoundry/store/keyprovider.py +192 -0
- messagefoundry/store/postgres.py +3447 -0
- messagefoundry/store/sqlserver.py +3014 -0
- messagefoundry/store/store.py +3790 -0
- messagefoundry/timezone.py +207 -0
- messagefoundry/transports/__init__.py +50 -0
- messagefoundry/transports/base.py +269 -0
- messagefoundry/transports/database.py +693 -0
- messagefoundry/transports/file.py +551 -0
- messagefoundry/transports/framing.py +164 -0
- messagefoundry/transports/loopback.py +53 -0
- messagefoundry/transports/mllp.py +644 -0
- messagefoundry/transports/remotefile.py +664 -0
- messagefoundry/transports/rest.py +281 -0
- messagefoundry/transports/signing.py +321 -0
- messagefoundry/transports/soap.py +507 -0
- messagefoundry/transports/tcp.py +307 -0
- messagefoundry/transports/timer.py +146 -0
- messagefoundry/transports/x12.py +323 -0
- messagefoundry-0.1.0.dist-info/METADATA +212 -0
- messagefoundry-0.1.0.dist-info/RECORD +142 -0
- messagefoundry-0.1.0.dist-info/WHEEL +4 -0
- messagefoundry-0.1.0.dist-info/entry_points.txt +2 -0
- messagefoundry-0.1.0.dist-info/licenses/LICENSE +662 -0
- messagefoundry-0.1.0.dist-info/licenses/NOTICE +27 -0
|
@@ -0,0 +1,3014 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
# Copyright (C) 2026 MessageFoundry Organization and contributors
|
|
3
|
+
"""Production SQL Server implementation of the :class:`~messagefoundry.store.base.Store` protocol.
|
|
4
|
+
|
|
5
|
+
Runs the full ADR-0001 staged pipeline (ingress -> routed -> outbound) + ADR-0013 query/response on a
|
|
6
|
+
unified ``queue`` table, mirroring the SQLite :class:`~messagefoundry.store.store.MessageStore`
|
|
7
|
+
semantics (at-least-once delivery, retries, replay, dead-lettering, retention, encryption-at-rest) in
|
|
8
|
+
T-SQL over ``aioodbc``. Concurrency uses SQL Server row-locking — ``claim_ready`` claims due rows with
|
|
9
|
+
``WITH (READPAST, UPDLOCK, ROWLOCK)`` so independent delivery workers don't block or double-claim — plus
|
|
10
|
+
RCSI and an ``sp_getapplock``-serialized finalizer, lifting SQLite's single-writer ceiling. Both
|
|
11
|
+
``supports_ingest_stage`` and ``supports_response_capture`` are True.
|
|
12
|
+
|
|
13
|
+
``aioodbc`` is an **optional extra** (``pip install 'messagefoundry[sqlserver]'``) and also needs the
|
|
14
|
+
Microsoft ODBC Driver 18 at the OS level. It's imported lazily in :meth:`SqlServerStore.open` so
|
|
15
|
+
SQLite-only installs never touch it. Verified against a real SQL Server by the CI service-container job
|
|
16
|
+
(the store suite + the SQL Server load smoke).
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import asyncio
|
|
22
|
+
import json
|
|
23
|
+
import logging
|
|
24
|
+
import time
|
|
25
|
+
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
|
|
26
|
+
from contextlib import asynccontextmanager
|
|
27
|
+
from types import MappingProxyType
|
|
28
|
+
from typing import Any
|
|
29
|
+
from uuid import uuid4
|
|
30
|
+
|
|
31
|
+
from messagefoundry.config.models import RetryPolicy
|
|
32
|
+
from messagefoundry.config.settings import (
|
|
33
|
+
INSECURE_TLS_ESCAPE_ENV,
|
|
34
|
+
SqlAuth,
|
|
35
|
+
StoreSettings,
|
|
36
|
+
insecure_tls_allowed,
|
|
37
|
+
)
|
|
38
|
+
from messagefoundry.redaction import safe_text
|
|
39
|
+
from messagefoundry.store.audit_tee import emit_audit_tee
|
|
40
|
+
from messagefoundry.store.crypto import PREFIX as _ENC_PREFIX
|
|
41
|
+
from messagefoundry.store.crypto import AesGcmCipher, Cipher, CipherError, IdentityCipher
|
|
42
|
+
from messagefoundry.store.store import (
|
|
43
|
+
ConnectionMetrics,
|
|
44
|
+
DbStatus,
|
|
45
|
+
DestinationMetrics,
|
|
46
|
+
InboundMetrics,
|
|
47
|
+
CapturedResponse,
|
|
48
|
+
MessageStatus,
|
|
49
|
+
MessageStore,
|
|
50
|
+
OutboxItem,
|
|
51
|
+
OutboxStatus,
|
|
52
|
+
SessionRecord,
|
|
53
|
+
Stage,
|
|
54
|
+
UserRecord,
|
|
55
|
+
_append_channel_scope,
|
|
56
|
+
audit_row_hash,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
log = logging.getLogger(__name__)
|
|
60
|
+
|
|
61
|
+
# Schema (T-SQL). Idempotent: guarded by OBJECT_ID / IndexProperty so re-open is a no-op. Epoch
|
|
62
|
+
# timestamps are FLOAT; ids are NVARCHAR(64) (uuid4 hex); bodies NVARCHAR(MAX).
|
|
63
|
+
_SCHEMA: list[str] = [
|
|
64
|
+
"""IF OBJECT_ID('messages','U') IS NULL CREATE TABLE messages (
|
|
65
|
+
id NVARCHAR(64) NOT NULL PRIMARY KEY, channel_id NVARCHAR(256) NOT NULL,
|
|
66
|
+
received_at FLOAT NOT NULL, source_type NVARCHAR(64) NULL, control_id NVARCHAR(256) NULL,
|
|
67
|
+
message_type NVARCHAR(64) NULL, raw NVARCHAR(MAX) NOT NULL, status NVARCHAR(32) NOT NULL,
|
|
68
|
+
error NVARCHAR(MAX) NULL, summary NVARCHAR(MAX) NULL, metadata NVARCHAR(MAX) NULL)""",
|
|
69
|
+
"""IF INDEXPROPERTY(OBJECT_ID('messages'),'ix_messages_channel','IndexID') IS NULL
|
|
70
|
+
CREATE INDEX ix_messages_channel ON messages(channel_id, received_at)""",
|
|
71
|
+
"""IF INDEXPROPERTY(OBJECT_ID('messages'),'ix_messages_control','IndexID') IS NULL
|
|
72
|
+
CREATE INDEX ix_messages_control ON messages(channel_id, control_id)""",
|
|
73
|
+
"""IF OBJECT_ID('outbox','U') IS NULL CREATE TABLE outbox (
|
|
74
|
+
id NVARCHAR(64) NOT NULL PRIMARY KEY, message_id NVARCHAR(64) NOT NULL,
|
|
75
|
+
channel_id NVARCHAR(256) NOT NULL, destination_name NVARCHAR(256) NOT NULL,
|
|
76
|
+
payload NVARCHAR(MAX) NOT NULL, status NVARCHAR(32) NOT NULL,
|
|
77
|
+
attempts INT NOT NULL DEFAULT 0, next_attempt_at FLOAT NOT NULL, last_error NVARCHAR(MAX) NULL,
|
|
78
|
+
created_at FLOAT NOT NULL, updated_at FLOAT NOT NULL)""",
|
|
79
|
+
"""IF INDEXPROPERTY(OBJECT_ID('outbox'),'ix_outbox_ready','IndexID') IS NULL
|
|
80
|
+
CREATE INDEX ix_outbox_ready ON outbox(status, next_attempt_at)""",
|
|
81
|
+
"""IF INDEXPROPERTY(OBJECT_ID('outbox'),'ix_outbox_message','IndexID') IS NULL
|
|
82
|
+
CREATE INDEX ix_outbox_message ON outbox(message_id)""",
|
|
83
|
+
# Unified staged queue (ADR 0001) — ingress -> routed -> outbound, one row per stage-unit with a
|
|
84
|
+
# `stage` discriminator. The SQL Server backend originally shipped only the flat `outbox`; the
|
|
85
|
+
# staged pipeline (enqueue_ingress + the handoffs) and ALL delivery-side methods now read/write
|
|
86
|
+
# this table (outbox is retained only for legacy read-compat). `seq` IDENTITY is the FIFO
|
|
87
|
+
# insertion-order tiebreak (PG uses BIGSERIAL); owner/lease_expires_at are present for parity but
|
|
88
|
+
# written NULL on this single-node backend (reset_stale_inflight is the recovery path).
|
|
89
|
+
"""IF OBJECT_ID('queue','U') IS NULL CREATE TABLE queue (
|
|
90
|
+
id NVARCHAR(64) NOT NULL PRIMARY KEY, seq BIGINT IDENTITY(1,1) NOT NULL,
|
|
91
|
+
message_id NVARCHAR(64) NOT NULL, stage NVARCHAR(16) NOT NULL,
|
|
92
|
+
channel_id NVARCHAR(256) NOT NULL, destination_name NVARCHAR(256) NULL,
|
|
93
|
+
handler_name NVARCHAR(256) NULL, payload NVARCHAR(MAX) NOT NULL, status NVARCHAR(32) NOT NULL,
|
|
94
|
+
attempts INT NOT NULL DEFAULT 0, next_attempt_at FLOAT NOT NULL, last_error NVARCHAR(MAX) NULL,
|
|
95
|
+
owner NVARCHAR(256) NULL, lease_expires_at FLOAT NULL,
|
|
96
|
+
created_at FLOAT NOT NULL, updated_at FLOAT NOT NULL,
|
|
97
|
+
CONSTRAINT fk_queue_message FOREIGN KEY (message_id) REFERENCES messages(id))""",
|
|
98
|
+
"""IF INDEXPROPERTY(OBJECT_ID('queue'),'ix_queue_ready','IndexID') IS NULL
|
|
99
|
+
CREATE INDEX ix_queue_ready ON queue(stage, status, next_attempt_at)""",
|
|
100
|
+
"""IF INDEXPROPERTY(OBJECT_ID('queue'),'ix_queue_message','IndexID') IS NULL
|
|
101
|
+
CREATE INDEX ix_queue_message ON queue(message_id)""",
|
|
102
|
+
"""IF INDEXPROPERTY(OBJECT_ID('queue'),'ix_queue_fifo_out','IndexID') IS NULL
|
|
103
|
+
CREATE INDEX ix_queue_fifo_out ON queue(stage, destination_name, status, created_at, seq)""",
|
|
104
|
+
"""IF INDEXPROPERTY(OBJECT_ID('queue'),'ix_queue_fifo_in','IndexID') IS NULL
|
|
105
|
+
CREATE INDEX ix_queue_fifo_in ON queue(stage, channel_id, status, created_at, seq)""",
|
|
106
|
+
# LOCK_ESCALATION=DISABLE: `queue` is a hot multi-writer table; a depth-triggered escalation to a
|
|
107
|
+
# TABLE X lock during a deep startup orphan sweep would block ALL claim/handoff workers. Degrade a
|
|
108
|
+
# deep sweep to many row locks under RCSI instead. Idempotent (re-running re-sets the same option).
|
|
109
|
+
# IF-guarded (like the indexes) so it fires at most once — a bare ALTER on every open() takes a
|
|
110
|
+
# Sch-M lock on the hot queue table (review). lock_escalation 2 = DISABLE.
|
|
111
|
+
"""IF (SELECT lock_escalation FROM sys.tables WHERE object_id=OBJECT_ID('queue')) <> 2
|
|
112
|
+
ALTER TABLE queue SET (LOCK_ESCALATION = DISABLE)""",
|
|
113
|
+
"""IF OBJECT_ID('message_events','U') IS NULL CREATE TABLE message_events (
|
|
114
|
+
id INT IDENTITY(1,1) PRIMARY KEY, message_id NVARCHAR(64) NOT NULL, ts FLOAT NOT NULL,
|
|
115
|
+
event NVARCHAR(64) NOT NULL, destination NVARCHAR(256) NULL, detail NVARCHAR(MAX) NULL)""",
|
|
116
|
+
"""IF INDEXPROPERTY(OBJECT_ID('message_events'),'ix_events_message','IndexID') IS NULL
|
|
117
|
+
CREATE INDEX ix_events_message ON message_events(message_id, ts)""",
|
|
118
|
+
# Transform-accessible state (ADR 0005). Written here via transform_handoff (parity with SQLite/
|
|
119
|
+
# Postgres): the read-through cache is loaded at open and refreshed post-commit, so a Handler's
|
|
120
|
+
# cross-message state_get(...) resolves in-process. Schema matches SQLite.
|
|
121
|
+
"""IF OBJECT_ID('state','U') IS NULL CREATE TABLE state (
|
|
122
|
+
namespace NVARCHAR(256) NOT NULL, [key] NVARCHAR(256) NOT NULL, value NVARCHAR(MAX) NOT NULL,
|
|
123
|
+
set_at FLOAT NOT NULL, message_id NVARCHAR(64) NULL,
|
|
124
|
+
CONSTRAINT pk_state PRIMARY KEY (namespace, [key]))""",
|
|
125
|
+
"""IF INDEXPROPERTY(OBJECT_ID('state'),'ix_state_set_at','IndexID') IS NULL
|
|
126
|
+
CREATE INDEX ix_state_set_at ON state(set_at)""",
|
|
127
|
+
"""IF OBJECT_ID('audit_log','U') IS NULL CREATE TABLE audit_log (
|
|
128
|
+
id INT IDENTITY(1,1) PRIMARY KEY, ts FLOAT NOT NULL, actor NVARCHAR(256) NULL,
|
|
129
|
+
action NVARCHAR(128) NOT NULL, channel_id NVARCHAR(256) NULL, detail NVARCHAR(MAX) NULL,
|
|
130
|
+
row_hash NVARCHAR(64) NULL)""",
|
|
131
|
+
"""IF COL_LENGTH('audit_log','row_hash') IS NULL
|
|
132
|
+
ALTER TABLE audit_log ADD row_hash NVARCHAR(64) NULL""",
|
|
133
|
+
"""IF INDEXPROPERTY(OBJECT_ID('audit_log'),'ix_audit_ts','IndexID') IS NULL
|
|
134
|
+
CREATE INDEX ix_audit_ts ON audit_log(ts)""",
|
|
135
|
+
"""IF OBJECT_ID('pending_approvals','U') IS NULL CREATE TABLE pending_approvals (
|
|
136
|
+
id NVARCHAR(64) NOT NULL PRIMARY KEY, operation NVARCHAR(128) NOT NULL,
|
|
137
|
+
params NVARCHAR(MAX) NOT NULL, requester NVARCHAR(256) NOT NULL,
|
|
138
|
+
requested_at FLOAT NOT NULL, status NVARCHAR(20) NOT NULL DEFAULT 'pending',
|
|
139
|
+
approver NVARCHAR(256) NULL, decided_at FLOAT NULL, expires_at FLOAT NULL)""",
|
|
140
|
+
"""IF INDEXPROPERTY(OBJECT_ID('pending_approvals'),'ix_pending_approvals_status','IndexID') IS NULL
|
|
141
|
+
CREATE INDEX ix_pending_approvals_status ON pending_approvals(status, requested_at)""",
|
|
142
|
+
"""IF OBJECT_ID('users','U') IS NULL CREATE TABLE users (
|
|
143
|
+
id NVARCHAR(64) NOT NULL PRIMARY KEY, username NVARCHAR(256) NOT NULL UNIQUE,
|
|
144
|
+
auth_provider NVARCHAR(16) NOT NULL, display_name NVARCHAR(256) NULL,
|
|
145
|
+
email NVARCHAR(256) NULL, disabled BIT NOT NULL DEFAULT 0, created_at FLOAT NOT NULL,
|
|
146
|
+
updated_at FLOAT NOT NULL, last_login_at FLOAT NULL, password_hash NVARCHAR(512) NULL,
|
|
147
|
+
password_changed_at FLOAT NULL, must_change_password BIT NOT NULL DEFAULT 0,
|
|
148
|
+
failed_attempts INT NOT NULL DEFAULT 0, locked_until FLOAT NULL,
|
|
149
|
+
channel_scope NVARCHAR(MAX) NULL, totp_secret NVARCHAR(MAX) NULL,
|
|
150
|
+
totp_enabled BIT NOT NULL DEFAULT 0, totp_enrolled_at FLOAT NULL,
|
|
151
|
+
totp_recovery_codes NVARCHAR(MAX) NULL, last_totp_step INT NULL)""",
|
|
152
|
+
"""IF COL_LENGTH('users','channel_scope') IS NULL
|
|
153
|
+
ALTER TABLE users ADD channel_scope NVARCHAR(MAX) NULL""",
|
|
154
|
+
# MFA (WP-14): TOTP columns ALTER-ed in for a pre-existing users table (idempotent).
|
|
155
|
+
"""IF COL_LENGTH('users','totp_secret') IS NULL
|
|
156
|
+
ALTER TABLE users ADD totp_secret NVARCHAR(MAX) NULL""",
|
|
157
|
+
"""IF COL_LENGTH('users','totp_enabled') IS NULL
|
|
158
|
+
ALTER TABLE users ADD totp_enabled BIT NOT NULL DEFAULT 0""",
|
|
159
|
+
"""IF COL_LENGTH('users','totp_enrolled_at') IS NULL
|
|
160
|
+
ALTER TABLE users ADD totp_enrolled_at FLOAT NULL""",
|
|
161
|
+
"""IF COL_LENGTH('users','totp_recovery_codes') IS NULL
|
|
162
|
+
ALTER TABLE users ADD totp_recovery_codes NVARCHAR(MAX) NULL""",
|
|
163
|
+
# Single-use TOTP within the step window (ASVS 6.5.1): highest consumed time-step.
|
|
164
|
+
"""IF COL_LENGTH('users','last_totp_step') IS NULL
|
|
165
|
+
ALTER TABLE users ADD last_totp_step INT NULL""",
|
|
166
|
+
"""IF OBJECT_ID('roles','U') IS NULL CREATE TABLE roles (
|
|
167
|
+
id NVARCHAR(64) NOT NULL PRIMARY KEY, display_name NVARCHAR(128) NOT NULL,
|
|
168
|
+
description NVARCHAR(512) NULL, builtin BIT NOT NULL DEFAULT 1)""",
|
|
169
|
+
"""IF OBJECT_ID('user_roles','U') IS NULL CREATE TABLE user_roles (
|
|
170
|
+
user_id NVARCHAR(64) NOT NULL, role_id NVARCHAR(64) NOT NULL, assigned_at FLOAT NOT NULL,
|
|
171
|
+
assigned_by NVARCHAR(256) NULL, CONSTRAINT pk_user_roles PRIMARY KEY (user_id, role_id))""",
|
|
172
|
+
"""IF OBJECT_ID('ad_group_role_map','U') IS NULL CREATE TABLE ad_group_role_map (
|
|
173
|
+
ad_group NVARCHAR(256) NOT NULL, role_id NVARCHAR(64) NOT NULL,
|
|
174
|
+
CONSTRAINT pk_ad_group_role_map PRIMARY KEY (ad_group, role_id))""",
|
|
175
|
+
"""IF OBJECT_ID('ad_group_scope_map','U') IS NULL CREATE TABLE ad_group_scope_map (
|
|
176
|
+
ad_group NVARCHAR(256) NOT NULL, channel NVARCHAR(256) NOT NULL,
|
|
177
|
+
CONSTRAINT pk_ad_group_scope_map PRIMARY KEY (ad_group, channel))""",
|
|
178
|
+
"""IF OBJECT_ID('sessions','U') IS NULL CREATE TABLE sessions (
|
|
179
|
+
token_hash NVARCHAR(64) NOT NULL PRIMARY KEY, user_id NVARCHAR(64) NOT NULL,
|
|
180
|
+
created_at FLOAT NOT NULL, expires_at FLOAT NOT NULL, last_used_at FLOAT NOT NULL,
|
|
181
|
+
revoked_at FLOAT NULL, client NVARCHAR(256) NULL, reauth_at FLOAT NULL,
|
|
182
|
+
mfa_verified_at FLOAT NULL)""",
|
|
183
|
+
"""IF COL_LENGTH('sessions','reauth_at') IS NULL
|
|
184
|
+
ALTER TABLE sessions ADD reauth_at FLOAT NULL""",
|
|
185
|
+
"""IF COL_LENGTH('sessions','mfa_verified_at') IS NULL
|
|
186
|
+
ALTER TABLE sessions ADD mfa_verified_at FLOAT NULL""",
|
|
187
|
+
"""IF INDEXPROPERTY(OBJECT_ID('sessions'),'ix_sessions_user','IndexID') IS NULL
|
|
188
|
+
CREATE INDEX ix_sessions_user ON sessions(user_id)""",
|
|
189
|
+
"""IF INDEXPROPERTY(OBJECT_ID('sessions'),'ix_sessions_expires','IndexID') IS NULL
|
|
190
|
+
CREATE INDEX ix_sessions_expires ON sessions(expires_at)""",
|
|
191
|
+
# Captured request/response replies (ADR 0013) — an IMMUTABLE ARTIFACT table (composite PK), NOT a
|
|
192
|
+
# queue stage, so it is invisible to _maybe_finalize's `FROM queue` scan. response_seq is replay-
|
|
193
|
+
# stable (1+MAX per (message_id,destination_name)). body + detail are BOTH ciphertext at rest for
|
|
194
|
+
# cross-backend read-API parity with PG/SQLite (which encrypt+purge+rotate detail); outcome stays
|
|
195
|
+
# plaintext. This is the ONE place this backend encrypts a "detail"-class column — queue.last_error,
|
|
196
|
+
# messages.error and message_events.detail stay plaintext here. NOTE (#120): those columns are NOT
|
|
197
|
+
# assumed non-PHI — every write goes through the safe_exc/safe_text PHI chokepoint (record_received /
|
|
198
|
+
# mark_failed / dead_letter_now / _event) so HL7-shaped content can't land, and on read they gate on
|
|
199
|
+
# messages:view_summary. Encrypting them at rest on this backend too is a tracked defense-in-depth
|
|
200
|
+
# follow-up (docs/PHI.md); until then the read gate + the "no PHI in exceptions" convention are the
|
|
201
|
+
# controls for any residual invented free-text.
|
|
202
|
+
"""IF OBJECT_ID('response','U') IS NULL CREATE TABLE response (
|
|
203
|
+
message_id NVARCHAR(64) NOT NULL, destination_name NVARCHAR(256) NOT NULL,
|
|
204
|
+
response_seq INT NOT NULL, body NVARCHAR(MAX) NULL, outcome NVARCHAR(64) NOT NULL,
|
|
205
|
+
detail NVARCHAR(MAX) NULL, captured_at FLOAT NOT NULL,
|
|
206
|
+
CONSTRAINT pk_response PRIMARY KEY (message_id, destination_name, response_seq),
|
|
207
|
+
CONSTRAINT fk_response_message FOREIGN KEY (message_id) REFERENCES messages(id))""",
|
|
208
|
+
"""IF INDEXPROPERTY(OBJECT_ID('response'),'ix_response_message','IndexID') IS NULL
|
|
209
|
+
CREATE INDEX ix_response_message ON response(message_id)""",
|
|
210
|
+
]
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _odbc_brace(value: str) -> str:
|
|
214
|
+
"""ODBC-quote a value in braces, doubling any internal ``}`` — neutralizes ``; { } =`` inside it
|
|
215
|
+
so an attacker-influenced value (e.g. a password) can't inject extra connection keywords."""
|
|
216
|
+
return "{" + value.replace("}", "}}") + "}"
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def connection_string(settings: StoreSettings) -> str:
|
|
220
|
+
"""Build an ODBC connection string for the Microsoft ODBC Driver 18 from store settings.
|
|
221
|
+
|
|
222
|
+
Free-text values are brace-quoted to prevent connection-string injection (STORE-5), and the
|
|
223
|
+
``Encrypt``/``TrustServerCertificate`` security flags are emitted **last** so — ODBC being
|
|
224
|
+
last-wins on duplicate keywords — nothing earlier can downgrade TLS. Identity fields are also
|
|
225
|
+
validated up front (see ``StoreSettings._no_odbc_injection``)."""
|
|
226
|
+
# A weakened TLS posture (TrustServerCertificate=yes, or Encrypt=no) is MITM-able, so it REFUSES
|
|
227
|
+
# unless the explicit MEFOR_ALLOW_INSECURE_TLS dev escape is set (ASVS 12.3.2) — it can't be
|
|
228
|
+
# silently turned on in production.
|
|
229
|
+
if (settings.trust_server_certificate or not settings.encrypt) and not insecure_tls_allowed():
|
|
230
|
+
raise ValueError(
|
|
231
|
+
"SQL Server TLS is weakened (trust_server_certificate=true or encrypt=false), which is "
|
|
232
|
+
f"MITM-able. Use a trusted server certificate, or set {INSECURE_TLS_ESCAPE_ENV}=1 to "
|
|
233
|
+
"explicitly allow it for a trusted-network dev/test bind."
|
|
234
|
+
)
|
|
235
|
+
parts = [
|
|
236
|
+
"DRIVER={ODBC Driver 18 for SQL Server}",
|
|
237
|
+
f"SERVER={settings.server},{settings.port}", # server validated; port is an int
|
|
238
|
+
f"DATABASE={_odbc_brace(settings.database or '')}",
|
|
239
|
+
f"Connection Timeout={settings.connect_timeout}",
|
|
240
|
+
f"APP={_odbc_brace(settings.application_name)}",
|
|
241
|
+
]
|
|
242
|
+
if settings.auth is SqlAuth.SQL:
|
|
243
|
+
parts.append(f"UID={_odbc_brace(settings.username or '')}")
|
|
244
|
+
parts.append(f"PWD={_odbc_brace(settings.password or '')}")
|
|
245
|
+
elif settings.auth is SqlAuth.INTEGRATED:
|
|
246
|
+
parts.append("Trusted_Connection=yes")
|
|
247
|
+
elif settings.auth is SqlAuth.ENTRA:
|
|
248
|
+
parts.append("Authentication=ActiveDirectoryDefault")
|
|
249
|
+
parts.append(f"Encrypt={'yes' if settings.encrypt else 'no'}")
|
|
250
|
+
parts.append(f"TrustServerCertificate={'yes' if settings.trust_server_certificate else 'no'}")
|
|
251
|
+
return ";".join(parts) + ";"
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
class SqlServerStore:
|
|
255
|
+
"""SQL Server-backed durable queue (the :class:`Store` protocol). Open with :meth:`open`."""
|
|
256
|
+
|
|
257
|
+
# The staged ingress pipeline (enqueue_ingress + the ingress->routed->outbound handoffs) is
|
|
258
|
+
# implemented on the unified ``queue`` table: atomic DELETE...OUTPUT claim-handoffs, an
|
|
259
|
+
# sp_getapplock-serialized queue-aware finalizer, and RCSI for non-blocking claim/finalize
|
|
260
|
+
# (ADR 0001; BACKLOG #1 closed). The engine runs the staged runner on this backend.
|
|
261
|
+
supports_ingest_stage = True
|
|
262
|
+
|
|
263
|
+
# Request/response capture + re-ingress (ADR 0013) IS supported on the SQL Server store: captured
|
|
264
|
+
# replies persist to the `response` table (body + detail ciphertext, outcome plaintext) and
|
|
265
|
+
# re-ingress rides a staged Stage.RESPONSE work-row. The runner may start a capturing outbound + the
|
|
266
|
+
# re-ingress worker on this backend.
|
|
267
|
+
supports_response_capture = True
|
|
268
|
+
|
|
269
|
+
def __init__(self, pool: Any, settings: StoreSettings, *, cipher: Cipher | None = None) -> None:
|
|
270
|
+
self._pool = pool
|
|
271
|
+
self._settings = settings
|
|
272
|
+
self._cipher: Cipher = cipher or IdentityCipher()
|
|
273
|
+
self.path = f"{settings.server}/{settings.database}" # descriptor for db_status
|
|
274
|
+
# ADR 0005 transform-state read-through cache (parity with SQLite/PG): loaded at open, updated
|
|
275
|
+
# post-commit by transform_handoff, surfaced via state_view() so a Handler's cross-message
|
|
276
|
+
# state_get(...) resolves in-process.
|
|
277
|
+
self._state_cache: dict[tuple[str, str], Any] = {}
|
|
278
|
+
# Serializes audit-chain appends in-process (the store is the single audit writer per engine
|
|
279
|
+
# process; active-passive = one active node) — see record_audit.
|
|
280
|
+
self._audit_lock = asyncio.Lock()
|
|
281
|
+
|
|
282
|
+
@classmethod
|
|
283
|
+
async def open(
|
|
284
|
+
cls, settings: StoreSettings, *, cipher: Cipher | None = None
|
|
285
|
+
) -> "SqlServerStore":
|
|
286
|
+
try:
|
|
287
|
+
import aioodbc
|
|
288
|
+
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
289
|
+
raise RuntimeError(
|
|
290
|
+
"SQL Server backend requires the 'sqlserver' extra: "
|
|
291
|
+
"pip install 'messagefoundry[sqlserver]' (plus the Microsoft ODBC Driver 18)"
|
|
292
|
+
) from exc
|
|
293
|
+
# RCSI must be enabled BEFORE the pool exists: its one-time ALTER ... WITH ROLLBACK IMMEDIATE
|
|
294
|
+
# takes momentary exclusivity, and with no MEFOR pool session open yet it has nothing of ours
|
|
295
|
+
# to terminate (concurrency_fixes (a)).
|
|
296
|
+
await cls._ensure_database_options(settings)
|
|
297
|
+
pool = await aioodbc.create_pool(
|
|
298
|
+
dsn=connection_string(settings),
|
|
299
|
+
minsize=1,
|
|
300
|
+
maxsize=max(1, settings.pool_size),
|
|
301
|
+
autocommit=False,
|
|
302
|
+
)
|
|
303
|
+
store = cls(pool, settings, cipher=cipher)
|
|
304
|
+
try:
|
|
305
|
+
await store._ensure_schema()
|
|
306
|
+
await store._encrypt_existing_rows() # one-time PHI-at-rest migration when a key is set
|
|
307
|
+
await store._backfill_audit_chain() # chain any pre-existing (unhashed) audit rows
|
|
308
|
+
await store._load_state_cache() # ADR 0005 read-through cache warm-up
|
|
309
|
+
except Exception:
|
|
310
|
+
# Don't leak the pool if first-open initialization fails (M-6).
|
|
311
|
+
pool.close()
|
|
312
|
+
await pool.wait_closed()
|
|
313
|
+
raise
|
|
314
|
+
return store
|
|
315
|
+
|
|
316
|
+
async def _backfill_audit_chain(self) -> None:
|
|
317
|
+
"""Fill ``row_hash`` for audit rows written before hash-chaining (idempotent; fills only
|
|
318
|
+
NULLs, chained from the prior row)."""
|
|
319
|
+
rows = await self._fetchall(
|
|
320
|
+
"SELECT id, ts, actor, action, channel_id, detail, row_hash FROM audit_log ORDER BY id"
|
|
321
|
+
)
|
|
322
|
+
prev = ""
|
|
323
|
+
updates: list[tuple[str, int]] = []
|
|
324
|
+
for r in rows:
|
|
325
|
+
if r["row_hash"]:
|
|
326
|
+
prev = r["row_hash"]
|
|
327
|
+
continue
|
|
328
|
+
prev = audit_row_hash(
|
|
329
|
+
prev,
|
|
330
|
+
ts=r["ts"],
|
|
331
|
+
actor=r["actor"],
|
|
332
|
+
action=r["action"],
|
|
333
|
+
channel_id=r["channel_id"],
|
|
334
|
+
detail=r["detail"],
|
|
335
|
+
)
|
|
336
|
+
updates.append((prev, r["id"]))
|
|
337
|
+
if updates:
|
|
338
|
+
# Runs in open() before the store is returned, so no concurrent record_audit can race it.
|
|
339
|
+
async with self._acquire() as conn:
|
|
340
|
+
cur = await conn.cursor()
|
|
341
|
+
try:
|
|
342
|
+
for row_hash, rid in updates:
|
|
343
|
+
await cur.execute(
|
|
344
|
+
"UPDATE audit_log SET row_hash=? WHERE id=?", (row_hash, rid)
|
|
345
|
+
)
|
|
346
|
+
await conn.commit()
|
|
347
|
+
except Exception:
|
|
348
|
+
await conn.rollback()
|
|
349
|
+
raise
|
|
350
|
+
|
|
351
|
+
async def _encrypt_existing_rows(self) -> None:
|
|
352
|
+
"""Re-encrypt legacy plaintext bodies in place when encryption is enabled (STORE-1).
|
|
353
|
+
|
|
354
|
+
Idempotent + batched: skips rows already carrying the ciphertext prefix."""
|
|
355
|
+
if not self._cipher.encrypts:
|
|
356
|
+
return
|
|
357
|
+
like = f"{_ENC_PREFIX}%"
|
|
358
|
+
total = 0
|
|
359
|
+
for table, column in (
|
|
360
|
+
("messages", "raw"),
|
|
361
|
+
("queue", "payload"),
|
|
362
|
+
("outbox", "payload"),
|
|
363
|
+
(
|
|
364
|
+
"users",
|
|
365
|
+
"totp_secret",
|
|
366
|
+
), # MFA secret (WP-14): id-keyed, NULL rows excluded by NOT LIKE
|
|
367
|
+
):
|
|
368
|
+
while True:
|
|
369
|
+
rows = await self._fetchall(
|
|
370
|
+
f"SELECT TOP (500) id, {column} FROM {table} WHERE {column} NOT LIKE ?", (like,)
|
|
371
|
+
)
|
|
372
|
+
if not rows:
|
|
373
|
+
break
|
|
374
|
+
async with self._acquire() as conn:
|
|
375
|
+
cur = await conn.cursor()
|
|
376
|
+
try:
|
|
377
|
+
for r in rows:
|
|
378
|
+
await cur.execute(
|
|
379
|
+
f"UPDATE {table} SET {column}=? WHERE id=?",
|
|
380
|
+
(self._cipher.encrypt(r[column]), r["id"]),
|
|
381
|
+
)
|
|
382
|
+
await conn.commit()
|
|
383
|
+
except Exception:
|
|
384
|
+
await conn.rollback()
|
|
385
|
+
raise
|
|
386
|
+
total += len(rows)
|
|
387
|
+
# `response` body + detail (composite PK) — a separate pass (can't ride the id-keyed loop above).
|
|
388
|
+
# PG/SQLite migrate these too; without it a no-key -> key -> restart leaves captured reply PHI as
|
|
389
|
+
# plaintext at rest. body/detail are nullable, so guard `<> '' AND IS NOT NULL`.
|
|
390
|
+
for rcol in ("body", "detail"):
|
|
391
|
+
while True:
|
|
392
|
+
rows = await self._fetchall(
|
|
393
|
+
f"SELECT TOP (500) message_id, destination_name, response_seq, {rcol} AS v"
|
|
394
|
+
f" FROM response WHERE {rcol} NOT LIKE ? AND {rcol} <> '' AND {rcol} IS NOT NULL",
|
|
395
|
+
(like,),
|
|
396
|
+
)
|
|
397
|
+
if not rows:
|
|
398
|
+
break
|
|
399
|
+
async with self._acquire() as conn:
|
|
400
|
+
cur = await conn.cursor()
|
|
401
|
+
try:
|
|
402
|
+
for r in rows:
|
|
403
|
+
await cur.execute(
|
|
404
|
+
f"UPDATE response SET {rcol}=?"
|
|
405
|
+
" WHERE message_id=? AND destination_name=? AND response_seq=?",
|
|
406
|
+
(
|
|
407
|
+
self._cipher.encrypt(r["v"]),
|
|
408
|
+
r["message_id"],
|
|
409
|
+
r["destination_name"],
|
|
410
|
+
r["response_seq"],
|
|
411
|
+
),
|
|
412
|
+
)
|
|
413
|
+
await conn.commit()
|
|
414
|
+
except Exception:
|
|
415
|
+
await conn.rollback()
|
|
416
|
+
raise
|
|
417
|
+
total += len(rows)
|
|
418
|
+
if total:
|
|
419
|
+
log.info("encrypted %d existing message/outbox/response row(s) at rest", total)
|
|
420
|
+
|
|
421
|
+
@staticmethod
|
|
422
|
+
async def _ensure_database_options(settings: StoreSettings) -> None:
|
|
423
|
+
"""Enable READ_COMMITTED_SNAPSHOT (RCSI) so the staged claim/finalize paths read on a
|
|
424
|
+
row-version snapshot rather than taking shared locks that deadlock writers under concurrent
|
|
425
|
+
load (concurrency_fixes (a)). Runs on its OWN autocommit connection BEFORE the pool is
|
|
426
|
+
created, so the momentary exclusivity of ``WITH ROLLBACK IMMEDIATE`` has no sibling MEFOR
|
|
427
|
+
session to terminate; IF-guarded on the live state, so the disruptive ALTER fires at most ONCE
|
|
428
|
+
(greenfield first boot) and every later open()/failover is a detect-and-skip no-op. Degrades
|
|
429
|
+
to a warning (never fails open()) when the principal lacks ALTER DATABASE or the lock cannot
|
|
430
|
+
be taken — emitting the exact statement for a DBA to run out-of-band."""
|
|
431
|
+
import aioodbc
|
|
432
|
+
|
|
433
|
+
db = settings.database
|
|
434
|
+
try:
|
|
435
|
+
conn = await aioodbc.connect(dsn=connection_string(settings), autocommit=True)
|
|
436
|
+
except Exception as exc: # noqa: BLE001 - the pool open below surfaces a real connect failure
|
|
437
|
+
log.warning("skipping the RCSI check on %r (could not connect): %s", db, exc)
|
|
438
|
+
return
|
|
439
|
+
try:
|
|
440
|
+
cur = await conn.cursor()
|
|
441
|
+
await cur.execute(
|
|
442
|
+
"SELECT is_read_committed_snapshot_on, snapshot_isolation_state "
|
|
443
|
+
"FROM sys.databases WHERE name = DB_NAME()"
|
|
444
|
+
)
|
|
445
|
+
row = await cur.fetchone()
|
|
446
|
+
# If we cannot read the state, do NOT attempt a disruptive ALTER.
|
|
447
|
+
rcsi_on = bool(row[0]) if row else True
|
|
448
|
+
snapshot_on = (row[1] in (1, 2)) if row else True
|
|
449
|
+
if not rcsi_on:
|
|
450
|
+
try:
|
|
451
|
+
await cur.execute(
|
|
452
|
+
"ALTER DATABASE CURRENT SET READ_COMMITTED_SNAPSHOT ON WITH ROLLBACK IMMEDIATE"
|
|
453
|
+
)
|
|
454
|
+
log.info("enabled READ_COMMITTED_SNAPSHOT on database %r", db)
|
|
455
|
+
except Exception as exc: # noqa: BLE001 - permission/lock: degrade to a DBA pointer
|
|
456
|
+
log.warning(
|
|
457
|
+
"could not enable READ_COMMITTED_SNAPSHOT on %r (%s); a DBA should run once: "
|
|
458
|
+
"ALTER DATABASE [%s] SET READ_COMMITTED_SNAPSHOT ON WITH ROLLBACK IMMEDIATE — "
|
|
459
|
+
"without it the staged claim/finalize paths are more deadlock-prone under load",
|
|
460
|
+
db,
|
|
461
|
+
exc,
|
|
462
|
+
db,
|
|
463
|
+
)
|
|
464
|
+
if not snapshot_on:
|
|
465
|
+
try:
|
|
466
|
+
# ALLOW_SNAPSHOT_ISOLATION is an online change (no exclusivity required).
|
|
467
|
+
await cur.execute("ALTER DATABASE CURRENT SET ALLOW_SNAPSHOT_ISOLATION ON")
|
|
468
|
+
except Exception as exc: # noqa: BLE001 - non-fatal
|
|
469
|
+
log.warning("could not enable ALLOW_SNAPSHOT_ISOLATION on %r: %s", db, exc)
|
|
470
|
+
finally:
|
|
471
|
+
await conn.close()
|
|
472
|
+
|
|
473
|
+
async def _ensure_schema(self) -> None:
|
|
474
|
+
async with self._acquire() as conn:
|
|
475
|
+
cur = await conn.cursor()
|
|
476
|
+
try:
|
|
477
|
+
for statement in _SCHEMA:
|
|
478
|
+
await cur.execute(statement)
|
|
479
|
+
await conn.commit()
|
|
480
|
+
except Exception:
|
|
481
|
+
await conn.rollback() # roll back the partial DDL batch (M-6)
|
|
482
|
+
raise
|
|
483
|
+
|
|
484
|
+
async def close(self) -> None:
|
|
485
|
+
self._pool.close()
|
|
486
|
+
await self._pool.wait_closed()
|
|
487
|
+
|
|
488
|
+
# --- helpers -------------------------------------------------------------
|
|
489
|
+
|
|
490
|
+
@asynccontextmanager
|
|
491
|
+
async def _acquire(self) -> AsyncIterator[Any]:
|
|
492
|
+
"""Acquire a pooled connection with the configured command (statement) timeout applied.
|
|
493
|
+
|
|
494
|
+
``Connection Timeout`` in the DSN is only the *login* timeout; the per-statement timeout is a
|
|
495
|
+
pyodbc **connection** attribute (STORE-3). aioodbc's wrapper exposes ``timeout`` read-only, so
|
|
496
|
+
we set it on the underlying ``pyodbc.Connection`` (``_conn``); aioodbc 0.5.0 has no creation
|
|
497
|
+
hook (``after_created``), so we apply it per-acquire (an idempotent int assignment). The prior
|
|
498
|
+
``conn.timeout = ...`` raised AttributeError and was silently swallowed, so no statement
|
|
499
|
+
timeout was ever applied — a hung statement then held its queue/messages row X-locks forever."""
|
|
500
|
+
async with self._pool.acquire() as conn:
|
|
501
|
+
raw = getattr(conn, "_conn", None)
|
|
502
|
+
if raw is not None:
|
|
503
|
+
raw.timeout = self._settings.command_timeout # seconds; 0 = no limit
|
|
504
|
+
yield conn
|
|
505
|
+
|
|
506
|
+
async def _fetchall(self, sql: str, params: tuple[Any, ...] = ()) -> list[dict[str, Any]]:
|
|
507
|
+
async with self._acquire() as conn:
|
|
508
|
+
cur = await conn.cursor()
|
|
509
|
+
try:
|
|
510
|
+
await cur.execute(sql, params) if params else await cur.execute(sql)
|
|
511
|
+
columns = [c[0] for c in cur.description]
|
|
512
|
+
rows = await cur.fetchall()
|
|
513
|
+
await conn.commit()
|
|
514
|
+
except Exception:
|
|
515
|
+
# autocommit=False: a failed read otherwise leaves the pooled connection mid-txn (and
|
|
516
|
+
# under RCSI an open snapshot pins the version store / bloats tempdb). Roll back before
|
|
517
|
+
# it returns to the pool so the next borrower starts clean (M-6).
|
|
518
|
+
await conn.rollback()
|
|
519
|
+
raise
|
|
520
|
+
return [dict(zip(columns, row)) for row in rows]
|
|
521
|
+
|
|
522
|
+
async def _fetchone(self, sql: str, params: tuple[Any, ...] = ()) -> dict[str, Any] | None:
|
|
523
|
+
rows = await self._fetchall(sql, params)
|
|
524
|
+
return rows[0] if rows else None
|
|
525
|
+
|
|
526
|
+
async def _execute(self, sql: str, params: tuple[Any, ...] = ()) -> None:
|
|
527
|
+
"""Run a single write statement (or T-SQL batch) in its own committed transaction."""
|
|
528
|
+
async with self._acquire() as conn:
|
|
529
|
+
cur = await conn.cursor()
|
|
530
|
+
try:
|
|
531
|
+
await cur.execute(sql, params)
|
|
532
|
+
await conn.commit()
|
|
533
|
+
except Exception:
|
|
534
|
+
await conn.rollback()
|
|
535
|
+
raise
|
|
536
|
+
|
|
537
|
+
@staticmethod
|
|
538
|
+
async def _event(
|
|
539
|
+
cur: Any,
|
|
540
|
+
message_id: str,
|
|
541
|
+
event: str,
|
|
542
|
+
destination: str | None,
|
|
543
|
+
detail: str | None,
|
|
544
|
+
now: float,
|
|
545
|
+
) -> None:
|
|
546
|
+
detail = (
|
|
547
|
+
safe_text(detail) if detail else detail
|
|
548
|
+
) # PHI chokepoint (#120); SQL Server stores plaintext
|
|
549
|
+
await cur.execute(
|
|
550
|
+
"INSERT INTO message_events (message_id, ts, event, destination, detail)"
|
|
551
|
+
" VALUES (?,?,?,?,?)",
|
|
552
|
+
(message_id, now, event, destination, detail),
|
|
553
|
+
)
|
|
554
|
+
|
|
555
|
+
async def _applock(self, cur: Any, resource: str) -> None:
|
|
556
|
+
"""Take a transaction-scoped exclusive ``sp_getapplock`` — the T-SQL analog of PG's advisory
|
|
557
|
+
lock. A NAMED lock in its own space: it never locks ``queue``/``messages`` rows, so it cannot
|
|
558
|
+
invert the producers' queue->messages lock order (no AB/BA deadlock), and it is re-entrant per
|
|
559
|
+
(resource, transaction). ``@LockOwner='Transaction'`` auto-releases it at the caller's commit/
|
|
560
|
+
rollback, so the caller MUST be in an open (autocommit=False) transaction. Raises on a negative
|
|
561
|
+
return code (timeout/deadlock/error) rather than proceeding unserialized — a swallowed timeout
|
|
562
|
+
would fork the audit chain or double-finalize a message."""
|
|
563
|
+
ct = self._settings.command_timeout
|
|
564
|
+
timeout_ms = (
|
|
565
|
+
int(ct * 1000) if ct else -1
|
|
566
|
+
) # ms; -1 = wait (the pyodbc query timeout backstops)
|
|
567
|
+
await cur.execute(
|
|
568
|
+
"SET NOCOUNT ON;"
|
|
569
|
+
" DECLARE @rc INT;"
|
|
570
|
+
" EXEC @rc = sp_getapplock @Resource=?, @LockMode='Exclusive',"
|
|
571
|
+
" @LockOwner='Transaction', @LockTimeout=?;"
|
|
572
|
+
" SELECT @rc",
|
|
573
|
+
(resource, timeout_ms),
|
|
574
|
+
)
|
|
575
|
+
row = await cur.fetchone()
|
|
576
|
+
rc = int(row[0]) if row and row[0] is not None else -999
|
|
577
|
+
if rc < 0: # -1 timeout, -2 cancelled, -3 deadlock victim, -999 bad param
|
|
578
|
+
raise RuntimeError(f"sp_getapplock({resource!r}) failed: rc={rc}")
|
|
579
|
+
|
|
580
|
+
async def _lock_finalize_batch(self, cur: Any, message_ids: Iterable[str]) -> None:
|
|
581
|
+
"""Pre-acquire the per-message finalize applock for every id in CANONICAL sorted order, so a
|
|
582
|
+
multi-message finalizer (an orphan sweep / cancel_queued) can never deadlock another by taking
|
|
583
|
+
the per-id locks in a different order. Re-entrant: a later ``_maybe_finalize`` re-take of the
|
|
584
|
+
same (resource, transaction) is a no-op."""
|
|
585
|
+
for mid in sorted(set(message_ids)):
|
|
586
|
+
await self._applock(cur, f"mefor:finalize:{mid}")
|
|
587
|
+
|
|
588
|
+
async def _fifo_created_at(
|
|
589
|
+
self, cur: Any, stage: str, lane_col: str, lane_val: str, now: float
|
|
590
|
+
) -> float:
|
|
591
|
+
"""The ``created_at`` to stamp on a new ``stage`` row so per-lane FIFO order stays monotonic
|
|
592
|
+
even if the wall clock regresses: ``max(now, the lane's current max created_at)``. ``lane_col``
|
|
593
|
+
is a code-controlled literal (allow-listed below — never user input). One grouped MAX per lane
|
|
594
|
+
bounds the lock-hold window under high fan-out; under RCSI it reads the committed snapshot
|
|
595
|
+
without blocking writers."""
|
|
596
|
+
if lane_col not in (
|
|
597
|
+
"channel_id",
|
|
598
|
+
"destination_name",
|
|
599
|
+
): # injection guard (survives python -O)
|
|
600
|
+
raise ValueError(f"invalid lane column: {lane_col!r}")
|
|
601
|
+
await cur.execute(
|
|
602
|
+
f"SELECT MAX(created_at) FROM queue WHERE stage=? AND {lane_col}=?", (stage, lane_val)
|
|
603
|
+
)
|
|
604
|
+
row = await cur.fetchone()
|
|
605
|
+
prior = row[0] if row and row[0] is not None else None
|
|
606
|
+
return max(now, prior) if prior is not None else now
|
|
607
|
+
|
|
608
|
+
async def _maybe_finalize(self, cur: Any, message_id: str, now: float) -> None:
|
|
609
|
+
"""Recompute and persist a message's terminal disposition — the SOLE authority for it. Scans
|
|
610
|
+
ALL stages of ``queue`` so a delivered handler can't finalize the message while a sibling
|
|
611
|
+
handler's routed row is still in flight. Serialized per-message on the finalize applock so two
|
|
612
|
+
concurrent finalizers (a delivery + a transform handoff) can't lost-update ``messages.status``.
|
|
613
|
+
Precedence: any pending/inflight at any stage -> still moving (return); else any dead -> ERROR;
|
|
614
|
+
else any outbound row -> PROCESSED; else no rows + messages.status='routed' -> FILTERED (every
|
|
615
|
+
handler ran, delivered nothing); else leave (UNROUTED/ERROR/in-progress not clobbered)."""
|
|
616
|
+
# A per-message NAMED lock — NOT a messages-row lock, which would invert the queue->messages
|
|
617
|
+
# lock order the producers take and deadlock (error 1205). Re-entrant; released at commit.
|
|
618
|
+
await self._applock(cur, f"mefor:finalize:{message_id}")
|
|
619
|
+
await cur.execute(
|
|
620
|
+
"SELECT stage, status, COUNT(*) AS n FROM queue WHERE message_id=? GROUP BY stage, status",
|
|
621
|
+
(message_id,),
|
|
622
|
+
)
|
|
623
|
+
rows = await cur.fetchall()
|
|
624
|
+
statuses = {r[1] for r in rows}
|
|
625
|
+
if OutboxStatus.PENDING.value in statuses or OutboxStatus.INFLIGHT.value in statuses:
|
|
626
|
+
return # still moving through a stage
|
|
627
|
+
if OutboxStatus.DEAD.value in statuses:
|
|
628
|
+
status = MessageStatus.ERROR.value
|
|
629
|
+
elif any(r[0] == Stage.OUTBOUND.value for r in rows):
|
|
630
|
+
status = MessageStatus.PROCESSED.value
|
|
631
|
+
elif not rows:
|
|
632
|
+
# No queue rows remain: the router/handlers produced no delivery. FILTERED only if it was
|
|
633
|
+
# actually routed; never clobber UNROUTED / ERROR / a status already set terminal.
|
|
634
|
+
await cur.execute("SELECT status FROM messages WHERE id=?", (message_id,))
|
|
635
|
+
mrow = await cur.fetchone()
|
|
636
|
+
if not mrow or mrow[0] != MessageStatus.ROUTED.value:
|
|
637
|
+
return
|
|
638
|
+
status = MessageStatus.FILTERED.value
|
|
639
|
+
else:
|
|
640
|
+
return # rows exist but all terminal, non-dead, non-outbound — leave as-is (rare)
|
|
641
|
+
await cur.execute("UPDATE messages SET status=? WHERE id=?", (status, message_id))
|
|
642
|
+
|
|
643
|
+
@staticmethod
|
|
644
|
+
def _message_filter(
|
|
645
|
+
channel_id: str | None,
|
|
646
|
+
status: str | None,
|
|
647
|
+
message_type: str | None,
|
|
648
|
+
control_id: str | None,
|
|
649
|
+
allowed_channels: Sequence[str] | None = None,
|
|
650
|
+
) -> tuple[str, tuple[Any, ...]]:
|
|
651
|
+
clauses: list[str] = []
|
|
652
|
+
params: list[Any] = []
|
|
653
|
+
for column, value in (
|
|
654
|
+
("channel_id", channel_id),
|
|
655
|
+
("status", status),
|
|
656
|
+
("message_type", message_type),
|
|
657
|
+
("control_id", control_id),
|
|
658
|
+
):
|
|
659
|
+
if value is not None:
|
|
660
|
+
clauses.append(f"{column}=?")
|
|
661
|
+
params.append(value)
|
|
662
|
+
_append_channel_scope(clauses, params, "channel_id", allowed_channels)
|
|
663
|
+
where = f" WHERE {' AND '.join(clauses)}" if clauses else ""
|
|
664
|
+
return where, tuple(params)
|
|
665
|
+
|
|
666
|
+
# --- write path ----------------------------------------------------------
|
|
667
|
+
|
|
668
|
+
async def enqueue_message(
|
|
669
|
+
self,
|
|
670
|
+
*,
|
|
671
|
+
channel_id: str,
|
|
672
|
+
raw: str,
|
|
673
|
+
deliveries: Sequence[tuple[str, str]],
|
|
674
|
+
control_id: str | None = None,
|
|
675
|
+
message_type: str | None = None,
|
|
676
|
+
source_type: str | None = None,
|
|
677
|
+
summary: str | None = None,
|
|
678
|
+
metadata: str | None = None,
|
|
679
|
+
now: float | None = None,
|
|
680
|
+
) -> str:
|
|
681
|
+
now = time.time() if now is None else now
|
|
682
|
+
mid = uuid4().hex
|
|
683
|
+
status = MessageStatus.RECEIVED.value if deliveries else MessageStatus.UNROUTED.value
|
|
684
|
+
async with self._acquire() as conn:
|
|
685
|
+
cur = await conn.cursor()
|
|
686
|
+
try:
|
|
687
|
+
await cur.execute(
|
|
688
|
+
"INSERT INTO messages (id, channel_id, received_at, source_type, control_id,"
|
|
689
|
+
" message_type, raw, status, error, summary, metadata)"
|
|
690
|
+
" VALUES (?,?,?,?,?,?,?,?,?,?,?)",
|
|
691
|
+
(
|
|
692
|
+
mid,
|
|
693
|
+
channel_id,
|
|
694
|
+
now,
|
|
695
|
+
source_type,
|
|
696
|
+
control_id,
|
|
697
|
+
message_type,
|
|
698
|
+
self._cipher.encrypt(raw),
|
|
699
|
+
status,
|
|
700
|
+
None,
|
|
701
|
+
summary,
|
|
702
|
+
metadata,
|
|
703
|
+
),
|
|
704
|
+
)
|
|
705
|
+
for dest_name, payload in deliveries:
|
|
706
|
+
await cur.execute(
|
|
707
|
+
"INSERT INTO queue (id, message_id, stage, channel_id, destination_name,"
|
|
708
|
+
" handler_name, payload, status, attempts, next_attempt_at, owner,"
|
|
709
|
+
" lease_expires_at, created_at, updated_at)"
|
|
710
|
+
" VALUES (?,?,?,?,?,NULL,?,?,0,?,NULL,NULL,?,?)",
|
|
711
|
+
(
|
|
712
|
+
uuid4().hex,
|
|
713
|
+
mid,
|
|
714
|
+
Stage.OUTBOUND.value,
|
|
715
|
+
channel_id,
|
|
716
|
+
dest_name,
|
|
717
|
+
self._cipher.encrypt(payload),
|
|
718
|
+
OutboxStatus.PENDING.value,
|
|
719
|
+
now,
|
|
720
|
+
now,
|
|
721
|
+
now,
|
|
722
|
+
),
|
|
723
|
+
)
|
|
724
|
+
await self._event(
|
|
725
|
+
cur, mid, "received", None, f"{len(deliveries)} destination(s)", now
|
|
726
|
+
)
|
|
727
|
+
await conn.commit()
|
|
728
|
+
except Exception:
|
|
729
|
+
await conn.rollback()
|
|
730
|
+
raise
|
|
731
|
+
return mid
|
|
732
|
+
|
|
733
|
+
async def _insert_outbound(
|
|
734
|
+
self, cur: Any, message_id: str, channel_id: str, dest_name: str, payload: str, now: float
|
|
735
|
+
) -> None:
|
|
736
|
+
"""Insert one ``stage='outbound'`` queue row (lane = destination_name), FIFO-clamped."""
|
|
737
|
+
created_at = await self._fifo_created_at(
|
|
738
|
+
cur, Stage.OUTBOUND.value, "destination_name", dest_name, now
|
|
739
|
+
)
|
|
740
|
+
await cur.execute(
|
|
741
|
+
"INSERT INTO queue (id, message_id, stage, channel_id, destination_name, handler_name,"
|
|
742
|
+
" payload, status, attempts, next_attempt_at, owner, lease_expires_at, created_at,"
|
|
743
|
+
" updated_at) VALUES (?,?,?,?,?,NULL,?,?,0,?,NULL,NULL,?,?)",
|
|
744
|
+
(
|
|
745
|
+
uuid4().hex,
|
|
746
|
+
message_id,
|
|
747
|
+
Stage.OUTBOUND.value,
|
|
748
|
+
channel_id,
|
|
749
|
+
dest_name,
|
|
750
|
+
self._cipher.encrypt(payload),
|
|
751
|
+
OutboxStatus.PENDING.value,
|
|
752
|
+
now,
|
|
753
|
+
created_at,
|
|
754
|
+
now,
|
|
755
|
+
),
|
|
756
|
+
)
|
|
757
|
+
|
|
758
|
+
async def _insert_routed(
|
|
759
|
+
self,
|
|
760
|
+
cur: Any,
|
|
761
|
+
message_id: str,
|
|
762
|
+
channel_id: str,
|
|
763
|
+
handler_name: str,
|
|
764
|
+
payload: str,
|
|
765
|
+
now: float,
|
|
766
|
+
) -> None:
|
|
767
|
+
"""Insert one ``stage='routed'`` queue row (lane = channel_id), FIFO-clamped."""
|
|
768
|
+
created_at = await self._fifo_created_at(
|
|
769
|
+
cur, Stage.ROUTED.value, "channel_id", channel_id, now
|
|
770
|
+
)
|
|
771
|
+
await cur.execute(
|
|
772
|
+
"INSERT INTO queue (id, message_id, stage, channel_id, destination_name, handler_name,"
|
|
773
|
+
" payload, status, attempts, next_attempt_at, owner, lease_expires_at, created_at,"
|
|
774
|
+
" updated_at) VALUES (?,?,?,?,NULL,?,?,?,0,?,NULL,NULL,?,?)",
|
|
775
|
+
(
|
|
776
|
+
uuid4().hex,
|
|
777
|
+
message_id,
|
|
778
|
+
Stage.ROUTED.value,
|
|
779
|
+
channel_id,
|
|
780
|
+
handler_name,
|
|
781
|
+
self._cipher.encrypt(payload),
|
|
782
|
+
OutboxStatus.PENDING.value,
|
|
783
|
+
now,
|
|
784
|
+
created_at,
|
|
785
|
+
now,
|
|
786
|
+
),
|
|
787
|
+
)
|
|
788
|
+
|
|
789
|
+
async def enqueue_ingress(
|
|
790
|
+
self,
|
|
791
|
+
*,
|
|
792
|
+
channel_id: str,
|
|
793
|
+
raw: str,
|
|
794
|
+
control_id: str | None = None,
|
|
795
|
+
message_type: str | None = None,
|
|
796
|
+
source_type: str | None = None,
|
|
797
|
+
summary: str | None = None,
|
|
798
|
+
metadata: str | None = None,
|
|
799
|
+
now: float | None = None,
|
|
800
|
+
) -> str:
|
|
801
|
+
"""Durably persist a freshly-received raw message to the ingress stage (status RECEIVED + one
|
|
802
|
+
``stage='ingress'`` queue row holding the raw) in ONE transaction — the staged pipeline's
|
|
803
|
+
ACK-on-receipt boundary (ADR 0001). The inbound may be ACKed once this returns. Returns the
|
|
804
|
+
message id."""
|
|
805
|
+
now = time.time() if now is None else now
|
|
806
|
+
mid = uuid4().hex
|
|
807
|
+
async with self._acquire() as conn:
|
|
808
|
+
cur = await conn.cursor()
|
|
809
|
+
try:
|
|
810
|
+
await cur.execute(
|
|
811
|
+
"INSERT INTO messages (id, channel_id, received_at, source_type, control_id,"
|
|
812
|
+
" message_type, raw, status, error, summary, metadata)"
|
|
813
|
+
" VALUES (?,?,?,?,?,?,?,?,?,?,?)",
|
|
814
|
+
(
|
|
815
|
+
mid,
|
|
816
|
+
channel_id,
|
|
817
|
+
now,
|
|
818
|
+
source_type,
|
|
819
|
+
control_id,
|
|
820
|
+
message_type,
|
|
821
|
+
self._cipher.encrypt(raw),
|
|
822
|
+
MessageStatus.RECEIVED.value,
|
|
823
|
+
None,
|
|
824
|
+
summary,
|
|
825
|
+
metadata,
|
|
826
|
+
),
|
|
827
|
+
)
|
|
828
|
+
created_at = await self._fifo_created_at(
|
|
829
|
+
cur, Stage.INGRESS.value, "channel_id", channel_id, now
|
|
830
|
+
)
|
|
831
|
+
await cur.execute(
|
|
832
|
+
"INSERT INTO queue (id, message_id, stage, channel_id, destination_name,"
|
|
833
|
+
" handler_name, payload, status, attempts, next_attempt_at, owner,"
|
|
834
|
+
" lease_expires_at, created_at, updated_at)"
|
|
835
|
+
" VALUES (?,?,?,?,NULL,NULL,?,?,0,?,NULL,NULL,?,?)",
|
|
836
|
+
(
|
|
837
|
+
uuid4().hex,
|
|
838
|
+
mid,
|
|
839
|
+
Stage.INGRESS.value,
|
|
840
|
+
channel_id,
|
|
841
|
+
self._cipher.encrypt(raw),
|
|
842
|
+
OutboxStatus.PENDING.value,
|
|
843
|
+
now,
|
|
844
|
+
created_at,
|
|
845
|
+
now,
|
|
846
|
+
),
|
|
847
|
+
)
|
|
848
|
+
await self._event(cur, mid, "received", None, "ingress", now)
|
|
849
|
+
await conn.commit()
|
|
850
|
+
except Exception:
|
|
851
|
+
await conn.rollback()
|
|
852
|
+
raise
|
|
853
|
+
return mid
|
|
854
|
+
|
|
855
|
+
async def handoff(
|
|
856
|
+
self,
|
|
857
|
+
*,
|
|
858
|
+
ingress_id: str,
|
|
859
|
+
message_id: str,
|
|
860
|
+
channel_id: str,
|
|
861
|
+
deliveries: Sequence[tuple[str, str]],
|
|
862
|
+
disposition: MessageStatus,
|
|
863
|
+
now: float | None = None,
|
|
864
|
+
) -> bool:
|
|
865
|
+
"""Advance a message from ingress straight to outbound in ONE transaction (the Step-A combined
|
|
866
|
+
primitive): consume the in-flight ingress row, insert one outbound row per delivery, set the
|
|
867
|
+
post-router disposition under the finalize applock. Idempotent: returns False (no-op) if the
|
|
868
|
+
ingress row was already consumed by a committed prior run."""
|
|
869
|
+
now = time.time() if now is None else now
|
|
870
|
+
async with self._acquire() as conn:
|
|
871
|
+
cur = await conn.cursor()
|
|
872
|
+
try:
|
|
873
|
+
await cur.execute(
|
|
874
|
+
"DELETE FROM queue OUTPUT deleted.id WHERE id=? AND stage=? AND status=?",
|
|
875
|
+
(ingress_id, Stage.INGRESS.value, OutboxStatus.INFLIGHT.value),
|
|
876
|
+
)
|
|
877
|
+
if await cur.fetchone() is None:
|
|
878
|
+
await conn.rollback()
|
|
879
|
+
return False # already handed off (crash-restart) — idempotent no-op
|
|
880
|
+
for dest_name, payload in deliveries:
|
|
881
|
+
await self._insert_outbound(
|
|
882
|
+
cur, message_id, channel_id, dest_name, payload, now
|
|
883
|
+
)
|
|
884
|
+
await self._applock(cur, f"mefor:finalize:{message_id}")
|
|
885
|
+
await cur.execute(
|
|
886
|
+
"UPDATE messages SET status=? WHERE id=?", (disposition.value, message_id)
|
|
887
|
+
)
|
|
888
|
+
event = {
|
|
889
|
+
MessageStatus.ROUTED: "routed",
|
|
890
|
+
MessageStatus.FILTERED: "filtered",
|
|
891
|
+
MessageStatus.UNROUTED: "unrouted",
|
|
892
|
+
}.get(disposition, "routed")
|
|
893
|
+
await self._event(
|
|
894
|
+
cur, message_id, event, None, f"{len(deliveries)} destination(s)", now
|
|
895
|
+
)
|
|
896
|
+
await conn.commit()
|
|
897
|
+
except Exception:
|
|
898
|
+
await conn.rollback()
|
|
899
|
+
raise
|
|
900
|
+
return True
|
|
901
|
+
|
|
902
|
+
async def route_handoff(
|
|
903
|
+
self,
|
|
904
|
+
*,
|
|
905
|
+
ingress_id: str,
|
|
906
|
+
message_id: str,
|
|
907
|
+
channel_id: str,
|
|
908
|
+
handlers: Sequence[tuple[str, str]],
|
|
909
|
+
disposition: MessageStatus,
|
|
910
|
+
now: float | None = None,
|
|
911
|
+
) -> bool:
|
|
912
|
+
"""Advance a message from the ingress stage to the routed stage in ONE transaction (the router
|
|
913
|
+
half of the split pipeline): consume the in-flight ingress row, insert one ``stage='routed'``
|
|
914
|
+
row per selected handler (handler-list order; ``seq`` preserves it), set the intermediate
|
|
915
|
+
disposition (ROUTED with handlers, UNROUTED with none) under the finalize applock. Idempotent:
|
|
916
|
+
False if the ingress row was already consumed."""
|
|
917
|
+
now = time.time() if now is None else now
|
|
918
|
+
async with self._acquire() as conn:
|
|
919
|
+
cur = await conn.cursor()
|
|
920
|
+
try:
|
|
921
|
+
await cur.execute(
|
|
922
|
+
"DELETE FROM queue OUTPUT deleted.id WHERE id=? AND stage=? AND status=?",
|
|
923
|
+
(ingress_id, Stage.INGRESS.value, OutboxStatus.INFLIGHT.value),
|
|
924
|
+
)
|
|
925
|
+
if await cur.fetchone() is None:
|
|
926
|
+
await conn.rollback()
|
|
927
|
+
return False # already handed off (crash-restart) — idempotent no-op
|
|
928
|
+
for handler_name, payload in handlers:
|
|
929
|
+
await self._insert_routed(
|
|
930
|
+
cur, message_id, channel_id, handler_name, payload, now
|
|
931
|
+
)
|
|
932
|
+
await self._applock(cur, f"mefor:finalize:{message_id}")
|
|
933
|
+
await cur.execute(
|
|
934
|
+
"UPDATE messages SET status=? WHERE id=?", (disposition.value, message_id)
|
|
935
|
+
)
|
|
936
|
+
event = "routed" if disposition is MessageStatus.ROUTED else "unrouted"
|
|
937
|
+
await self._event(cur, message_id, event, None, f"{len(handlers)} handler(s)", now)
|
|
938
|
+
await conn.commit()
|
|
939
|
+
except Exception:
|
|
940
|
+
await conn.rollback()
|
|
941
|
+
raise
|
|
942
|
+
return True
|
|
943
|
+
|
|
944
|
+
async def transform_handoff(
|
|
945
|
+
self,
|
|
946
|
+
*,
|
|
947
|
+
routed_id: str,
|
|
948
|
+
message_id: str,
|
|
949
|
+
channel_id: str,
|
|
950
|
+
deliveries: Sequence[tuple[str, str]],
|
|
951
|
+
state_ops: Sequence[tuple[str, str, Any]] = (),
|
|
952
|
+
now: float | None = None,
|
|
953
|
+
) -> bool:
|
|
954
|
+
"""Advance one handler assignment from the routed stage to outbound in ONE transaction (the
|
|
955
|
+
transform half): consume the in-flight routed row, apply each declared state write (ADR 0005),
|
|
956
|
+
insert one outbound row per delivery, then let the finalizer recompute the terminal disposition
|
|
957
|
+
(this method NEVER writes ``messages.status`` itself). State writes are applied in sorted
|
|
958
|
+
(namespace, key) order under HOLDLOCK to bound MERGE range-deadlocks, and commit atomically
|
|
959
|
+
with the outbound rows (exactly-once per re-run); the read-through cache is updated only AFTER
|
|
960
|
+
commit. Idempotent: False if the routed row was already consumed."""
|
|
961
|
+
now = time.time() if now is None else now
|
|
962
|
+
applied: list[tuple[tuple[str, str], Any]] = []
|
|
963
|
+
async with self._acquire() as conn:
|
|
964
|
+
cur = await conn.cursor()
|
|
965
|
+
try:
|
|
966
|
+
await cur.execute(
|
|
967
|
+
"DELETE FROM queue OUTPUT deleted.id WHERE id=? AND stage=? AND status=?",
|
|
968
|
+
(routed_id, Stage.ROUTED.value, OutboxStatus.INFLIGHT.value),
|
|
969
|
+
)
|
|
970
|
+
if await cur.fetchone() is None:
|
|
971
|
+
await conn.rollback()
|
|
972
|
+
return False # already handed off (crash-restart) — idempotent no-op
|
|
973
|
+
for namespace, key, value in sorted(state_ops, key=lambda op: (op[0], op[1])):
|
|
974
|
+
enc = self._cipher.encrypt(json.dumps(value))
|
|
975
|
+
await cur.execute(
|
|
976
|
+
"MERGE state WITH (HOLDLOCK) AS t"
|
|
977
|
+
" USING (SELECT ? AS namespace, ? AS [key]) AS s"
|
|
978
|
+
" ON t.namespace=s.namespace AND t.[key]=s.[key]"
|
|
979
|
+
" WHEN MATCHED THEN UPDATE SET value=?, set_at=?, message_id=?"
|
|
980
|
+
" WHEN NOT MATCHED THEN INSERT (namespace, [key], value, set_at, message_id)"
|
|
981
|
+
" VALUES (?,?,?,?,?);",
|
|
982
|
+
(
|
|
983
|
+
namespace,
|
|
984
|
+
key,
|
|
985
|
+
enc,
|
|
986
|
+
now,
|
|
987
|
+
message_id,
|
|
988
|
+
namespace,
|
|
989
|
+
key,
|
|
990
|
+
enc,
|
|
991
|
+
now,
|
|
992
|
+
message_id,
|
|
993
|
+
),
|
|
994
|
+
)
|
|
995
|
+
applied.append(((namespace, key), value))
|
|
996
|
+
for dest_name, payload in deliveries:
|
|
997
|
+
await self._insert_outbound(
|
|
998
|
+
cur, message_id, channel_id, dest_name, payload, now
|
|
999
|
+
)
|
|
1000
|
+
await self._event(
|
|
1001
|
+
cur, message_id, "transformed", None, f"{len(deliveries)} destination(s)", now
|
|
1002
|
+
)
|
|
1003
|
+
# Finalizer is the sole disposition authority here (no direct messages.status write).
|
|
1004
|
+
await self._maybe_finalize(cur, message_id, now)
|
|
1005
|
+
await conn.commit()
|
|
1006
|
+
except Exception:
|
|
1007
|
+
await conn.rollback()
|
|
1008
|
+
raise
|
|
1009
|
+
# Commit succeeded → publish the committed state writes to the read-through cache.
|
|
1010
|
+
for ck, cv in applied:
|
|
1011
|
+
self._state_cache[ck] = cv
|
|
1012
|
+
return True
|
|
1013
|
+
|
|
1014
|
+
async def complete_with_response(
|
|
1015
|
+
self,
|
|
1016
|
+
outbox_id: str,
|
|
1017
|
+
*,
|
|
1018
|
+
body: str,
|
|
1019
|
+
outcome: str,
|
|
1020
|
+
detail: str | None = None,
|
|
1021
|
+
reingress_to: str | None = None,
|
|
1022
|
+
now: float | None = None,
|
|
1023
|
+
) -> None:
|
|
1024
|
+
"""Mark one outbound row delivered AND persist the partner's captured reply in ONE transaction
|
|
1025
|
+
(ADR 0013). ``response_seq`` is ``1 + MAX`` per (message_id, destination_name) so it is replay-
|
|
1026
|
+
stable; the ``response`` table is invisible to the finalizer (it scans ``queue`` only). When
|
|
1027
|
+
``reingress_to`` is set the same transaction also inserts the drainable ``Stage.RESPONSE`` work-
|
|
1028
|
+
row (which holds the origin non-terminal until ``ingress_handoff`` consumes it). body + detail
|
|
1029
|
+
are ciphertext; outcome is plaintext."""
|
|
1030
|
+
now = time.time() if now is None else now
|
|
1031
|
+
async with self._acquire() as conn:
|
|
1032
|
+
cur = await conn.cursor()
|
|
1033
|
+
try:
|
|
1034
|
+
# Leading SELECT (also opens the txn so _maybe_finalize's applock is never first).
|
|
1035
|
+
await cur.execute(
|
|
1036
|
+
"SELECT message_id, destination_name, attempts FROM queue WHERE id=?",
|
|
1037
|
+
(outbox_id,),
|
|
1038
|
+
)
|
|
1039
|
+
row = await cur.fetchone()
|
|
1040
|
+
if row is None:
|
|
1041
|
+
await conn.commit()
|
|
1042
|
+
return
|
|
1043
|
+
message_id, destination_name, attempts = row[0], row[1], row[2]
|
|
1044
|
+
await cur.execute(
|
|
1045
|
+
"UPDATE queue SET status=?, last_error=NULL, updated_at=?, owner=NULL,"
|
|
1046
|
+
" lease_expires_at=NULL WHERE id=?",
|
|
1047
|
+
(OutboxStatus.DONE.value, now, outbox_id),
|
|
1048
|
+
)
|
|
1049
|
+
await cur.execute(
|
|
1050
|
+
"SELECT COALESCE(MAX(response_seq), 0) + 1 FROM response"
|
|
1051
|
+
" WHERE message_id=? AND destination_name=?",
|
|
1052
|
+
(message_id, destination_name),
|
|
1053
|
+
)
|
|
1054
|
+
seq = int((await cur.fetchone())[0])
|
|
1055
|
+
# Inline the PG _enc empty-guard: encrypt only a truthy value (never '' / None).
|
|
1056
|
+
enc_body = self._cipher.encrypt(body) if body else body
|
|
1057
|
+
enc_detail = self._cipher.encrypt(detail) if detail else detail
|
|
1058
|
+
await cur.execute(
|
|
1059
|
+
"INSERT INTO response"
|
|
1060
|
+
" (message_id, destination_name, response_seq, body, outcome, detail, captured_at)"
|
|
1061
|
+
" VALUES (?,?,?,?,?,?,?)",
|
|
1062
|
+
(message_id, destination_name, seq, enc_body, outcome, enc_detail, now),
|
|
1063
|
+
)
|
|
1064
|
+
if reingress_to is not None:
|
|
1065
|
+
# ADR 0013 Increment 2: a drainable Stage.RESPONSE work-row in the SAME txn (orphan-
|
|
1066
|
+
# free) — a token referencing the immutable artifact by PK, on the loopback lane.
|
|
1067
|
+
artifact_ref = f"{message_id}\x1f{destination_name}\x1f{seq}"
|
|
1068
|
+
work_created = await self._fifo_created_at(
|
|
1069
|
+
cur, Stage.RESPONSE.value, "channel_id", reingress_to, now
|
|
1070
|
+
)
|
|
1071
|
+
await cur.execute(
|
|
1072
|
+
"INSERT INTO queue (id, message_id, stage, channel_id, destination_name,"
|
|
1073
|
+
" handler_name, payload, status, attempts, next_attempt_at, owner,"
|
|
1074
|
+
" lease_expires_at, created_at, updated_at)"
|
|
1075
|
+
" VALUES (?,?,?,?,NULL,NULL,?,?,0,?,NULL,NULL,?,?)",
|
|
1076
|
+
(
|
|
1077
|
+
uuid4().hex,
|
|
1078
|
+
message_id,
|
|
1079
|
+
Stage.RESPONSE.value,
|
|
1080
|
+
reingress_to,
|
|
1081
|
+
self._cipher.encrypt(artifact_ref),
|
|
1082
|
+
OutboxStatus.PENDING.value,
|
|
1083
|
+
now,
|
|
1084
|
+
work_created,
|
|
1085
|
+
now,
|
|
1086
|
+
),
|
|
1087
|
+
)
|
|
1088
|
+
await self._event(
|
|
1089
|
+
cur,
|
|
1090
|
+
message_id,
|
|
1091
|
+
"delivered",
|
|
1092
|
+
destination_name,
|
|
1093
|
+
f"attempt {attempts} (response {outcome})",
|
|
1094
|
+
now,
|
|
1095
|
+
)
|
|
1096
|
+
# Finalizer last; preceded by the SELECT above so its applock is not the first statement.
|
|
1097
|
+
await self._maybe_finalize(cur, message_id, now)
|
|
1098
|
+
await conn.commit()
|
|
1099
|
+
except Exception:
|
|
1100
|
+
await conn.rollback()
|
|
1101
|
+
raise
|
|
1102
|
+
|
|
1103
|
+
async def correlate_response(self, message_id: str) -> list[CapturedResponse]:
|
|
1104
|
+
"""Captured replies for a message (ADR 0013), ordered by destination then ``response_seq`` ASC
|
|
1105
|
+
(so the latest reply per destination is last). ``body`` + ``detail`` are both decrypted (both
|
|
1106
|
+
ciphertext); a NULL (never-captured or purged) body/detail returns ``None`` while an empty ``''``
|
|
1107
|
+
round-trips as ``''`` — parity with PG/SQLite ``_dec``; ``outcome`` is plaintext."""
|
|
1108
|
+
rows = await self._fetchall(
|
|
1109
|
+
"SELECT message_id, destination_name, response_seq, body, outcome, detail, captured_at"
|
|
1110
|
+
" FROM response WHERE message_id=? ORDER BY destination_name, response_seq",
|
|
1111
|
+
(message_id,),
|
|
1112
|
+
)
|
|
1113
|
+
return [
|
|
1114
|
+
CapturedResponse(
|
|
1115
|
+
message_id=r["message_id"],
|
|
1116
|
+
destination_name=r["destination_name"],
|
|
1117
|
+
response_seq=int(r["response_seq"]),
|
|
1118
|
+
outcome=r["outcome"],
|
|
1119
|
+
detail=self._cipher.decrypt(r["detail"]) if r["detail"] is not None else None,
|
|
1120
|
+
captured_at=float(r["captured_at"]),
|
|
1121
|
+
body=self._cipher.decrypt(r["body"]) if r["body"] is not None else None,
|
|
1122
|
+
)
|
|
1123
|
+
for r in rows
|
|
1124
|
+
]
|
|
1125
|
+
|
|
1126
|
+
async def ingress_handoff(
|
|
1127
|
+
self,
|
|
1128
|
+
*,
|
|
1129
|
+
response_row_id: str,
|
|
1130
|
+
loopback_channel_id: str,
|
|
1131
|
+
correlation_depth_cap: int,
|
|
1132
|
+
control_id: str | None,
|
|
1133
|
+
message_type: str | None,
|
|
1134
|
+
summary: str | None,
|
|
1135
|
+
peek_failed: bool = False,
|
|
1136
|
+
now: float | None = None,
|
|
1137
|
+
) -> bool:
|
|
1138
|
+
"""Consume one in-flight ``Stage.RESPONSE`` work-row and re-ingress the captured reply as a new
|
|
1139
|
+
message on the loopback inbound (ADR 0013 Increment 2), in ONE transaction. Idempotent: returns
|
|
1140
|
+
``False`` if the work-row was already consumed. A corrupt/unparseable ref or a re-ingress that
|
|
1141
|
+
would exceed ``correlation_depth_cap`` is dead-lettered (and the token still consumed). The
|
|
1142
|
+
re-ingress message id is content-addressed (deterministic), so a re-run never double-inserts the
|
|
1143
|
+
child."""
|
|
1144
|
+
now = time.time() if now is None else now
|
|
1145
|
+
async with self._acquire() as conn:
|
|
1146
|
+
cur = await conn.cursor()
|
|
1147
|
+
try:
|
|
1148
|
+
# (1) Guard-read the in-flight work-row (also opens the txn -> applock not first).
|
|
1149
|
+
await cur.execute(
|
|
1150
|
+
"SELECT message_id, payload FROM queue WHERE id=? AND stage=? AND status=?",
|
|
1151
|
+
(response_row_id, Stage.RESPONSE.value, OutboxStatus.INFLIGHT.value),
|
|
1152
|
+
)
|
|
1153
|
+
wr = await cur.fetchone()
|
|
1154
|
+
if wr is None:
|
|
1155
|
+
await conn.rollback()
|
|
1156
|
+
return False # already consumed by a committed prior run (idempotent no-op)
|
|
1157
|
+
origin_id = wr[0]
|
|
1158
|
+
# (2) Decrypt + parse the artifact ref; ANY failure -> consume-and-dead-letter.
|
|
1159
|
+
try:
|
|
1160
|
+
ref = self._cipher.decrypt(wr[1]) or ""
|
|
1161
|
+
origin_msg_id, dest, seq_s = ref.split("\x1f")
|
|
1162
|
+
seq = int(seq_s)
|
|
1163
|
+
except Exception: # noqa: BLE001 - any decrypt/parse failure = an unrecoverable ref
|
|
1164
|
+
await cur.execute(
|
|
1165
|
+
"UPDATE queue SET status=?, last_error=?, next_attempt_at=?, updated_at=?"
|
|
1166
|
+
" WHERE id=?",
|
|
1167
|
+
(
|
|
1168
|
+
OutboxStatus.DEAD.value,
|
|
1169
|
+
"re-ingress work-row reference is corrupt/unparseable",
|
|
1170
|
+
now,
|
|
1171
|
+
now,
|
|
1172
|
+
response_row_id,
|
|
1173
|
+
),
|
|
1174
|
+
)
|
|
1175
|
+
await self._event(cur, origin_id, "dead", None, "re-ingress ref corrupt", now)
|
|
1176
|
+
await self._maybe_finalize(cur, origin_id, now) # preceded by step-1 SELECT
|
|
1177
|
+
await conn.commit()
|
|
1178
|
+
return True # CONSUME (status flipped), never re-loop
|
|
1179
|
+
# (3) Read the immutable artifact body.
|
|
1180
|
+
await cur.execute(
|
|
1181
|
+
"SELECT body FROM response"
|
|
1182
|
+
" WHERE message_id=? AND destination_name=? AND response_seq=?",
|
|
1183
|
+
(origin_msg_id, dest, seq),
|
|
1184
|
+
)
|
|
1185
|
+
art = await cur.fetchone()
|
|
1186
|
+
body = (self._cipher.decrypt(art[0]) if (art and art[0]) else "") or ""
|
|
1187
|
+
# (4) Correlation lineage from the origin's metadata (parse once).
|
|
1188
|
+
await cur.execute("SELECT metadata FROM messages WHERE id=?", (origin_id,))
|
|
1189
|
+
mrow = await cur.fetchone()
|
|
1190
|
+
loaded = json.loads(mrow[0]) if (mrow and mrow[0]) else {}
|
|
1191
|
+
origin_meta = loaded if isinstance(loaded, dict) else {}
|
|
1192
|
+
child_depth = int(origin_meta.get("correlation_depth", 0) or 0) + 1
|
|
1193
|
+
root = origin_meta.get("correlation_root_id") or origin_id
|
|
1194
|
+
# (5) Depth-cap -> consume-and-dead-letter.
|
|
1195
|
+
if child_depth > correlation_depth_cap:
|
|
1196
|
+
await cur.execute(
|
|
1197
|
+
"UPDATE queue SET status=?, last_error=?, next_attempt_at=?, updated_at=?"
|
|
1198
|
+
" WHERE id=?",
|
|
1199
|
+
(
|
|
1200
|
+
OutboxStatus.DEAD.value,
|
|
1201
|
+
f"re-ingress correlation depth exceeded "
|
|
1202
|
+
f"({child_depth} > {correlation_depth_cap})",
|
|
1203
|
+
now,
|
|
1204
|
+
now,
|
|
1205
|
+
response_row_id,
|
|
1206
|
+
),
|
|
1207
|
+
)
|
|
1208
|
+
await self._event(
|
|
1209
|
+
cur, origin_id, "dead", dest, f"re-ingress depth cap ({child_depth})", now
|
|
1210
|
+
)
|
|
1211
|
+
await self._maybe_finalize(cur, origin_id, now) # preceded by step-1 SELECT
|
|
1212
|
+
await conn.commit()
|
|
1213
|
+
return True
|
|
1214
|
+
# (6) Deterministic child id + idempotent insert (the guarded DELETE is the real gate).
|
|
1215
|
+
new_mid = MessageStore._reingress_message_id(origin_id, dest, seq, body)
|
|
1216
|
+
await cur.execute("SELECT 1 FROM messages WHERE id=?", (new_mid,))
|
|
1217
|
+
if await cur.fetchone() is None:
|
|
1218
|
+
child_meta = json.dumps(
|
|
1219
|
+
{
|
|
1220
|
+
"correlation_id": origin_id,
|
|
1221
|
+
"correlation_root_id": root,
|
|
1222
|
+
"correlation_depth": child_depth,
|
|
1223
|
+
"reingress_of_seq": seq,
|
|
1224
|
+
}
|
|
1225
|
+
)
|
|
1226
|
+
await cur.execute(
|
|
1227
|
+
"INSERT INTO messages (id, channel_id, received_at, source_type, control_id,"
|
|
1228
|
+
" message_type, raw, status, error, summary, metadata)"
|
|
1229
|
+
" VALUES (?,?,?,?,?,?,?,?,?,?,?)",
|
|
1230
|
+
(
|
|
1231
|
+
new_mid,
|
|
1232
|
+
loopback_channel_id,
|
|
1233
|
+
now,
|
|
1234
|
+
"reingress",
|
|
1235
|
+
control_id,
|
|
1236
|
+
message_type,
|
|
1237
|
+
self._cipher.encrypt(body),
|
|
1238
|
+
MessageStatus.ERROR.value
|
|
1239
|
+
if peek_failed
|
|
1240
|
+
else MessageStatus.RECEIVED.value,
|
|
1241
|
+
"re-ingress body failed HL7 peek" if peek_failed else None,
|
|
1242
|
+
summary,
|
|
1243
|
+
child_meta,
|
|
1244
|
+
),
|
|
1245
|
+
)
|
|
1246
|
+
if not peek_failed:
|
|
1247
|
+
ingress_created = await self._fifo_created_at(
|
|
1248
|
+
cur, Stage.INGRESS.value, "channel_id", loopback_channel_id, now
|
|
1249
|
+
)
|
|
1250
|
+
await cur.execute(
|
|
1251
|
+
"INSERT INTO queue (id, message_id, stage, channel_id, destination_name,"
|
|
1252
|
+
" handler_name, payload, status, attempts, next_attempt_at, owner,"
|
|
1253
|
+
" lease_expires_at, created_at, updated_at)"
|
|
1254
|
+
" VALUES (?,?,?,?,NULL,NULL,?,?,0,?,NULL,NULL,?,?)",
|
|
1255
|
+
(
|
|
1256
|
+
uuid4().hex,
|
|
1257
|
+
new_mid,
|
|
1258
|
+
Stage.INGRESS.value,
|
|
1259
|
+
loopback_channel_id,
|
|
1260
|
+
self._cipher.encrypt(body),
|
|
1261
|
+
OutboxStatus.PENDING.value,
|
|
1262
|
+
now,
|
|
1263
|
+
ingress_created,
|
|
1264
|
+
now,
|
|
1265
|
+
),
|
|
1266
|
+
)
|
|
1267
|
+
await self._event(
|
|
1268
|
+
cur,
|
|
1269
|
+
new_mid,
|
|
1270
|
+
"received",
|
|
1271
|
+
None,
|
|
1272
|
+
f"reingress from {origin_id}/{dest}/seq{seq}",
|
|
1273
|
+
now,
|
|
1274
|
+
)
|
|
1275
|
+
await self._event(
|
|
1276
|
+
cur,
|
|
1277
|
+
origin_id,
|
|
1278
|
+
"reingressed",
|
|
1279
|
+
dest,
|
|
1280
|
+
f"-> {new_mid} depth {child_depth}",
|
|
1281
|
+
now,
|
|
1282
|
+
)
|
|
1283
|
+
# (7) Consume the token — exactly-once commit point (OUTPUT readback, never rowcount).
|
|
1284
|
+
await cur.execute(
|
|
1285
|
+
"DELETE FROM queue OUTPUT deleted.id WHERE id=? AND stage=? AND status=?",
|
|
1286
|
+
(response_row_id, Stage.RESPONSE.value, OutboxStatus.INFLIGHT.value),
|
|
1287
|
+
)
|
|
1288
|
+
if await cur.fetchone() is None:
|
|
1289
|
+
await conn.rollback()
|
|
1290
|
+
return False # defensive; unreachable under single-owner claim
|
|
1291
|
+
# (8) Finalize the origin (its last RESPONSE row is now gone).
|
|
1292
|
+
await self._maybe_finalize(cur, origin_id, now)
|
|
1293
|
+
await conn.commit()
|
|
1294
|
+
return True
|
|
1295
|
+
except Exception:
|
|
1296
|
+
await conn.rollback()
|
|
1297
|
+
raise
|
|
1298
|
+
|
|
1299
|
+
async def response_body_for_work_row(self, response_row_id: str) -> str | None:
|
|
1300
|
+
"""The decrypted reply body behind a ``Stage.RESPONSE`` work-row (ADR 0013) — for the re-ingress
|
|
1301
|
+
worker's HL7 peek. ``None`` if the work-row is missing/consumed or its ref is unparseable; ``''``
|
|
1302
|
+
if the artifact body is missing/empty. Reads the SAME immutable artifact ``ingress_handoff``
|
|
1303
|
+
re-reads, so the peek and the persisted raw always agree."""
|
|
1304
|
+
row = await self._fetchone(
|
|
1305
|
+
"SELECT payload FROM queue WHERE id=? AND stage=?",
|
|
1306
|
+
(response_row_id, Stage.RESPONSE.value),
|
|
1307
|
+
)
|
|
1308
|
+
if row is None:
|
|
1309
|
+
return None
|
|
1310
|
+
ref = self._cipher.decrypt(row["payload"]) or ""
|
|
1311
|
+
try:
|
|
1312
|
+
mid, dest, seq_s = ref.split("\x1f")
|
|
1313
|
+
except ValueError:
|
|
1314
|
+
return None
|
|
1315
|
+
art = await self._fetchone(
|
|
1316
|
+
"SELECT body FROM response WHERE message_id=? AND destination_name=? AND response_seq=?",
|
|
1317
|
+
(mid, dest, int(seq_s)),
|
|
1318
|
+
)
|
|
1319
|
+
return self._cipher.decrypt(art["body"]) if (art and art["body"]) else ""
|
|
1320
|
+
|
|
1321
|
+
def state_view(self) -> Mapping[tuple[str, str], Any]:
|
|
1322
|
+
"""Read-only view of the ADR 0005 transform-state read-through cache (parity with SQLite/PG).
|
|
1323
|
+
The runner publishes it around each router/transform run so a Handler's ``state_get(...)``
|
|
1324
|
+
resolves cross-message; ``transform_handoff`` refreshes it post-commit."""
|
|
1325
|
+
return MappingProxyType(self._state_cache)
|
|
1326
|
+
|
|
1327
|
+
async def _load_state_cache(self) -> None:
|
|
1328
|
+
"""Warm the transform-state read-through cache from the ``state`` table at open (ADR 0005)."""
|
|
1329
|
+
rows = await self._fetchall("SELECT namespace, [key], value FROM state")
|
|
1330
|
+
cache: dict[tuple[str, str], Any] = {}
|
|
1331
|
+
for r in rows:
|
|
1332
|
+
try:
|
|
1333
|
+
cache[(r["namespace"], r["key"])] = json.loads(self._cipher.decrypt(r["value"]))
|
|
1334
|
+
except (CipherError, ValueError) as exc: # skip a corrupt/undecryptable state row
|
|
1335
|
+
log.warning(
|
|
1336
|
+
"skipping unreadable state row %r/%r: %s", r["namespace"], r["key"], exc
|
|
1337
|
+
)
|
|
1338
|
+
self._state_cache = cache
|
|
1339
|
+
|
|
1340
|
+
def reference_view(self) -> Mapping[str, Mapping[str, Any]]:
|
|
1341
|
+
"""Empty reference view on the SQL Server backend (ADR 0006). Reference snapshots live in the
|
|
1342
|
+
SQLite store; returning an empty read-only mapping keeps a Handler's ``reference("name")``
|
|
1343
|
+
call shaped correctly — though it will raise ``ReferenceError`` for any name (no set synced),
|
|
1344
|
+
which is the honest state on this backend (reference snapshots are SQLite/Postgres-only here)."""
|
|
1345
|
+
return MappingProxyType({})
|
|
1346
|
+
|
|
1347
|
+
async def write_reference_snapshot(
|
|
1348
|
+
self, *, name: str, version: str, rows: Mapping[str, Any]
|
|
1349
|
+
) -> None:
|
|
1350
|
+
"""Not supported on the SQL Server backend — reference snapshots (ADR 0006) are SQLite/Postgres-
|
|
1351
|
+
only. NOTE: the engine DOES run on this backend now (staged pipeline + capture), so if a
|
|
1352
|
+
reference SET is configured against the SQL Server store the ReferenceSyncRunner logs a sync
|
|
1353
|
+
failure each interval until ADR 0006 lands here (the engine survives; the set never materializes
|
|
1354
|
+
and ``reference()`` raises). Present for ``Store`` protocol completeness."""
|
|
1355
|
+
raise NotImplementedError(
|
|
1356
|
+
"write_reference_snapshot is not supported on the SQL Server backend (ADR 0006 reference "
|
|
1357
|
+
"sets are SQLite/Postgres-only); use the SQLite or Postgres backend"
|
|
1358
|
+
)
|
|
1359
|
+
|
|
1360
|
+
async def converge_reference_cache(self) -> list[str]:
|
|
1361
|
+
"""No-op on the SQL Server backend (Track B Step 6). Reference snapshots are SQLite/Postgres-only
|
|
1362
|
+
here (write_reference_snapshot raises), so there is no shared snapshot to read through. Present
|
|
1363
|
+
for ``Store`` protocol completeness; returns ``[]``."""
|
|
1364
|
+
return []
|
|
1365
|
+
|
|
1366
|
+
async def converge_state_cache(self) -> list[str]:
|
|
1367
|
+
"""No-op on the SQL Server backend (Track B Step 6b): a single-node backend with no cross-node
|
|
1368
|
+
state convergence (transform state IS written here via transform_handoff, but never converged
|
|
1369
|
+
across nodes). Present for ``Store`` protocol completeness; returns ``[]``."""
|
|
1370
|
+
return []
|
|
1371
|
+
|
|
1372
|
+
def enable_state_convergence(self) -> None:
|
|
1373
|
+
"""No-op on the SQL Server backend (Track B Step 6b): there is no cross-node convergence here, so
|
|
1374
|
+
there is no per-namespace version to bump. Present for ``Store`` protocol completeness."""
|
|
1375
|
+
return None
|
|
1376
|
+
|
|
1377
|
+
async def dead_letter_missing_handlers(
|
|
1378
|
+
self, valid_names: set[str], now: float | None = None
|
|
1379
|
+
) -> int:
|
|
1380
|
+
"""Dead-letter non-terminal routed queue rows whose handler_name is no longer in the registry
|
|
1381
|
+
(a removed/renamed handler) — no transform worker would drain them, so they'd strand forever.
|
|
1382
|
+
Call ONCE at startup, AFTER reset_stale_inflight. Per-message finalize applocks are pre-acquired
|
|
1383
|
+
in sorted id order to avoid multi-message deadlock; a killed routed row -> DEAD -> the finalizer
|
|
1384
|
+
resolves the message to ERROR."""
|
|
1385
|
+
now = time.time() if now is None else now
|
|
1386
|
+
async with self._acquire() as conn:
|
|
1387
|
+
cur = await conn.cursor()
|
|
1388
|
+
try:
|
|
1389
|
+
await cur.execute(
|
|
1390
|
+
"SELECT id, message_id, handler_name FROM queue"
|
|
1391
|
+
" WHERE stage=? AND status IN (?, ?)",
|
|
1392
|
+
(Stage.ROUTED.value, OutboxStatus.PENDING.value, OutboxStatus.INFLIGHT.value),
|
|
1393
|
+
)
|
|
1394
|
+
rows = await cur.fetchall() # positional: (id, message_id, handler_name)
|
|
1395
|
+
orphans = [r for r in rows if r[2] not in valid_names]
|
|
1396
|
+
if not orphans:
|
|
1397
|
+
await conn.commit()
|
|
1398
|
+
return 0
|
|
1399
|
+
error = "handler removed from registry"
|
|
1400
|
+
await self._lock_finalize_batch(cur, {r[1] for r in orphans})
|
|
1401
|
+
for row in orphans:
|
|
1402
|
+
await cur.execute(
|
|
1403
|
+
"UPDATE queue SET status=?, next_attempt_at=?, last_error=?, updated_at=?,"
|
|
1404
|
+
" owner=NULL, lease_expires_at=NULL WHERE id=?",
|
|
1405
|
+
(OutboxStatus.DEAD.value, now, error, now, row[0]),
|
|
1406
|
+
)
|
|
1407
|
+
await self._event(cur, row[1], "dead", None, error, now)
|
|
1408
|
+
await self._maybe_finalize(cur, row[1], now)
|
|
1409
|
+
await conn.commit()
|
|
1410
|
+
except Exception:
|
|
1411
|
+
await conn.rollback()
|
|
1412
|
+
raise
|
|
1413
|
+
log.warning(
|
|
1414
|
+
"dead-lettered %d orphaned routed row(s) at startup for missing handler(s): %s",
|
|
1415
|
+
len(orphans),
|
|
1416
|
+
", ".join(sorted({r[2] for r in orphans})),
|
|
1417
|
+
)
|
|
1418
|
+
return len(orphans)
|
|
1419
|
+
|
|
1420
|
+
# --- retention / purge + maintenance (PHI.md §8) -------------------------
|
|
1421
|
+
# The RetentionRunner drives these once the staged pipeline is enabled. Bodies are blanked to ''
|
|
1422
|
+
# (not deleted) so cipher re-encrypt scans skip them and the FK to messages stays intact. SQL
|
|
1423
|
+
# Server TDE remains the at-rest baseline; this engine-side rotation/purge complements it.
|
|
1424
|
+
|
|
1425
|
+
async def reencrypt_to_active(self, *, batch: int = 500) -> int:
|
|
1426
|
+
"""Re-encrypt body columns sitting under a RETIRED key to the active key (key rotation),
|
|
1427
|
+
batched per (table, column). No-op (returns 0) unless an AES-GCM keyring cipher is configured.
|
|
1428
|
+
Each batch's re-encrypt list is built UP FRONT so an undecryptable value raises BEFORE any
|
|
1429
|
+
UPDATE (all-or-nothing; PHI never dropped). Skips rows already under the active key and
|
|
1430
|
+
blank/purged values."""
|
|
1431
|
+
if not isinstance(self._cipher, AesGcmCipher):
|
|
1432
|
+
return 0
|
|
1433
|
+
active_like = f"{_ENC_PREFIX}{self._cipher.active_key_id}:%"
|
|
1434
|
+
total = 0
|
|
1435
|
+
for table, column in (("messages", "raw"), ("queue", "payload"), ("users", "totp_secret")):
|
|
1436
|
+
while True:
|
|
1437
|
+
rows = await self._fetchall(
|
|
1438
|
+
f"SELECT TOP (?) id, {column} AS v FROM {table}"
|
|
1439
|
+
f" WHERE {column} NOT LIKE ? AND {column} <> ''",
|
|
1440
|
+
(batch, active_like),
|
|
1441
|
+
)
|
|
1442
|
+
if not rows:
|
|
1443
|
+
break
|
|
1444
|
+
# Decrypt+re-encrypt UP FRONT: a CipherError aborts the batch before any write.
|
|
1445
|
+
updates = [
|
|
1446
|
+
(self._cipher.encrypt(self._cipher.decrypt(r["v"])), r["id"]) for r in rows
|
|
1447
|
+
]
|
|
1448
|
+
async with self._acquire() as conn:
|
|
1449
|
+
cur = await conn.cursor()
|
|
1450
|
+
try:
|
|
1451
|
+
for enc, rid in updates:
|
|
1452
|
+
await cur.execute(
|
|
1453
|
+
f"UPDATE {table} SET {column}=? WHERE id=?", (enc, rid)
|
|
1454
|
+
)
|
|
1455
|
+
await conn.commit()
|
|
1456
|
+
except Exception:
|
|
1457
|
+
await conn.rollback()
|
|
1458
|
+
raise
|
|
1459
|
+
total += len(rows)
|
|
1460
|
+
# `state` has a composite PK (namespace, [key]) — its own pass (can't ride the id-keyed loop
|
|
1461
|
+
# above). transform_handoff writes state.value encrypted, so a rotation MUST rotate it too or a
|
|
1462
|
+
# later retired-key drop silently loses all transform cross-message state (review HIGH).
|
|
1463
|
+
while True:
|
|
1464
|
+
rows = await self._fetchall(
|
|
1465
|
+
"SELECT TOP (?) namespace, [key], value FROM state"
|
|
1466
|
+
" WHERE value NOT LIKE ? AND value <> ''",
|
|
1467
|
+
(batch, active_like),
|
|
1468
|
+
)
|
|
1469
|
+
if not rows:
|
|
1470
|
+
break
|
|
1471
|
+
state_updates = [
|
|
1472
|
+
(self._cipher.encrypt(self._cipher.decrypt(r["value"])), r["namespace"], r["key"])
|
|
1473
|
+
for r in rows
|
|
1474
|
+
]
|
|
1475
|
+
async with self._acquire() as conn:
|
|
1476
|
+
cur = await conn.cursor()
|
|
1477
|
+
try:
|
|
1478
|
+
for enc, ns, skey in state_updates:
|
|
1479
|
+
await cur.execute(
|
|
1480
|
+
"UPDATE state SET value=? WHERE namespace=? AND [key]=?",
|
|
1481
|
+
(enc, ns, skey),
|
|
1482
|
+
)
|
|
1483
|
+
await conn.commit()
|
|
1484
|
+
except Exception:
|
|
1485
|
+
await conn.rollback()
|
|
1486
|
+
raise
|
|
1487
|
+
total += len(rows)
|
|
1488
|
+
# `response` body + detail are ciphertext with a composite PK (message_id, destination_name,
|
|
1489
|
+
# response_seq) — their own passes. IS NOT NULL is explicit/defensive: NOT LIKE already excludes
|
|
1490
|
+
# NULLs (three-valued logic) and a NULL has no ciphertext to rotate — but these columns are
|
|
1491
|
+
# nullable (unlike state.value/messages.raw/queue.payload), so the guard documents that intent.
|
|
1492
|
+
for rcol in ("body", "detail"):
|
|
1493
|
+
while True:
|
|
1494
|
+
rows = await self._fetchall(
|
|
1495
|
+
f"SELECT TOP (?) message_id, destination_name, response_seq, {rcol} AS v"
|
|
1496
|
+
f" FROM response WHERE {rcol} NOT LIKE ? AND {rcol} <> '' AND {rcol} IS NOT NULL",
|
|
1497
|
+
(batch, active_like),
|
|
1498
|
+
)
|
|
1499
|
+
if not rows:
|
|
1500
|
+
break
|
|
1501
|
+
resp_updates = [
|
|
1502
|
+
(
|
|
1503
|
+
self._cipher.encrypt(self._cipher.decrypt(r["v"])),
|
|
1504
|
+
r["message_id"],
|
|
1505
|
+
r["destination_name"],
|
|
1506
|
+
r["response_seq"],
|
|
1507
|
+
)
|
|
1508
|
+
for r in rows
|
|
1509
|
+
]
|
|
1510
|
+
async with self._acquire() as conn:
|
|
1511
|
+
cur = await conn.cursor()
|
|
1512
|
+
try:
|
|
1513
|
+
for enc, rmid, rdest, rseq in resp_updates:
|
|
1514
|
+
await cur.execute(
|
|
1515
|
+
f"UPDATE response SET {rcol}=?"
|
|
1516
|
+
" WHERE message_id=? AND destination_name=? AND response_seq=?",
|
|
1517
|
+
(enc, rmid, rdest, rseq),
|
|
1518
|
+
)
|
|
1519
|
+
await conn.commit()
|
|
1520
|
+
except Exception:
|
|
1521
|
+
await conn.rollback()
|
|
1522
|
+
raise
|
|
1523
|
+
total += len(rows)
|
|
1524
|
+
if total:
|
|
1525
|
+
log.info("re-encrypted %d row(s) to the active key", total)
|
|
1526
|
+
return total
|
|
1527
|
+
|
|
1528
|
+
async def purge_message_bodies(self, *, older_than: float, now: float | None = None) -> int:
|
|
1529
|
+
"""Blank message bodies (and terminal outbound payloads + event details) for messages received
|
|
1530
|
+
before ``older_than`` whose queue rows are all terminal — retention (PHI.md §8). Bodies are
|
|
1531
|
+
blanked to '' (not deleted) so the cipher re-encrypt scans skip them and the FK to messages
|
|
1532
|
+
stays intact. The eligible set is materialized ONCE so all three tables purge exactly the same
|
|
1533
|
+
messages. Returns the number of message bodies blanked."""
|
|
1534
|
+
async with self._acquire() as conn:
|
|
1535
|
+
cur = await conn.cursor()
|
|
1536
|
+
try:
|
|
1537
|
+
# CREATE (no params) so the temp table lives at CONNECTION scope; a parameterized
|
|
1538
|
+
# SELECT...INTO runs under sp_executesql and would scope #eligible to that proc (gone
|
|
1539
|
+
# before the UPDATEs). The parameterized INSERT below still populates it.
|
|
1540
|
+
await cur.execute("CREATE TABLE #eligible (id NVARCHAR(64) PRIMARY KEY)")
|
|
1541
|
+
await cur.execute(
|
|
1542
|
+
"INSERT INTO #eligible SELECT id FROM messages m WHERE m.received_at < ?"
|
|
1543
|
+
" AND NOT EXISTS (SELECT 1 FROM queue q WHERE q.message_id=m.id"
|
|
1544
|
+
" AND q.status IN (?, ?))",
|
|
1545
|
+
(older_than, OutboxStatus.PENDING.value, OutboxStatus.INFLIGHT.value),
|
|
1546
|
+
)
|
|
1547
|
+
await cur.execute(
|
|
1548
|
+
"UPDATE messages SET raw='', summary=NULL, error=NULL"
|
|
1549
|
+
" WHERE raw <> '' AND id IN (SELECT id FROM #eligible)"
|
|
1550
|
+
)
|
|
1551
|
+
purged = cur.rowcount
|
|
1552
|
+
await cur.execute(
|
|
1553
|
+
"UPDATE queue SET payload='', last_error=NULL"
|
|
1554
|
+
" WHERE stage=? AND status IN (?, ?) AND payload <> ''"
|
|
1555
|
+
" AND message_id IN (SELECT id FROM #eligible)",
|
|
1556
|
+
(Stage.OUTBOUND.value, OutboxStatus.DONE.value, OutboxStatus.CANCELLED.value),
|
|
1557
|
+
)
|
|
1558
|
+
await cur.execute(
|
|
1559
|
+
"UPDATE message_events SET detail=NULL"
|
|
1560
|
+
" WHERE detail IS NOT NULL AND message_id IN (SELECT id FROM #eligible)"
|
|
1561
|
+
)
|
|
1562
|
+
# NULL captured response bodies/details for eligible messages (ADR 0013 retention) — to
|
|
1563
|
+
# NULL (matching PG/SQLite: correlate then reads None; reencrypt's IS NOT NULL skips them).
|
|
1564
|
+
await cur.execute(
|
|
1565
|
+
"UPDATE response SET body=NULL, detail=NULL"
|
|
1566
|
+
" WHERE (body IS NOT NULL OR detail IS NOT NULL)"
|
|
1567
|
+
" AND message_id IN (SELECT id FROM #eligible)"
|
|
1568
|
+
)
|
|
1569
|
+
await cur.execute("DROP TABLE #eligible")
|
|
1570
|
+
await conn.commit()
|
|
1571
|
+
except Exception:
|
|
1572
|
+
await conn.rollback()
|
|
1573
|
+
raise
|
|
1574
|
+
return int(purged) if purged is not None else 0
|
|
1575
|
+
|
|
1576
|
+
async def purge_state(self, *, older_than: float, now: float | None = None) -> int:
|
|
1577
|
+
"""Delete transform-state rows last set before ``older_than`` (ADR 0005 retention), evicting
|
|
1578
|
+
them from the read-through cache post-commit. Returns the number deleted."""
|
|
1579
|
+
async with self._acquire() as conn:
|
|
1580
|
+
cur = await conn.cursor()
|
|
1581
|
+
try:
|
|
1582
|
+
await cur.execute(
|
|
1583
|
+
"SELECT namespace, [key] FROM state WHERE set_at < ?", (older_than,)
|
|
1584
|
+
)
|
|
1585
|
+
purged_keys = [(r[0], r[1]) for r in await cur.fetchall()]
|
|
1586
|
+
if not purged_keys:
|
|
1587
|
+
await conn.commit()
|
|
1588
|
+
return 0
|
|
1589
|
+
await cur.execute("DELETE FROM state WHERE set_at < ?", (older_than,))
|
|
1590
|
+
await conn.commit()
|
|
1591
|
+
except Exception:
|
|
1592
|
+
await conn.rollback()
|
|
1593
|
+
raise
|
|
1594
|
+
for ck in purged_keys:
|
|
1595
|
+
self._state_cache.pop(ck, None)
|
|
1596
|
+
return len(purged_keys)
|
|
1597
|
+
|
|
1598
|
+
async def purge_dead_letters(self, *, older_than: float, now: float | None = None) -> int:
|
|
1599
|
+
"""Blank the payload of dead outbound rows updated before ``older_than`` (retention). Keeps the
|
|
1600
|
+
dead row + 'dead' status (counts/disposition) but frees the body; idempotent (payload <> '')."""
|
|
1601
|
+
async with self._acquire() as conn:
|
|
1602
|
+
cur = await conn.cursor()
|
|
1603
|
+
try:
|
|
1604
|
+
await cur.execute(
|
|
1605
|
+
"UPDATE queue SET payload='', last_error=NULL"
|
|
1606
|
+
" WHERE stage=? AND status=? AND payload <> '' AND updated_at < ?",
|
|
1607
|
+
(Stage.OUTBOUND.value, OutboxStatus.DEAD.value, older_than),
|
|
1608
|
+
)
|
|
1609
|
+
purged = cur.rowcount
|
|
1610
|
+
await conn.commit()
|
|
1611
|
+
except Exception:
|
|
1612
|
+
await conn.rollback()
|
|
1613
|
+
raise
|
|
1614
|
+
return int(purged) if purged is not None else 0
|
|
1615
|
+
|
|
1616
|
+
async def wal_checkpoint(self) -> None:
|
|
1617
|
+
"""No-op on SQL Server — there is no SQLite WAL to checkpoint (the engine never calls this on
|
|
1618
|
+
this backend; present for ``Store`` protocol completeness)."""
|
|
1619
|
+
|
|
1620
|
+
async def vacuum(self) -> None:
|
|
1621
|
+
"""No-op on SQL Server — file compaction is a DBA operation here, not an engine concern (the
|
|
1622
|
+
engine never calls this on this backend; present for ``Store`` protocol completeness)."""
|
|
1623
|
+
|
|
1624
|
+
async def record_received(
|
|
1625
|
+
self,
|
|
1626
|
+
*,
|
|
1627
|
+
channel_id: str,
|
|
1628
|
+
raw: str,
|
|
1629
|
+
status: MessageStatus,
|
|
1630
|
+
error: str | None = None,
|
|
1631
|
+
control_id: str | None = None,
|
|
1632
|
+
message_type: str | None = None,
|
|
1633
|
+
source_type: str | None = None,
|
|
1634
|
+
summary: str | None = None,
|
|
1635
|
+
metadata: str | None = None,
|
|
1636
|
+
now: float | None = None,
|
|
1637
|
+
) -> str:
|
|
1638
|
+
error = (
|
|
1639
|
+
safe_text(error) if error else error
|
|
1640
|
+
) # PHI chokepoint (#120); SQL Server stores plaintext
|
|
1641
|
+
now = time.time() if now is None else now
|
|
1642
|
+
mid = uuid4().hex
|
|
1643
|
+
event = "error" if status is MessageStatus.ERROR else "filtered"
|
|
1644
|
+
async with self._acquire() as conn:
|
|
1645
|
+
cur = await conn.cursor()
|
|
1646
|
+
try:
|
|
1647
|
+
await cur.execute(
|
|
1648
|
+
"INSERT INTO messages (id, channel_id, received_at, source_type, control_id,"
|
|
1649
|
+
" message_type, raw, status, error, summary, metadata)"
|
|
1650
|
+
" VALUES (?,?,?,?,?,?,?,?,?,?,?)",
|
|
1651
|
+
(
|
|
1652
|
+
mid,
|
|
1653
|
+
channel_id,
|
|
1654
|
+
now,
|
|
1655
|
+
source_type,
|
|
1656
|
+
control_id,
|
|
1657
|
+
message_type,
|
|
1658
|
+
self._cipher.encrypt(raw),
|
|
1659
|
+
status.value,
|
|
1660
|
+
error,
|
|
1661
|
+
summary,
|
|
1662
|
+
metadata,
|
|
1663
|
+
),
|
|
1664
|
+
)
|
|
1665
|
+
await self._event(cur, mid, event, None, error, now)
|
|
1666
|
+
await conn.commit()
|
|
1667
|
+
except Exception:
|
|
1668
|
+
await conn.rollback()
|
|
1669
|
+
raise
|
|
1670
|
+
return mid
|
|
1671
|
+
|
|
1672
|
+
# --- delivery worker path ------------------------------------------------
|
|
1673
|
+
|
|
1674
|
+
@staticmethod
|
|
1675
|
+
def _lane_col(stage: str) -> str:
|
|
1676
|
+
"""The FIFO/depth lane column for a stage (code-controlled literal): ``channel_id`` for
|
|
1677
|
+
ingress/routed/response, ``destination_name`` for outbound."""
|
|
1678
|
+
return (
|
|
1679
|
+
"channel_id"
|
|
1680
|
+
if stage in (Stage.INGRESS.value, Stage.ROUTED.value, Stage.RESPONSE.value)
|
|
1681
|
+
else "destination_name"
|
|
1682
|
+
)
|
|
1683
|
+
|
|
1684
|
+
async def claim_ready(
|
|
1685
|
+
self,
|
|
1686
|
+
limit: int = 10,
|
|
1687
|
+
now: float | None = None,
|
|
1688
|
+
*,
|
|
1689
|
+
stage: str = Stage.OUTBOUND.value,
|
|
1690
|
+
channel_id: str | None = None,
|
|
1691
|
+
destination_name: str | None = None,
|
|
1692
|
+
) -> list[OutboxItem]:
|
|
1693
|
+
"""Atomically claim up to ``limit`` due rows at ``stage`` (UNORDERED — skips a backing-off row
|
|
1694
|
+
to drain others), flipping them to ``inflight`` and bumping ``attempts``. ``READPAST, UPDLOCK,
|
|
1695
|
+
ROWLOCK`` is the T-SQL ``FOR UPDATE SKIP LOCKED`` analog so concurrent workers skip rather than
|
|
1696
|
+
block or double-claim. An undecryptable payload is dead-lettered and dropped (poison-row
|
|
1697
|
+
containment), not raised."""
|
|
1698
|
+
now = time.time() if now is None else now
|
|
1699
|
+
where = ["stage=?", "status=?", "next_attempt_at<=?"]
|
|
1700
|
+
filters: list[Any] = [stage, OutboxStatus.PENDING.value, now]
|
|
1701
|
+
if channel_id is not None:
|
|
1702
|
+
where.append("channel_id=?")
|
|
1703
|
+
filters.append(channel_id)
|
|
1704
|
+
if destination_name is not None:
|
|
1705
|
+
where.append("destination_name=?")
|
|
1706
|
+
filters.append(destination_name)
|
|
1707
|
+
sql = (
|
|
1708
|
+
"WITH due AS (SELECT TOP (?) * FROM queue WITH (READPAST, UPDLOCK, ROWLOCK)"
|
|
1709
|
+
f" WHERE {' AND '.join(where)} ORDER BY next_attempt_at)"
|
|
1710
|
+
" UPDATE due SET status=?, attempts=attempts+1, updated_at=?,"
|
|
1711
|
+
" owner=NULL, lease_expires_at=NULL"
|
|
1712
|
+
" OUTPUT inserted.id, inserted.message_id, inserted.channel_id,"
|
|
1713
|
+
" inserted.destination_name, inserted.handler_name, inserted.payload,"
|
|
1714
|
+
" inserted.attempts"
|
|
1715
|
+
)
|
|
1716
|
+
args = (limit, *filters, OutboxStatus.INFLIGHT.value, now)
|
|
1717
|
+
async with self._acquire() as conn:
|
|
1718
|
+
cur = await conn.cursor()
|
|
1719
|
+
try:
|
|
1720
|
+
await cur.execute(sql, args)
|
|
1721
|
+
columns = [c[0] for c in cur.description]
|
|
1722
|
+
rows = await cur.fetchall()
|
|
1723
|
+
await conn.commit()
|
|
1724
|
+
except Exception:
|
|
1725
|
+
await conn.rollback()
|
|
1726
|
+
raise
|
|
1727
|
+
items = []
|
|
1728
|
+
for row in rows:
|
|
1729
|
+
d = dict(zip(columns, row))
|
|
1730
|
+
try:
|
|
1731
|
+
payload = self._cipher.decrypt(d["payload"])
|
|
1732
|
+
except CipherError as exc:
|
|
1733
|
+
log.warning("dead-lettering undecryptable queue row %s: %s", d["id"], exc)
|
|
1734
|
+
await self.dead_letter_now(d["id"], f"undecryptable payload: {exc}")
|
|
1735
|
+
continue
|
|
1736
|
+
items.append(
|
|
1737
|
+
OutboxItem(
|
|
1738
|
+
id=d["id"],
|
|
1739
|
+
message_id=d["message_id"],
|
|
1740
|
+
channel_id=d["channel_id"],
|
|
1741
|
+
destination_name=d["destination_name"],
|
|
1742
|
+
handler_name=d["handler_name"],
|
|
1743
|
+
payload=payload,
|
|
1744
|
+
attempts=d["attempts"],
|
|
1745
|
+
stage=stage,
|
|
1746
|
+
)
|
|
1747
|
+
)
|
|
1748
|
+
return items
|
|
1749
|
+
|
|
1750
|
+
async def claim_next_fifo(
|
|
1751
|
+
self,
|
|
1752
|
+
name: str,
|
|
1753
|
+
now: float | None = None,
|
|
1754
|
+
*,
|
|
1755
|
+
stage: str = Stage.OUTBOUND.value,
|
|
1756
|
+
owner: str | None = None,
|
|
1757
|
+
) -> OutboxItem | None:
|
|
1758
|
+
"""Claim the single oldest *due* pending row for one lane at ``stage`` (strict FIFO — the head
|
|
1759
|
+
blocks the lane while it backs off, via the WHERE on the UPDATE). The lane key is stage-aware
|
|
1760
|
+
(``destination_name`` outbound, ``channel_id`` ingress/routed); ordering is ``created_at, seq``
|
|
1761
|
+
(the IDENTITY tiebreak preserving same-txn insertion order — NOT the random uuid ``id``).
|
|
1762
|
+
``owner`` (Track B lane ownership) is accepted for protocol uniformity and IGNORED: this backend
|
|
1763
|
+
is single-node (one worker per lane), so owner/lease stay NULL and the runner never passes a
|
|
1764
|
+
non-None owner.
|
|
1765
|
+
|
|
1766
|
+
NB: the head SELECT takes ``(UPDLOCK, ROWLOCK)`` but deliberately **NOT** ``READPAST``. With one
|
|
1767
|
+
serial consumer per lane, the only transaction that can hold a lock on the FIFO head is the
|
|
1768
|
+
*producer* (the ``transform_handoff``/finalizer that just inserted it, milliseconds from commit).
|
|
1769
|
+
``READPAST`` would SKIP that locked head and claim the next-oldest row instead — delivering seq
|
|
1770
|
+
N+1 before N (a per-lane FIFO break; issue #285). Head-of-line *blocking* on a transiently-locked
|
|
1771
|
+
head is the correct FIFO semantic here — it briefly waits for the rightful head, matching Postgres'
|
|
1772
|
+
``FOR UPDATE`` (no ``SKIP LOCKED`` skip of a producer-locked head). A pathologically long lock is
|
|
1773
|
+
bounded by ``command_timeout``. The UNORDERED :meth:`claim_ready` keeps ``READPAST`` (there,
|
|
1774
|
+
skipping a locked sibling to drain the lane is intended and order is explicitly not promised)."""
|
|
1775
|
+
now = time.time() if now is None else now
|
|
1776
|
+
lane_col = self._lane_col(stage) # code-controlled literal
|
|
1777
|
+
sql = (
|
|
1778
|
+
"WITH head AS (SELECT TOP (1) * FROM queue WITH (UPDLOCK, ROWLOCK)"
|
|
1779
|
+
f" WHERE stage=? AND {lane_col}=? AND status=? ORDER BY created_at, seq)"
|
|
1780
|
+
" UPDATE head SET status=?, attempts=attempts+1, updated_at=?,"
|
|
1781
|
+
" owner=NULL, lease_expires_at=NULL"
|
|
1782
|
+
" OUTPUT inserted.id, inserted.message_id, inserted.channel_id,"
|
|
1783
|
+
" inserted.destination_name, inserted.handler_name, inserted.payload,"
|
|
1784
|
+
" inserted.attempts"
|
|
1785
|
+
" WHERE next_attempt_at<=?"
|
|
1786
|
+
)
|
|
1787
|
+
args = (stage, name, OutboxStatus.PENDING.value, OutboxStatus.INFLIGHT.value, now, now)
|
|
1788
|
+
async with self._acquire() as conn:
|
|
1789
|
+
cur = await conn.cursor()
|
|
1790
|
+
try:
|
|
1791
|
+
await cur.execute(sql, args)
|
|
1792
|
+
columns = [c[0] for c in cur.description] if cur.description else []
|
|
1793
|
+
row = await cur.fetchone()
|
|
1794
|
+
await conn.commit()
|
|
1795
|
+
except Exception:
|
|
1796
|
+
await conn.rollback()
|
|
1797
|
+
raise
|
|
1798
|
+
if row is None:
|
|
1799
|
+
return None
|
|
1800
|
+
d = dict(zip(columns, row))
|
|
1801
|
+
try:
|
|
1802
|
+
payload = self._cipher.decrypt(d["payload"])
|
|
1803
|
+
except CipherError as exc:
|
|
1804
|
+
log.warning("dead-lettering undecryptable queue row %s: %s", d["id"], exc)
|
|
1805
|
+
await self.dead_letter_now(d["id"], f"undecryptable payload: {exc}")
|
|
1806
|
+
return None
|
|
1807
|
+
return OutboxItem(
|
|
1808
|
+
id=d["id"],
|
|
1809
|
+
message_id=d["message_id"],
|
|
1810
|
+
channel_id=d["channel_id"],
|
|
1811
|
+
destination_name=d["destination_name"],
|
|
1812
|
+
handler_name=d["handler_name"],
|
|
1813
|
+
payload=payload,
|
|
1814
|
+
attempts=d["attempts"],
|
|
1815
|
+
stage=stage,
|
|
1816
|
+
)
|
|
1817
|
+
|
|
1818
|
+
async def mark_done(self, outbox_id: str, now: float | None = None) -> None:
|
|
1819
|
+
now = time.time() if now is None else now
|
|
1820
|
+
async with self._acquire() as conn:
|
|
1821
|
+
cur = await conn.cursor()
|
|
1822
|
+
try:
|
|
1823
|
+
await cur.execute(
|
|
1824
|
+
"SELECT message_id, destination_name, attempts FROM queue WHERE id=?",
|
|
1825
|
+
(outbox_id,),
|
|
1826
|
+
)
|
|
1827
|
+
row = await cur.fetchone()
|
|
1828
|
+
if row is None:
|
|
1829
|
+
await conn.commit()
|
|
1830
|
+
return
|
|
1831
|
+
message_id, destination_name, attempts = row[0], row[1], row[2]
|
|
1832
|
+
await cur.execute(
|
|
1833
|
+
"UPDATE queue SET status=?, last_error=NULL, updated_at=? WHERE id=?",
|
|
1834
|
+
(OutboxStatus.DONE.value, now, outbox_id),
|
|
1835
|
+
)
|
|
1836
|
+
await self._event(
|
|
1837
|
+
cur, message_id, "delivered", destination_name, f"attempt {attempts}", now
|
|
1838
|
+
)
|
|
1839
|
+
await self._maybe_finalize(cur, message_id, now)
|
|
1840
|
+
await conn.commit()
|
|
1841
|
+
except Exception:
|
|
1842
|
+
await conn.rollback()
|
|
1843
|
+
raise
|
|
1844
|
+
|
|
1845
|
+
async def mark_failed(
|
|
1846
|
+
self, outbox_id: str, error: str, retry: RetryPolicy, now: float | None = None
|
|
1847
|
+
) -> None:
|
|
1848
|
+
error = safe_text(error) # PHI chokepoint (#120); SQL Server stores plaintext
|
|
1849
|
+
now = time.time() if now is None else now
|
|
1850
|
+
async with self._acquire() as conn:
|
|
1851
|
+
cur = await conn.cursor()
|
|
1852
|
+
try:
|
|
1853
|
+
await cur.execute(
|
|
1854
|
+
"SELECT message_id, destination_name, attempts FROM queue WHERE id=?",
|
|
1855
|
+
(outbox_id,),
|
|
1856
|
+
)
|
|
1857
|
+
row = await cur.fetchone()
|
|
1858
|
+
if row is None:
|
|
1859
|
+
await conn.commit()
|
|
1860
|
+
return
|
|
1861
|
+
message_id, destination_name, attempts = row[0], row[1], row[2]
|
|
1862
|
+
# max_attempts None = retry forever (never dead-letter here); a finite cap dead-letters
|
|
1863
|
+
# once exhausted (mirrors the SQLite backend's mark_failed).
|
|
1864
|
+
if retry.max_attempts is not None and attempts >= retry.max_attempts:
|
|
1865
|
+
status, next_at, event = OutboxStatus.DEAD.value, now, "dead"
|
|
1866
|
+
else:
|
|
1867
|
+
backoff = min(
|
|
1868
|
+
retry.max_backoff_seconds,
|
|
1869
|
+
retry.backoff_seconds * (retry.backoff_multiplier ** (attempts - 1)),
|
|
1870
|
+
)
|
|
1871
|
+
status, next_at, event = OutboxStatus.PENDING.value, now + backoff, "failed"
|
|
1872
|
+
await cur.execute(
|
|
1873
|
+
"UPDATE queue SET status=?, next_attempt_at=?, last_error=?, updated_at=? WHERE id=?",
|
|
1874
|
+
(status, next_at, error, now, outbox_id),
|
|
1875
|
+
)
|
|
1876
|
+
await self._event(
|
|
1877
|
+
cur, message_id, event, destination_name, f"attempt {attempts}: {error}", now
|
|
1878
|
+
)
|
|
1879
|
+
if status == OutboxStatus.DEAD.value:
|
|
1880
|
+
await self._maybe_finalize(cur, message_id, now)
|
|
1881
|
+
await conn.commit()
|
|
1882
|
+
except Exception:
|
|
1883
|
+
await conn.rollback()
|
|
1884
|
+
raise
|
|
1885
|
+
|
|
1886
|
+
# --- recovery / replay ---------------------------------------------------
|
|
1887
|
+
|
|
1888
|
+
async def reset_stale_inflight(
|
|
1889
|
+
self, now: float | None = None, *, stage: str | None = None
|
|
1890
|
+
) -> int:
|
|
1891
|
+
"""Return in-flight rows to ``pending`` (startup crash recovery) across ALL stages by default —
|
|
1892
|
+
an ingress/routed row left inflight by a crash MUST be re-pended or the message hangs forever
|
|
1893
|
+
(count-and-log invariant). ``stage`` optionally narrows it; owner/lease are cleared (single-node
|
|
1894
|
+
parity)."""
|
|
1895
|
+
now = time.time() if now is None else now
|
|
1896
|
+
clauses = ["status=?"]
|
|
1897
|
+
params: list[Any] = [OutboxStatus.INFLIGHT.value]
|
|
1898
|
+
if stage is not None:
|
|
1899
|
+
clauses.append("stage=?")
|
|
1900
|
+
params.append(stage)
|
|
1901
|
+
sql = (
|
|
1902
|
+
"UPDATE queue SET status=?, next_attempt_at=?, updated_at=?, owner=NULL,"
|
|
1903
|
+
f" lease_expires_at=NULL WHERE {' AND '.join(clauses)}"
|
|
1904
|
+
)
|
|
1905
|
+
async with self._acquire() as conn:
|
|
1906
|
+
cur = await conn.cursor()
|
|
1907
|
+
try:
|
|
1908
|
+
await cur.execute(sql, (OutboxStatus.PENDING.value, now, now, *params))
|
|
1909
|
+
count = cur.rowcount
|
|
1910
|
+
await conn.commit()
|
|
1911
|
+
except Exception:
|
|
1912
|
+
await conn.rollback()
|
|
1913
|
+
raise
|
|
1914
|
+
return int(count)
|
|
1915
|
+
|
|
1916
|
+
async def dead_letter_now(self, outbox_id: str, error: str, now: float | None = None) -> None:
|
|
1917
|
+
"""Force one row terminal (``DEAD``) immediately — fail-fast, no retry consumed. See the
|
|
1918
|
+
:meth:`~messagefoundry.store.base.QueueStore.dead_letter_now` contract."""
|
|
1919
|
+
error = safe_text(
|
|
1920
|
+
error
|
|
1921
|
+
) # PHI chokepoint (#120) — incl. f"undecryptable payload: {exc}" callers
|
|
1922
|
+
now = time.time() if now is None else now
|
|
1923
|
+
async with self._acquire() as conn:
|
|
1924
|
+
cur = await conn.cursor()
|
|
1925
|
+
try:
|
|
1926
|
+
await cur.execute(
|
|
1927
|
+
"SELECT message_id, destination_name FROM queue WHERE id=?", (outbox_id,)
|
|
1928
|
+
)
|
|
1929
|
+
row = await cur.fetchone()
|
|
1930
|
+
if row is None:
|
|
1931
|
+
await conn.commit()
|
|
1932
|
+
return
|
|
1933
|
+
message_id, destination_name = row[0], row[1]
|
|
1934
|
+
await cur.execute(
|
|
1935
|
+
"UPDATE queue SET status=?, next_attempt_at=?, last_error=?, updated_at=?,"
|
|
1936
|
+
" owner=NULL, lease_expires_at=NULL WHERE id=?",
|
|
1937
|
+
(OutboxStatus.DEAD.value, now, error, now, outbox_id),
|
|
1938
|
+
)
|
|
1939
|
+
await self._event(cur, message_id, "dead", destination_name, error, now)
|
|
1940
|
+
await self._maybe_finalize(cur, message_id, now)
|
|
1941
|
+
await conn.commit()
|
|
1942
|
+
except Exception:
|
|
1943
|
+
await conn.rollback()
|
|
1944
|
+
raise
|
|
1945
|
+
|
|
1946
|
+
async def pending_depth(
|
|
1947
|
+
self, name: str, *, stage: str = Stage.OUTBOUND.value
|
|
1948
|
+
) -> tuple[int, float | None]:
|
|
1949
|
+
"""``(pending_count, oldest_created_at)`` for one lane at ``stage`` (see the protocol). The lane
|
|
1950
|
+
key is stage-aware (``destination_name`` outbound, ``channel_id`` ingress/routed)."""
|
|
1951
|
+
lane_col = self._lane_col(stage) # code-controlled literal
|
|
1952
|
+
# Route through _fetchone (which commits) so we never return the pooled connection mid read-txn
|
|
1953
|
+
# under RCSI (M-6 read hygiene; mirrors postgres.py).
|
|
1954
|
+
row = await self._fetchone(
|
|
1955
|
+
f"SELECT COUNT(*) AS c, MIN(created_at) AS m FROM queue"
|
|
1956
|
+
f" WHERE stage=? AND {lane_col}=? AND status=?",
|
|
1957
|
+
(stage, name, OutboxStatus.PENDING.value),
|
|
1958
|
+
)
|
|
1959
|
+
count = int(row["c"]) if row is not None and row["c"] is not None else 0
|
|
1960
|
+
oldest = row["m"] if row is not None else None
|
|
1961
|
+
return count, (float(oldest) if oldest is not None else None)
|
|
1962
|
+
|
|
1963
|
+
async def dead_letter_missing_destinations(
|
|
1964
|
+
self, valid_names: set[str], now: float | None = None
|
|
1965
|
+
) -> int:
|
|
1966
|
+
"""Dead-letter non-terminal outbound queue rows whose destination_name is no longer in the
|
|
1967
|
+
registry (a removed/renamed outbound) — they have no delivery worker and would strand forever
|
|
1968
|
+
(H-5). The per-message finalize applocks are pre-acquired in sorted id order so two concurrent
|
|
1969
|
+
multi-message finalizers can't deadlock."""
|
|
1970
|
+
now = time.time() if now is None else now
|
|
1971
|
+
async with self._acquire() as conn:
|
|
1972
|
+
cur = await conn.cursor()
|
|
1973
|
+
try:
|
|
1974
|
+
await cur.execute(
|
|
1975
|
+
"SELECT id, message_id, destination_name FROM queue"
|
|
1976
|
+
" WHERE stage=? AND status IN (?, ?)",
|
|
1977
|
+
(Stage.OUTBOUND.value, OutboxStatus.PENDING.value, OutboxStatus.INFLIGHT.value),
|
|
1978
|
+
)
|
|
1979
|
+
rows = await cur.fetchall() # positional: (id, message_id, destination_name)
|
|
1980
|
+
orphans = [r for r in rows if r[2] not in valid_names]
|
|
1981
|
+
if not orphans:
|
|
1982
|
+
await conn.commit() # release the read txn cleanly (M-6)
|
|
1983
|
+
return 0
|
|
1984
|
+
error = "destination removed from outbound registry"
|
|
1985
|
+
await self._lock_finalize_batch(cur, {r[1] for r in orphans})
|
|
1986
|
+
for row in orphans:
|
|
1987
|
+
await cur.execute(
|
|
1988
|
+
"UPDATE queue SET status=?, next_attempt_at=?, last_error=?, updated_at=?,"
|
|
1989
|
+
" owner=NULL, lease_expires_at=NULL WHERE id=?",
|
|
1990
|
+
(OutboxStatus.DEAD.value, now, error, now, row[0]),
|
|
1991
|
+
)
|
|
1992
|
+
await self._event(cur, row[1], "dead", row[2], error, now)
|
|
1993
|
+
await self._maybe_finalize(cur, row[1], now)
|
|
1994
|
+
await conn.commit()
|
|
1995
|
+
except Exception:
|
|
1996
|
+
await conn.rollback()
|
|
1997
|
+
raise
|
|
1998
|
+
log.warning(
|
|
1999
|
+
"dead-lettered %d orphaned outbound row(s) at startup for missing destination(s): %s",
|
|
2000
|
+
len(orphans),
|
|
2001
|
+
", ".join(sorted({r[2] for r in orphans})),
|
|
2002
|
+
)
|
|
2003
|
+
return len(orphans)
|
|
2004
|
+
|
|
2005
|
+
async def replay(self, message_id: str, now: float | None = None) -> int:
|
|
2006
|
+
"""Re-queue a message's stuck/dead deliveries — or, if none are stuck, re-send the delivered
|
|
2007
|
+
ones. Two-mode (M-2): if any row is dead/pending, replay ONLY those (never re-fire a DONE
|
|
2008
|
+
sibling); else replay the done rows. messages.status -> RECEIVED if a pending ingress/routed
|
|
2009
|
+
row remains (needs re-routing), else ROUTED."""
|
|
2010
|
+
now = time.time() if now is None else now
|
|
2011
|
+
async with self._acquire() as conn:
|
|
2012
|
+
cur = await conn.cursor()
|
|
2013
|
+
try:
|
|
2014
|
+
await cur.execute(
|
|
2015
|
+
"SELECT COUNT(*) FROM queue WHERE message_id=? AND status IN (?, ?)",
|
|
2016
|
+
(message_id, OutboxStatus.DEAD.value, OutboxStatus.PENDING.value),
|
|
2017
|
+
)
|
|
2018
|
+
row = await cur.fetchone()
|
|
2019
|
+
stuck = int(row[0]) if row and row[0] is not None else 0
|
|
2020
|
+
replay_from = (
|
|
2021
|
+
(OutboxStatus.DEAD.value, OutboxStatus.PENDING.value)
|
|
2022
|
+
if stuck
|
|
2023
|
+
else (OutboxStatus.DONE.value,)
|
|
2024
|
+
)
|
|
2025
|
+
placeholders = ",".join("?" * len(replay_from))
|
|
2026
|
+
await cur.execute(
|
|
2027
|
+
f"UPDATE queue SET status=?, attempts=0, next_attempt_at=?, last_error=NULL,"
|
|
2028
|
+
f" updated_at=? WHERE message_id=? AND status IN ({placeholders})",
|
|
2029
|
+
(OutboxStatus.PENDING.value, now, now, message_id, *replay_from),
|
|
2030
|
+
)
|
|
2031
|
+
count = cur.rowcount
|
|
2032
|
+
if (
|
|
2033
|
+
count
|
|
2034
|
+
): # no rows => errored/filtered/unrouted: don't falsify it or strand it (M-2)
|
|
2035
|
+
await cur.execute(
|
|
2036
|
+
"SELECT 1 FROM queue WHERE message_id=? AND stage IN (?, ?) AND status=?",
|
|
2037
|
+
(
|
|
2038
|
+
message_id,
|
|
2039
|
+
Stage.INGRESS.value,
|
|
2040
|
+
Stage.ROUTED.value,
|
|
2041
|
+
OutboxStatus.PENDING.value,
|
|
2042
|
+
),
|
|
2043
|
+
)
|
|
2044
|
+
new_status = (
|
|
2045
|
+
MessageStatus.RECEIVED.value
|
|
2046
|
+
if await cur.fetchone()
|
|
2047
|
+
else MessageStatus.ROUTED.value
|
|
2048
|
+
)
|
|
2049
|
+
await cur.execute(
|
|
2050
|
+
"UPDATE messages SET status=?, error=NULL WHERE id=?",
|
|
2051
|
+
(new_status, message_id),
|
|
2052
|
+
)
|
|
2053
|
+
await self._event(
|
|
2054
|
+
cur, message_id, "replayed", None, f"{count} destination(s)", now
|
|
2055
|
+
)
|
|
2056
|
+
await conn.commit()
|
|
2057
|
+
except Exception:
|
|
2058
|
+
await conn.rollback()
|
|
2059
|
+
raise
|
|
2060
|
+
return int(count)
|
|
2061
|
+
|
|
2062
|
+
async def replay_dead(
|
|
2063
|
+
self,
|
|
2064
|
+
*,
|
|
2065
|
+
channel_id: str | None = None,
|
|
2066
|
+
destination_name: str | None = None,
|
|
2067
|
+
now: float | None = None,
|
|
2068
|
+
) -> int:
|
|
2069
|
+
now = time.time() if now is None else now
|
|
2070
|
+
where = ["stage=?", "status=?"]
|
|
2071
|
+
params: list[Any] = [Stage.OUTBOUND.value, OutboxStatus.DEAD.value]
|
|
2072
|
+
if channel_id is not None:
|
|
2073
|
+
where.append("channel_id=?")
|
|
2074
|
+
params.append(channel_id)
|
|
2075
|
+
if destination_name is not None:
|
|
2076
|
+
where.append("destination_name=?")
|
|
2077
|
+
params.append(destination_name)
|
|
2078
|
+
clause = " AND ".join(where)
|
|
2079
|
+
async with self._acquire() as conn:
|
|
2080
|
+
cur = await conn.cursor()
|
|
2081
|
+
try:
|
|
2082
|
+
await cur.execute(
|
|
2083
|
+
f"SELECT DISTINCT message_id FROM queue WHERE {clause}", tuple(params)
|
|
2084
|
+
)
|
|
2085
|
+
message_ids = [r[0] for r in await cur.fetchall()]
|
|
2086
|
+
if not message_ids:
|
|
2087
|
+
await conn.commit()
|
|
2088
|
+
return 0
|
|
2089
|
+
await cur.execute(
|
|
2090
|
+
f"UPDATE queue SET status=?, attempts=0, next_attempt_at=?, last_error=NULL,"
|
|
2091
|
+
f" updated_at=? WHERE {clause}",
|
|
2092
|
+
(OutboxStatus.PENDING.value, now, now, *params),
|
|
2093
|
+
)
|
|
2094
|
+
count = cur.rowcount
|
|
2095
|
+
for message_id in message_ids:
|
|
2096
|
+
await cur.execute(
|
|
2097
|
+
"UPDATE messages SET status=?, error=NULL WHERE id=? AND status=?",
|
|
2098
|
+
(MessageStatus.ROUTED.value, message_id, MessageStatus.ERROR.value),
|
|
2099
|
+
)
|
|
2100
|
+
await self._event(cur, message_id, "replayed", None, "dead-letter replay", now)
|
|
2101
|
+
await conn.commit()
|
|
2102
|
+
except Exception:
|
|
2103
|
+
await conn.rollback()
|
|
2104
|
+
raise
|
|
2105
|
+
return int(count)
|
|
2106
|
+
|
|
2107
|
+
async def cancel_queued(
|
|
2108
|
+
self,
|
|
2109
|
+
channel_id: str | None,
|
|
2110
|
+
destination_name: str,
|
|
2111
|
+
*,
|
|
2112
|
+
top_only: bool = False,
|
|
2113
|
+
now: float | None = None,
|
|
2114
|
+
) -> int:
|
|
2115
|
+
now = time.time() if now is None else now
|
|
2116
|
+
where = ["stage=?", "destination_name=?", "status=?"]
|
|
2117
|
+
params: list[Any] = [Stage.OUTBOUND.value, destination_name, OutboxStatus.PENDING.value]
|
|
2118
|
+
if channel_id is not None:
|
|
2119
|
+
where.insert(1, "channel_id=?")
|
|
2120
|
+
params.insert(1, channel_id)
|
|
2121
|
+
top = "TOP (1) " if top_only else ""
|
|
2122
|
+
async with self._acquire() as conn:
|
|
2123
|
+
cur = await conn.cursor()
|
|
2124
|
+
try:
|
|
2125
|
+
await cur.execute(
|
|
2126
|
+
f"SELECT {top}id, message_id FROM queue WHERE {' AND '.join(where)}"
|
|
2127
|
+
" ORDER BY next_attempt_at, created_at",
|
|
2128
|
+
tuple(params),
|
|
2129
|
+
)
|
|
2130
|
+
rows = [(r[0], r[1]) for r in await cur.fetchall()]
|
|
2131
|
+
if not rows:
|
|
2132
|
+
await conn.commit()
|
|
2133
|
+
return 0
|
|
2134
|
+
ids = [r[0] for r in rows]
|
|
2135
|
+
placeholders = ",".join("?" * len(ids))
|
|
2136
|
+
await cur.execute(
|
|
2137
|
+
f"UPDATE queue SET status=?, updated_at=? WHERE id IN ({placeholders})",
|
|
2138
|
+
(OutboxStatus.CANCELLED.value, now, *ids),
|
|
2139
|
+
)
|
|
2140
|
+
for _id, message_id in rows:
|
|
2141
|
+
await self._event(
|
|
2142
|
+
cur, message_id, "cancelled", destination_name, "manual purge", now
|
|
2143
|
+
)
|
|
2144
|
+
mids = {r[1] for r in rows}
|
|
2145
|
+
await self._lock_finalize_batch(cur, mids)
|
|
2146
|
+
for message_id in sorted(mids):
|
|
2147
|
+
await self._maybe_finalize(cur, message_id, now)
|
|
2148
|
+
await conn.commit()
|
|
2149
|
+
except Exception:
|
|
2150
|
+
await conn.rollback()
|
|
2151
|
+
raise
|
|
2152
|
+
return len(ids)
|
|
2153
|
+
|
|
2154
|
+
# --- read helpers --------------------------------------------------------
|
|
2155
|
+
|
|
2156
|
+
async def get_message(self, message_id: str) -> dict[str, Any] | None:
|
|
2157
|
+
record = await self._fetchone("SELECT * FROM messages WHERE id=?", (message_id,))
|
|
2158
|
+
if record is not None:
|
|
2159
|
+
record["raw"] = self._cipher.decrypt(record["raw"]) # decrypt the body for display
|
|
2160
|
+
return record
|
|
2161
|
+
|
|
2162
|
+
async def list_messages(
|
|
2163
|
+
self,
|
|
2164
|
+
*,
|
|
2165
|
+
channel_id: str | None = None,
|
|
2166
|
+
status: str | None = None,
|
|
2167
|
+
message_type: str | None = None,
|
|
2168
|
+
control_id: str | None = None,
|
|
2169
|
+
limit: int = 50,
|
|
2170
|
+
offset: int = 0,
|
|
2171
|
+
allowed_channels: Sequence[str] | None = None,
|
|
2172
|
+
) -> list[dict[str, Any]]:
|
|
2173
|
+
where, params = self._message_filter(
|
|
2174
|
+
channel_id, status, message_type, control_id, allowed_channels
|
|
2175
|
+
)
|
|
2176
|
+
return await self._fetchall(
|
|
2177
|
+
"SELECT id, channel_id, received_at, source_type, control_id, message_type,"
|
|
2178
|
+
" status, error, summary, metadata,"
|
|
2179
|
+
" (SELECT TOP 1 event FROM message_events e WHERE e.message_id = messages.id"
|
|
2180
|
+
" ORDER BY e.id DESC) AS last_event"
|
|
2181
|
+
f" FROM messages{where}"
|
|
2182
|
+
" ORDER BY received_at DESC, id DESC OFFSET ? ROWS FETCH NEXT ? ROWS ONLY",
|
|
2183
|
+
(*params, offset, limit),
|
|
2184
|
+
)
|
|
2185
|
+
|
|
2186
|
+
async def count_messages(
|
|
2187
|
+
self,
|
|
2188
|
+
*,
|
|
2189
|
+
channel_id: str | None = None,
|
|
2190
|
+
status: str | None = None,
|
|
2191
|
+
message_type: str | None = None,
|
|
2192
|
+
control_id: str | None = None,
|
|
2193
|
+
allowed_channels: Sequence[str] | None = None,
|
|
2194
|
+
) -> int:
|
|
2195
|
+
where, params = self._message_filter(
|
|
2196
|
+
channel_id, status, message_type, control_id, allowed_channels
|
|
2197
|
+
)
|
|
2198
|
+
row = await self._fetchone(f"SELECT COUNT(*) AS n FROM messages{where}", params)
|
|
2199
|
+
return int(row["n"]) if row else 0
|
|
2200
|
+
|
|
2201
|
+
async def list_dead(
|
|
2202
|
+
self,
|
|
2203
|
+
*,
|
|
2204
|
+
channel_id: str | None = None,
|
|
2205
|
+
destination_name: str | None = None,
|
|
2206
|
+
limit: int = 50,
|
|
2207
|
+
offset: int = 0,
|
|
2208
|
+
allowed_channels: Sequence[str] | None = None,
|
|
2209
|
+
) -> list[dict[str, Any]]:
|
|
2210
|
+
where, params = self._dead_filter(channel_id, destination_name, allowed_channels)
|
|
2211
|
+
return await self._fetchall(
|
|
2212
|
+
"SELECT o.id AS outbox_id, o.message_id, o.channel_id, o.destination_name,"
|
|
2213
|
+
" o.attempts, o.last_error, o.updated_at,"
|
|
2214
|
+
" m.control_id, m.message_type, m.received_at, m.summary"
|
|
2215
|
+
f" FROM queue o JOIN messages m ON m.id = o.message_id{where}"
|
|
2216
|
+
" ORDER BY o.updated_at DESC, o.id DESC OFFSET ? ROWS FETCH NEXT ? ROWS ONLY",
|
|
2217
|
+
(*params, offset, limit),
|
|
2218
|
+
)
|
|
2219
|
+
|
|
2220
|
+
async def count_dead(
|
|
2221
|
+
self,
|
|
2222
|
+
*,
|
|
2223
|
+
channel_id: str | None = None,
|
|
2224
|
+
destination_name: str | None = None,
|
|
2225
|
+
allowed_channels: Sequence[str] | None = None,
|
|
2226
|
+
) -> int:
|
|
2227
|
+
where, params = self._dead_filter(channel_id, destination_name, allowed_channels)
|
|
2228
|
+
row = await self._fetchone(f"SELECT COUNT(*) AS n FROM queue o{where}", params)
|
|
2229
|
+
return int(row["n"]) if row else 0
|
|
2230
|
+
|
|
2231
|
+
@staticmethod
|
|
2232
|
+
def _dead_filter(
|
|
2233
|
+
channel_id: str | None,
|
|
2234
|
+
destination_name: str | None,
|
|
2235
|
+
allowed_channels: Sequence[str] | None = None,
|
|
2236
|
+
) -> tuple[str, tuple[Any, ...]]:
|
|
2237
|
+
clauses = ["o.stage=?", "o.status=?"]
|
|
2238
|
+
params: list[Any] = [Stage.OUTBOUND.value, OutboxStatus.DEAD.value]
|
|
2239
|
+
if channel_id is not None:
|
|
2240
|
+
clauses.append("o.channel_id=?")
|
|
2241
|
+
params.append(channel_id)
|
|
2242
|
+
if destination_name is not None:
|
|
2243
|
+
clauses.append("o.destination_name=?")
|
|
2244
|
+
params.append(destination_name)
|
|
2245
|
+
_append_channel_scope(clauses, params, "o.channel_id", allowed_channels)
|
|
2246
|
+
return f" WHERE {' AND '.join(clauses)}", tuple(params)
|
|
2247
|
+
|
|
2248
|
+
async def outbox_for(self, message_id: str) -> list[dict[str, Any]]:
|
|
2249
|
+
return await self._fetchall(
|
|
2250
|
+
"SELECT * FROM queue WHERE message_id=? AND stage=? ORDER BY destination_name",
|
|
2251
|
+
(message_id, Stage.OUTBOUND.value),
|
|
2252
|
+
)
|
|
2253
|
+
|
|
2254
|
+
async def outbox_payloads_for(self, message_id: str) -> list[dict[str, Any]]:
|
|
2255
|
+
"""Like :meth:`outbox_for`, but also decrypts the transformed ``payload`` (PHI body) per
|
|
2256
|
+
destination for the parity-comparison read path (#14). The outbound ``payload`` column is the
|
|
2257
|
+
encrypted body directly (no artifact indirection at this stage — cf. :meth:`claim_ready`); the
|
|
2258
|
+
API gates this on ``MESSAGES_VIEW_RAW`` and audits it."""
|
|
2259
|
+
rows = await self._fetchall(
|
|
2260
|
+
"SELECT * FROM queue WHERE message_id=? AND stage=? ORDER BY destination_name",
|
|
2261
|
+
(message_id, Stage.OUTBOUND.value),
|
|
2262
|
+
)
|
|
2263
|
+
for r in rows:
|
|
2264
|
+
r["payload"] = self._cipher.decrypt(r["payload"])
|
|
2265
|
+
if r["last_error"] is not None:
|
|
2266
|
+
r["last_error"] = self._cipher.decrypt(r["last_error"])
|
|
2267
|
+
return rows
|
|
2268
|
+
|
|
2269
|
+
async def events_for(self, message_id: str) -> list[dict[str, Any]]:
|
|
2270
|
+
return await self._fetchall(
|
|
2271
|
+
"SELECT * FROM message_events WHERE message_id=? ORDER BY id", (message_id,)
|
|
2272
|
+
)
|
|
2273
|
+
|
|
2274
|
+
async def record_view(
|
|
2275
|
+
self, message_id: str, *, actor: str | None = None, now: float | None = None
|
|
2276
|
+
) -> None:
|
|
2277
|
+
now = time.time() if now is None else now
|
|
2278
|
+
async with self._acquire() as conn:
|
|
2279
|
+
cur = await conn.cursor()
|
|
2280
|
+
try:
|
|
2281
|
+
await self._event(cur, message_id, "viewed", None, actor or "", now)
|
|
2282
|
+
await conn.commit()
|
|
2283
|
+
except Exception:
|
|
2284
|
+
await conn.rollback()
|
|
2285
|
+
raise
|
|
2286
|
+
|
|
2287
|
+
async def record_audit(
|
|
2288
|
+
self,
|
|
2289
|
+
action: str,
|
|
2290
|
+
*,
|
|
2291
|
+
actor: str | None = None,
|
|
2292
|
+
channel_id: str | None = None,
|
|
2293
|
+
detail: str | None = None,
|
|
2294
|
+
now: float | None = None,
|
|
2295
|
+
) -> None:
|
|
2296
|
+
now = time.time() if now is None else now
|
|
2297
|
+
# Serialize the read-prev-then-insert append in-process so two concurrent audited actions can't
|
|
2298
|
+
# read the same prev hash and FORK the hash chain (H-7). The store is the single audit writer
|
|
2299
|
+
# per engine process (active-passive = one active node), so an in-process lock is sufficient and
|
|
2300
|
+
# reliable — unlike a txn-scoped sp_getapplock taken as the connection's first statement, which
|
|
2301
|
+
# does not release on commit and strands under concurrent contention.
|
|
2302
|
+
async with self._audit_lock:
|
|
2303
|
+
async with self._acquire() as conn:
|
|
2304
|
+
cur = await conn.cursor()
|
|
2305
|
+
try:
|
|
2306
|
+
await cur.execute("SELECT TOP (1) row_hash FROM audit_log ORDER BY id DESC")
|
|
2307
|
+
last = await cur.fetchone()
|
|
2308
|
+
prev = last[0] if last and last[0] else ""
|
|
2309
|
+
row_hash = audit_row_hash(
|
|
2310
|
+
prev,
|
|
2311
|
+
ts=now,
|
|
2312
|
+
actor=actor,
|
|
2313
|
+
action=action,
|
|
2314
|
+
channel_id=channel_id,
|
|
2315
|
+
detail=detail,
|
|
2316
|
+
)
|
|
2317
|
+
await cur.execute(
|
|
2318
|
+
"INSERT INTO audit_log (ts, actor, action, channel_id, detail, row_hash)"
|
|
2319
|
+
" VALUES (?,?,?,?,?,?)",
|
|
2320
|
+
(now, actor, action, channel_id, detail, row_hash),
|
|
2321
|
+
)
|
|
2322
|
+
await conn.commit()
|
|
2323
|
+
except Exception:
|
|
2324
|
+
await conn.rollback()
|
|
2325
|
+
raise
|
|
2326
|
+
# Tee off-box AFTER commit + outside the audit lock / pooled connection (only forward what
|
|
2327
|
+
# truly persisted; a synchronous syslog send must never hold the lock). Shared redaction path.
|
|
2328
|
+
emit_audit_tee(action=action, actor=actor, channel_id=channel_id, detail=detail, ts=now)
|
|
2329
|
+
|
|
2330
|
+
async def audit_anchor(self) -> tuple[int, str]:
|
|
2331
|
+
"""The audit log's external anchor — ``(row_count, head_hash)`` — see the SQLite store (low-1)."""
|
|
2332
|
+
rows = await self._fetchall(
|
|
2333
|
+
"SELECT COUNT(*) AS n, "
|
|
2334
|
+
"(SELECT TOP (1) row_hash FROM audit_log ORDER BY id DESC) AS head FROM audit_log"
|
|
2335
|
+
)
|
|
2336
|
+
if not rows:
|
|
2337
|
+
return 0, ""
|
|
2338
|
+
return int(rows[0]["n"]), (rows[0]["head"] or "")
|
|
2339
|
+
|
|
2340
|
+
async def verify_audit_chain(
|
|
2341
|
+
self, *, expected_anchor: tuple[int, str] | None = None
|
|
2342
|
+
) -> tuple[bool, str | None]:
|
|
2343
|
+
"""Recompute the audit hash-chain in order; returns (ok, message) — see the SQLite store.
|
|
2344
|
+
|
|
2345
|
+
Re-walking can't catch tail-truncation (the surviving prefix still verifies); pass
|
|
2346
|
+
``expected_anchor`` from :meth:`audit_anchor`, held out-of-band, to detect it (review low-1)."""
|
|
2347
|
+
rows = await self._fetchall(
|
|
2348
|
+
"SELECT id, ts, actor, action, channel_id, detail, row_hash FROM audit_log ORDER BY id"
|
|
2349
|
+
)
|
|
2350
|
+
prev = ""
|
|
2351
|
+
for r in rows:
|
|
2352
|
+
expected = audit_row_hash(
|
|
2353
|
+
prev,
|
|
2354
|
+
ts=r["ts"],
|
|
2355
|
+
actor=r["actor"],
|
|
2356
|
+
action=r["action"],
|
|
2357
|
+
channel_id=r["channel_id"],
|
|
2358
|
+
detail=r["detail"],
|
|
2359
|
+
)
|
|
2360
|
+
if r["row_hash"] != expected:
|
|
2361
|
+
return False, f"audit chain broken at row id={r['id']}"
|
|
2362
|
+
prev = r["row_hash"]
|
|
2363
|
+
if expected_anchor is not None:
|
|
2364
|
+
exp_count, exp_head = expected_anchor
|
|
2365
|
+
if len(rows) < exp_count or prev != exp_head:
|
|
2366
|
+
return (
|
|
2367
|
+
False,
|
|
2368
|
+
f"audit log diverges from recorded anchor (have {len(rows)} row(s) head "
|
|
2369
|
+
f"{prev[:12]!r}, expected {exp_count} head {exp_head[:12]!r}) — truncated or rewritten",
|
|
2370
|
+
)
|
|
2371
|
+
return True, f"verified {len(rows)} audit row(s)"
|
|
2372
|
+
|
|
2373
|
+
# --- auth: users / roles / sessions --------------------------------------
|
|
2374
|
+
|
|
2375
|
+
async def list_audit(self, *, limit: int = 50) -> list[dict[str, Any]]:
|
|
2376
|
+
return await self._fetchall("SELECT TOP (?) * FROM audit_log ORDER BY id DESC", (limit,))
|
|
2377
|
+
|
|
2378
|
+
async def security_events_for_user(
|
|
2379
|
+
self, username: str, *, limit: int = 100
|
|
2380
|
+
) -> list[dict[str, Any]]:
|
|
2381
|
+
"""A user's own security events (``auth.*``), most-recent-first — for ``GET
|
|
2382
|
+
/me/security-events`` (ASVS 6.3.5/6.3.7); admin-initiated changes go out-of-band by email."""
|
|
2383
|
+
return await self._fetchall(
|
|
2384
|
+
"SELECT TOP (?) ts, action, detail FROM audit_log "
|
|
2385
|
+
"WHERE actor = ? AND action LIKE 'auth.%' ORDER BY id DESC",
|
|
2386
|
+
(limit, username),
|
|
2387
|
+
)
|
|
2388
|
+
|
|
2389
|
+
# --- dual-control approvals (ASVS 2.3.5) ---------------------------------
|
|
2390
|
+
|
|
2391
|
+
async def create_pending_approval(
|
|
2392
|
+
self,
|
|
2393
|
+
*,
|
|
2394
|
+
approval_id: str,
|
|
2395
|
+
operation: str,
|
|
2396
|
+
params: str,
|
|
2397
|
+
requester: str,
|
|
2398
|
+
requested_at: float,
|
|
2399
|
+
expires_at: float | None,
|
|
2400
|
+
) -> None:
|
|
2401
|
+
"""Persist a high-value action awaiting a distinct second approver (dual-control, 2.3.5)."""
|
|
2402
|
+
await self._execute(
|
|
2403
|
+
"INSERT INTO pending_approvals "
|
|
2404
|
+
"(id, operation, params, requester, requested_at, status, expires_at) "
|
|
2405
|
+
"VALUES (?,?,?,?,?,'pending',?)",
|
|
2406
|
+
(approval_id, operation, params, requester, requested_at, expires_at),
|
|
2407
|
+
)
|
|
2408
|
+
|
|
2409
|
+
async def get_pending_approval(self, approval_id: str) -> dict[str, Any] | None:
|
|
2410
|
+
return await self._fetchone(
|
|
2411
|
+
"SELECT id, operation, params, requester, requested_at, status, approver, decided_at,"
|
|
2412
|
+
" expires_at FROM pending_approvals WHERE id = ?",
|
|
2413
|
+
(approval_id,),
|
|
2414
|
+
)
|
|
2415
|
+
|
|
2416
|
+
async def list_pending_approvals(self, *, now: float, limit: int = 100) -> list[dict[str, Any]]:
|
|
2417
|
+
"""Open (still-``pending``, unexpired) approval requests, newest-first."""
|
|
2418
|
+
return await self._fetchall(
|
|
2419
|
+
"SELECT TOP (?) id, operation, params, requester, requested_at, status, approver,"
|
|
2420
|
+
" decided_at, expires_at FROM pending_approvals"
|
|
2421
|
+
" WHERE status = 'pending' AND (expires_at IS NULL OR expires_at > ?)"
|
|
2422
|
+
" ORDER BY requested_at DESC",
|
|
2423
|
+
(limit, now),
|
|
2424
|
+
)
|
|
2425
|
+
|
|
2426
|
+
async def decide_pending_approval(
|
|
2427
|
+
self, approval_id: str, *, status: str, approver: str | None, decided_at: float
|
|
2428
|
+
) -> bool:
|
|
2429
|
+
"""Atomically move a still-``pending`` request to ``status`` (approved/rejected/expired).
|
|
2430
|
+
Returns ``True`` iff this call made the transition — guards against a double decision."""
|
|
2431
|
+
async with self._acquire() as conn:
|
|
2432
|
+
cur = await conn.cursor()
|
|
2433
|
+
try:
|
|
2434
|
+
await cur.execute(
|
|
2435
|
+
"UPDATE pending_approvals SET status = ?, approver = ?, decided_at = ?"
|
|
2436
|
+
" WHERE id = ? AND status = 'pending'",
|
|
2437
|
+
(status, approver, decided_at, approval_id),
|
|
2438
|
+
)
|
|
2439
|
+
count = cur.rowcount
|
|
2440
|
+
await conn.commit()
|
|
2441
|
+
except Exception:
|
|
2442
|
+
await conn.rollback()
|
|
2443
|
+
raise
|
|
2444
|
+
return int(count) > 0
|
|
2445
|
+
|
|
2446
|
+
async def create_user(
|
|
2447
|
+
self,
|
|
2448
|
+
*,
|
|
2449
|
+
user_id: str,
|
|
2450
|
+
username: str,
|
|
2451
|
+
auth_provider: str,
|
|
2452
|
+
display_name: str | None = None,
|
|
2453
|
+
email: str | None = None,
|
|
2454
|
+
password_hash: str | None = None,
|
|
2455
|
+
must_change_password: bool = False,
|
|
2456
|
+
now: float | None = None,
|
|
2457
|
+
) -> None:
|
|
2458
|
+
now = time.time() if now is None else now
|
|
2459
|
+
await self._execute(
|
|
2460
|
+
"INSERT INTO users (id, username, auth_provider, display_name, email, disabled,"
|
|
2461
|
+
" created_at, updated_at, last_login_at, password_hash, password_changed_at,"
|
|
2462
|
+
" must_change_password, failed_attempts, locked_until)"
|
|
2463
|
+
" VALUES (?,?,?,?,?,0,?,?,NULL,?,?,?,0,NULL)",
|
|
2464
|
+
(
|
|
2465
|
+
user_id,
|
|
2466
|
+
username,
|
|
2467
|
+
auth_provider,
|
|
2468
|
+
display_name,
|
|
2469
|
+
email,
|
|
2470
|
+
now,
|
|
2471
|
+
now,
|
|
2472
|
+
password_hash,
|
|
2473
|
+
now if password_hash is not None else None,
|
|
2474
|
+
1 if must_change_password else 0,
|
|
2475
|
+
),
|
|
2476
|
+
)
|
|
2477
|
+
|
|
2478
|
+
async def get_user(self, user_id: str) -> UserRecord | None:
|
|
2479
|
+
d = await self._fetchone("SELECT * FROM users WHERE id=?", (user_id,))
|
|
2480
|
+
return UserRecord.from_mapping(d) if d else None
|
|
2481
|
+
|
|
2482
|
+
async def get_user_by_username(self, username: str) -> UserRecord | None:
|
|
2483
|
+
d = await self._fetchone("SELECT * FROM users WHERE username=?", (username,))
|
|
2484
|
+
return UserRecord.from_mapping(d) if d else None
|
|
2485
|
+
|
|
2486
|
+
async def list_users(self) -> list[UserRecord]:
|
|
2487
|
+
rows = await self._fetchall("SELECT * FROM users ORDER BY username")
|
|
2488
|
+
return [UserRecord.from_mapping(d) for d in rows]
|
|
2489
|
+
|
|
2490
|
+
async def count_users(self) -> int:
|
|
2491
|
+
return await self._count("users")
|
|
2492
|
+
|
|
2493
|
+
async def set_password(
|
|
2494
|
+
self,
|
|
2495
|
+
user_id: str,
|
|
2496
|
+
*,
|
|
2497
|
+
password_hash: str,
|
|
2498
|
+
must_change_password: bool = False,
|
|
2499
|
+
now: float | None = None,
|
|
2500
|
+
) -> None:
|
|
2501
|
+
now = time.time() if now is None else now
|
|
2502
|
+
await self._execute(
|
|
2503
|
+
"UPDATE users SET password_hash=?, password_changed_at=?, must_change_password=?,"
|
|
2504
|
+
" failed_attempts=0, locked_until=NULL, updated_at=? WHERE id=?",
|
|
2505
|
+
(password_hash, now, 1 if must_change_password else 0, now, user_id),
|
|
2506
|
+
)
|
|
2507
|
+
|
|
2508
|
+
# --- MFA: native TOTP second factor (local accounts, WP-14) --------------
|
|
2509
|
+
|
|
2510
|
+
async def set_totp_secret(
|
|
2511
|
+
self, user_id: str, *, secret: str | None, now: float | None = None
|
|
2512
|
+
) -> None:
|
|
2513
|
+
"""Stage (or clear) a user's base32 TOTP secret, store-cipher encrypted. Does not enable MFA."""
|
|
2514
|
+
now = time.time() if now is None else now
|
|
2515
|
+
enc = self._cipher.encrypt(secret) if secret else None
|
|
2516
|
+
await self._execute(
|
|
2517
|
+
"UPDATE users SET totp_secret=?, updated_at=? WHERE id=?", (enc, now, user_id)
|
|
2518
|
+
)
|
|
2519
|
+
|
|
2520
|
+
async def get_totp_secret(self, user_id: str) -> str | None:
|
|
2521
|
+
d = await self._fetchone("SELECT totp_secret FROM users WHERE id=?", (user_id,))
|
|
2522
|
+
if not d or d["totp_secret"] is None:
|
|
2523
|
+
return None
|
|
2524
|
+
return self._cipher.decrypt(d["totp_secret"])
|
|
2525
|
+
|
|
2526
|
+
async def enable_totp(
|
|
2527
|
+
self, user_id: str, *, recovery_code_hashes: list[str], now: float | None = None
|
|
2528
|
+
) -> None:
|
|
2529
|
+
now = time.time() if now is None else now
|
|
2530
|
+
await self._execute(
|
|
2531
|
+
"UPDATE users SET totp_enabled=1, totp_enrolled_at=?, totp_recovery_codes=?,"
|
|
2532
|
+
" updated_at=? WHERE id=?",
|
|
2533
|
+
(now, json.dumps(recovery_code_hashes), now, user_id),
|
|
2534
|
+
)
|
|
2535
|
+
|
|
2536
|
+
async def disable_totp(self, user_id: str, *, now: float | None = None) -> None:
|
|
2537
|
+
now = time.time() if now is None else now
|
|
2538
|
+
await self._execute(
|
|
2539
|
+
"UPDATE users SET totp_secret=NULL, totp_enabled=0, totp_enrolled_at=NULL,"
|
|
2540
|
+
" totp_recovery_codes=NULL, updated_at=? WHERE id=?",
|
|
2541
|
+
(now, user_id),
|
|
2542
|
+
)
|
|
2543
|
+
|
|
2544
|
+
async def get_recovery_code_hashes(self, user_id: str) -> list[str]:
|
|
2545
|
+
d = await self._fetchone("SELECT totp_recovery_codes FROM users WHERE id=?", (user_id,))
|
|
2546
|
+
if not d or d["totp_recovery_codes"] is None:
|
|
2547
|
+
return []
|
|
2548
|
+
return [str(h) for h in json.loads(d["totp_recovery_codes"])]
|
|
2549
|
+
|
|
2550
|
+
async def consume_recovery_code_hash(
|
|
2551
|
+
self, user_id: str, code_hash: str, *, now: float | None = None
|
|
2552
|
+
) -> bool:
|
|
2553
|
+
"""Atomically remove one recovery-code hash; ``True`` iff present. The ``UPDLOCK`` SELECT +
|
|
2554
|
+
UPDATE run in one transaction, so concurrent verifications can't double-spend a single-use
|
|
2555
|
+
recovery code (WP-14)."""
|
|
2556
|
+
now = time.time() if now is None else now
|
|
2557
|
+
async with self._acquire() as conn:
|
|
2558
|
+
cur = await conn.cursor()
|
|
2559
|
+
try:
|
|
2560
|
+
await cur.execute(
|
|
2561
|
+
"SELECT totp_recovery_codes FROM users WITH (UPDLOCK, ROWLOCK) WHERE id=?",
|
|
2562
|
+
(user_id,),
|
|
2563
|
+
)
|
|
2564
|
+
row = await cur.fetchone()
|
|
2565
|
+
raw = row[0] if row else None
|
|
2566
|
+
if raw is None:
|
|
2567
|
+
await conn.commit()
|
|
2568
|
+
return False
|
|
2569
|
+
hashes = [str(h) for h in json.loads(raw)]
|
|
2570
|
+
if code_hash not in hashes:
|
|
2571
|
+
await conn.commit()
|
|
2572
|
+
return False # already consumed by a concurrent caller
|
|
2573
|
+
hashes.remove(code_hash)
|
|
2574
|
+
await cur.execute(
|
|
2575
|
+
"UPDATE users SET totp_recovery_codes=?, updated_at=? WHERE id=?",
|
|
2576
|
+
(json.dumps(hashes), now, user_id),
|
|
2577
|
+
)
|
|
2578
|
+
await conn.commit()
|
|
2579
|
+
return True
|
|
2580
|
+
except Exception:
|
|
2581
|
+
await conn.rollback()
|
|
2582
|
+
raise
|
|
2583
|
+
|
|
2584
|
+
async def consume_totp_step(self, user_id: str, step: int) -> bool:
|
|
2585
|
+
"""Atomically record ``step`` as the user's highest consumed TOTP time-step; ``True`` iff newly
|
|
2586
|
+
consumed (strictly greater than any prior step). A code replayed inside its ±1-step verify
|
|
2587
|
+
window resolves to a non-greater step and returns ``False`` — single-use per ASVS 6.5.1. The
|
|
2588
|
+
``UPDLOCK`` SELECT + UPDATE run in one transaction so concurrent verifications can't both win."""
|
|
2589
|
+
async with self._acquire() as conn:
|
|
2590
|
+
cur = await conn.cursor()
|
|
2591
|
+
try:
|
|
2592
|
+
await cur.execute(
|
|
2593
|
+
"SELECT last_totp_step FROM users WITH (UPDLOCK, ROWLOCK) WHERE id=?",
|
|
2594
|
+
(user_id,),
|
|
2595
|
+
)
|
|
2596
|
+
row = await cur.fetchone()
|
|
2597
|
+
if row is None:
|
|
2598
|
+
await conn.commit()
|
|
2599
|
+
return False
|
|
2600
|
+
last = row[0]
|
|
2601
|
+
if last is not None and last >= step:
|
|
2602
|
+
await conn.commit()
|
|
2603
|
+
return False # already consumed (or an older step) — replay within the window
|
|
2604
|
+
await cur.execute("UPDATE users SET last_totp_step=? WHERE id=?", (step, user_id))
|
|
2605
|
+
await conn.commit()
|
|
2606
|
+
return True
|
|
2607
|
+
except Exception:
|
|
2608
|
+
await conn.rollback()
|
|
2609
|
+
raise
|
|
2610
|
+
|
|
2611
|
+
async def set_user_disabled(
|
|
2612
|
+
self, user_id: str, *, disabled: bool, now: float | None = None
|
|
2613
|
+
) -> None:
|
|
2614
|
+
now = time.time() if now is None else now
|
|
2615
|
+
await self._execute(
|
|
2616
|
+
"UPDATE users SET disabled=?, updated_at=? WHERE id=?",
|
|
2617
|
+
(1 if disabled else 0, now, user_id),
|
|
2618
|
+
)
|
|
2619
|
+
|
|
2620
|
+
async def update_user_profile(
|
|
2621
|
+
self,
|
|
2622
|
+
user_id: str,
|
|
2623
|
+
*,
|
|
2624
|
+
display_name: str | None,
|
|
2625
|
+
email: str | None,
|
|
2626
|
+
now: float | None = None,
|
|
2627
|
+
) -> None:
|
|
2628
|
+
now = time.time() if now is None else now
|
|
2629
|
+
await self._execute(
|
|
2630
|
+
"UPDATE users SET display_name=?, email=?, updated_at=? WHERE id=?",
|
|
2631
|
+
(display_name, email, now, user_id),
|
|
2632
|
+
)
|
|
2633
|
+
|
|
2634
|
+
async def delete_user(self, user_id: str) -> None:
|
|
2635
|
+
async with self._acquire() as conn:
|
|
2636
|
+
cur = await conn.cursor()
|
|
2637
|
+
try:
|
|
2638
|
+
await cur.execute("DELETE FROM user_roles WHERE user_id=?", (user_id,))
|
|
2639
|
+
await cur.execute("DELETE FROM sessions WHERE user_id=?", (user_id,))
|
|
2640
|
+
await cur.execute("DELETE FROM users WHERE id=?", (user_id,))
|
|
2641
|
+
await conn.commit()
|
|
2642
|
+
except Exception:
|
|
2643
|
+
await conn.rollback()
|
|
2644
|
+
raise
|
|
2645
|
+
|
|
2646
|
+
async def record_login_success(self, user_id: str, *, now: float | None = None) -> None:
|
|
2647
|
+
now = time.time() if now is None else now
|
|
2648
|
+
await self._execute(
|
|
2649
|
+
"UPDATE users SET last_login_at=?, failed_attempts=0, locked_until=NULL,"
|
|
2650
|
+
" updated_at=? WHERE id=?",
|
|
2651
|
+
(now, now, user_id),
|
|
2652
|
+
)
|
|
2653
|
+
|
|
2654
|
+
async def record_login_failure(
|
|
2655
|
+
self,
|
|
2656
|
+
user_id: str,
|
|
2657
|
+
*,
|
|
2658
|
+
failed_attempts: int,
|
|
2659
|
+
locked_until: float | None,
|
|
2660
|
+
now: float | None = None,
|
|
2661
|
+
) -> None:
|
|
2662
|
+
now = time.time() if now is None else now
|
|
2663
|
+
await self._execute(
|
|
2664
|
+
"UPDATE users SET failed_attempts=?, locked_until=?, updated_at=? WHERE id=?",
|
|
2665
|
+
(failed_attempts, locked_until, now, user_id),
|
|
2666
|
+
)
|
|
2667
|
+
|
|
2668
|
+
async def upsert_role(
|
|
2669
|
+
self,
|
|
2670
|
+
*,
|
|
2671
|
+
role_id: str,
|
|
2672
|
+
display_name: str,
|
|
2673
|
+
description: str | None = None,
|
|
2674
|
+
builtin: bool = True,
|
|
2675
|
+
) -> None:
|
|
2676
|
+
# Single atomic MERGE under HOLDLOCK (range-locks the key) so two concurrent seeders can't both
|
|
2677
|
+
# find the row absent and both INSERT the same PK -> violation (the UPDATE-then-INSERT race).
|
|
2678
|
+
await self._execute(
|
|
2679
|
+
"MERGE roles WITH (HOLDLOCK) AS t"
|
|
2680
|
+
" USING (SELECT ? AS id, ? AS display_name, ? AS description, ? AS builtin) AS s"
|
|
2681
|
+
" ON t.id=s.id"
|
|
2682
|
+
" WHEN MATCHED THEN UPDATE SET display_name=s.display_name,"
|
|
2683
|
+
" description=s.description, builtin=s.builtin"
|
|
2684
|
+
" WHEN NOT MATCHED THEN INSERT (id, display_name, description, builtin)"
|
|
2685
|
+
" VALUES (s.id, s.display_name, s.description, s.builtin);",
|
|
2686
|
+
(role_id, display_name, description, 1 if builtin else 0),
|
|
2687
|
+
)
|
|
2688
|
+
|
|
2689
|
+
async def list_roles(self) -> list[dict[str, Any]]:
|
|
2690
|
+
return await self._fetchall("SELECT * FROM roles ORDER BY id")
|
|
2691
|
+
|
|
2692
|
+
async def get_user_role_ids(self, user_id: str) -> list[str]:
|
|
2693
|
+
rows = await self._fetchall(
|
|
2694
|
+
"SELECT role_id FROM user_roles WHERE user_id=? ORDER BY role_id", (user_id,)
|
|
2695
|
+
)
|
|
2696
|
+
return [str(r["role_id"]) for r in rows]
|
|
2697
|
+
|
|
2698
|
+
async def set_user_roles(
|
|
2699
|
+
self,
|
|
2700
|
+
user_id: str,
|
|
2701
|
+
role_ids: Sequence[str],
|
|
2702
|
+
*,
|
|
2703
|
+
assigned_by: str | None = None,
|
|
2704
|
+
now: float | None = None,
|
|
2705
|
+
) -> None:
|
|
2706
|
+
now = time.time() if now is None else now
|
|
2707
|
+
async with self._acquire() as conn:
|
|
2708
|
+
cur = await conn.cursor()
|
|
2709
|
+
try:
|
|
2710
|
+
await cur.execute("DELETE FROM user_roles WHERE user_id=?", (user_id,))
|
|
2711
|
+
for role_id in role_ids:
|
|
2712
|
+
await cur.execute(
|
|
2713
|
+
"INSERT INTO user_roles (user_id, role_id, assigned_at, assigned_by)"
|
|
2714
|
+
" VALUES (?,?,?,?)",
|
|
2715
|
+
(user_id, role_id, now, assigned_by),
|
|
2716
|
+
)
|
|
2717
|
+
await conn.commit()
|
|
2718
|
+
except Exception:
|
|
2719
|
+
await conn.rollback()
|
|
2720
|
+
raise
|
|
2721
|
+
|
|
2722
|
+
async def set_user_channel_scope(
|
|
2723
|
+
self, user_id: str, scope_json: str | None, *, now: float | None = None
|
|
2724
|
+
) -> None:
|
|
2725
|
+
now = time.time() if now is None else now
|
|
2726
|
+
await self._execute(
|
|
2727
|
+
"UPDATE users SET channel_scope=?, updated_at=? WHERE id=?",
|
|
2728
|
+
(scope_json, now, user_id),
|
|
2729
|
+
)
|
|
2730
|
+
|
|
2731
|
+
async def roles_for_ad_groups(self, groups: Iterable[str]) -> set[str]:
|
|
2732
|
+
normalized = sorted({g.strip().lower() for g in groups if g.strip()})
|
|
2733
|
+
if not normalized:
|
|
2734
|
+
return set()
|
|
2735
|
+
placeholders = ",".join("?" * len(normalized)) # count-bound, not user text
|
|
2736
|
+
rows = await self._fetchall(
|
|
2737
|
+
f"SELECT DISTINCT role_id FROM ad_group_role_map WHERE ad_group IN ({placeholders})",
|
|
2738
|
+
tuple(normalized),
|
|
2739
|
+
)
|
|
2740
|
+
return {str(r["role_id"]) for r in rows}
|
|
2741
|
+
|
|
2742
|
+
async def list_ad_group_role_map(self) -> list[dict[str, Any]]:
|
|
2743
|
+
return await self._fetchall(
|
|
2744
|
+
"SELECT ad_group, role_id FROM ad_group_role_map ORDER BY ad_group, role_id"
|
|
2745
|
+
)
|
|
2746
|
+
|
|
2747
|
+
async def set_ad_group_role_map(self, entries: Iterable[tuple[str, str]]) -> None:
|
|
2748
|
+
pairs = sorted({(g.strip().lower(), r) for g, r in entries if g.strip()})
|
|
2749
|
+
async with self._acquire() as conn:
|
|
2750
|
+
cur = await conn.cursor()
|
|
2751
|
+
try:
|
|
2752
|
+
await cur.execute("DELETE FROM ad_group_role_map")
|
|
2753
|
+
for ad_group, role_id in pairs:
|
|
2754
|
+
await cur.execute(
|
|
2755
|
+
"INSERT INTO ad_group_role_map (ad_group, role_id) VALUES (?,?)",
|
|
2756
|
+
(ad_group, role_id),
|
|
2757
|
+
)
|
|
2758
|
+
await conn.commit()
|
|
2759
|
+
except Exception:
|
|
2760
|
+
await conn.rollback()
|
|
2761
|
+
raise
|
|
2762
|
+
|
|
2763
|
+
async def channels_for_ad_groups(self, groups: Iterable[str]) -> set[str]:
|
|
2764
|
+
normalized = sorted({g.strip().lower() for g in groups if g.strip()})
|
|
2765
|
+
if not normalized:
|
|
2766
|
+
return set()
|
|
2767
|
+
placeholders = ",".join("?" * len(normalized)) # count-bound, not user text
|
|
2768
|
+
rows = await self._fetchall(
|
|
2769
|
+
f"SELECT DISTINCT channel FROM ad_group_scope_map WHERE ad_group IN ({placeholders})",
|
|
2770
|
+
tuple(normalized),
|
|
2771
|
+
)
|
|
2772
|
+
return {str(r["channel"]) for r in rows}
|
|
2773
|
+
|
|
2774
|
+
async def list_ad_group_scope_map(self) -> list[dict[str, Any]]:
|
|
2775
|
+
return await self._fetchall(
|
|
2776
|
+
"SELECT ad_group, channel FROM ad_group_scope_map ORDER BY ad_group, channel"
|
|
2777
|
+
)
|
|
2778
|
+
|
|
2779
|
+
async def set_ad_group_scope_map(self, entries: Iterable[tuple[str, str]]) -> None:
|
|
2780
|
+
pairs = sorted(
|
|
2781
|
+
{(g.strip().lower(), c.strip()) for g, c in entries if g.strip() and c.strip()}
|
|
2782
|
+
)
|
|
2783
|
+
async with self._acquire() as conn:
|
|
2784
|
+
cur = await conn.cursor()
|
|
2785
|
+
try:
|
|
2786
|
+
await cur.execute("DELETE FROM ad_group_scope_map")
|
|
2787
|
+
for ad_group, channel in pairs:
|
|
2788
|
+
await cur.execute(
|
|
2789
|
+
"INSERT INTO ad_group_scope_map (ad_group, channel) VALUES (?,?)",
|
|
2790
|
+
(ad_group, channel),
|
|
2791
|
+
)
|
|
2792
|
+
await conn.commit()
|
|
2793
|
+
except Exception:
|
|
2794
|
+
await conn.rollback()
|
|
2795
|
+
raise
|
|
2796
|
+
|
|
2797
|
+
async def create_session(
|
|
2798
|
+
self,
|
|
2799
|
+
*,
|
|
2800
|
+
token_hash: str,
|
|
2801
|
+
user_id: str,
|
|
2802
|
+
expires_at: float,
|
|
2803
|
+
client: str | None = None,
|
|
2804
|
+
seed_reauth: bool = True,
|
|
2805
|
+
now: float | None = None,
|
|
2806
|
+
) -> None:
|
|
2807
|
+
now = time.time() if now is None else now
|
|
2808
|
+
await self._execute(
|
|
2809
|
+
# reauth_at seeds the step-up window from login (ASVS 7.5.3); seed_reauth=False leaves it
|
|
2810
|
+
# NULL for an MFA-PENDING session (WP-14) so a stolen pre-MFA token can't enroll/step-up.
|
|
2811
|
+
"INSERT INTO sessions (token_hash, user_id, created_at, expires_at, last_used_at,"
|
|
2812
|
+
" revoked_at, client, reauth_at) VALUES (?,?,?,?,?,NULL,?,?)",
|
|
2813
|
+
(token_hash, user_id, now, expires_at, now, client, now if seed_reauth else None),
|
|
2814
|
+
)
|
|
2815
|
+
|
|
2816
|
+
async def get_session(self, token_hash: str) -> SessionRecord | None:
|
|
2817
|
+
d = await self._fetchone("SELECT * FROM sessions WHERE token_hash=?", (token_hash,))
|
|
2818
|
+
return SessionRecord.from_mapping(d) if d else None
|
|
2819
|
+
|
|
2820
|
+
async def list_sessions(self, user_id: str, *, now: float | None = None) -> list[SessionRecord]:
|
|
2821
|
+
"""A user's active (not revoked/expired) sessions, most-recently-used first (WP-10)."""
|
|
2822
|
+
now = time.time() if now is None else now
|
|
2823
|
+
rows = await self._fetchall(
|
|
2824
|
+
"SELECT * FROM sessions WHERE user_id=? AND revoked_at IS NULL AND expires_at > ?"
|
|
2825
|
+
" ORDER BY last_used_at DESC",
|
|
2826
|
+
(user_id, now),
|
|
2827
|
+
)
|
|
2828
|
+
return [SessionRecord.from_mapping(r) for r in rows]
|
|
2829
|
+
|
|
2830
|
+
async def touch_session(self, token_hash: str, *, now: float | None = None) -> None:
|
|
2831
|
+
now = time.time() if now is None else now
|
|
2832
|
+
await self._execute(
|
|
2833
|
+
"UPDATE sessions SET last_used_at=? WHERE token_hash=?", (now, token_hash)
|
|
2834
|
+
)
|
|
2835
|
+
|
|
2836
|
+
async def mark_session_reauthed(
|
|
2837
|
+
self, token_hash: str, *, now: float | None = None, client: str | None = None
|
|
2838
|
+
) -> None:
|
|
2839
|
+
now = time.time() if now is None else now
|
|
2840
|
+
# COALESCE keeps the stored client when none is supplied; a re-verify carrying the current
|
|
2841
|
+
# address re-anchors the session to it (WP-L3-13 new-client-IP step-up).
|
|
2842
|
+
await self._execute(
|
|
2843
|
+
"UPDATE sessions SET reauth_at=?, client=COALESCE(?, client) WHERE token_hash=?",
|
|
2844
|
+
(now, client, token_hash),
|
|
2845
|
+
)
|
|
2846
|
+
|
|
2847
|
+
async def mark_session_mfa_verified(self, token_hash: str, *, now: float | None = None) -> None:
|
|
2848
|
+
now = time.time() if now is None else now
|
|
2849
|
+
await self._execute(
|
|
2850
|
+
"UPDATE sessions SET mfa_verified_at=? WHERE token_hash=?", (now, token_hash)
|
|
2851
|
+
)
|
|
2852
|
+
|
|
2853
|
+
async def revoke_session(self, token_hash: str, *, now: float | None = None) -> None:
|
|
2854
|
+
now = time.time() if now is None else now
|
|
2855
|
+
await self._execute(
|
|
2856
|
+
"UPDATE sessions SET revoked_at=? WHERE token_hash=? AND revoked_at IS NULL",
|
|
2857
|
+
(now, token_hash),
|
|
2858
|
+
)
|
|
2859
|
+
|
|
2860
|
+
async def revoke_user_sessions(
|
|
2861
|
+
self, user_id: str, *, except_token_hash: str | None = None, now: float | None = None
|
|
2862
|
+
) -> int:
|
|
2863
|
+
"""Revoke a user's active sessions (all, or all but ``except_token_hash``). Returns the count."""
|
|
2864
|
+
now = time.time() if now is None else now
|
|
2865
|
+
sql = "UPDATE sessions SET revoked_at=? WHERE user_id=? AND revoked_at IS NULL"
|
|
2866
|
+
params: list[Any] = [now, user_id]
|
|
2867
|
+
if except_token_hash is not None:
|
|
2868
|
+
sql += " AND token_hash != ?"
|
|
2869
|
+
params.append(except_token_hash)
|
|
2870
|
+
async with self._acquire() as conn:
|
|
2871
|
+
cur = await conn.cursor()
|
|
2872
|
+
try:
|
|
2873
|
+
await cur.execute(sql, tuple(params))
|
|
2874
|
+
count = cur.rowcount
|
|
2875
|
+
await conn.commit()
|
|
2876
|
+
except Exception:
|
|
2877
|
+
await conn.rollback()
|
|
2878
|
+
raise
|
|
2879
|
+
return int(count) if count is not None else 0
|
|
2880
|
+
|
|
2881
|
+
async def enforce_session_cap(
|
|
2882
|
+
self, user_id: str, *, keep: int, now: float | None = None
|
|
2883
|
+
) -> None:
|
|
2884
|
+
"""Revoke a user's active sessions beyond the ``keep`` most recently created (AUTH-SESS-CAP)."""
|
|
2885
|
+
if keep <= 0:
|
|
2886
|
+
return
|
|
2887
|
+
now = time.time() if now is None else now
|
|
2888
|
+
await self._execute(
|
|
2889
|
+
"UPDATE sessions SET revoked_at=? WHERE user_id=? AND revoked_at IS NULL"
|
|
2890
|
+
" AND token_hash NOT IN ("
|
|
2891
|
+
" SELECT TOP (?) token_hash FROM sessions WHERE user_id=? AND revoked_at IS NULL"
|
|
2892
|
+
" ORDER BY created_at DESC, token_hash DESC"
|
|
2893
|
+
")",
|
|
2894
|
+
(now, user_id, keep, user_id),
|
|
2895
|
+
)
|
|
2896
|
+
|
|
2897
|
+
async def purge_expired_sessions(self, *, now: float | None = None) -> int:
|
|
2898
|
+
now = time.time() if now is None else now
|
|
2899
|
+
async with self._acquire() as conn:
|
|
2900
|
+
cur = await conn.cursor()
|
|
2901
|
+
try:
|
|
2902
|
+
await cur.execute("DELETE FROM sessions WHERE expires_at < ?", (now,))
|
|
2903
|
+
count = cur.rowcount
|
|
2904
|
+
await conn.commit()
|
|
2905
|
+
except Exception:
|
|
2906
|
+
await conn.rollback()
|
|
2907
|
+
raise
|
|
2908
|
+
return int(count) if count is not None else 0
|
|
2909
|
+
|
|
2910
|
+
async def stats(self) -> dict[str, int]:
|
|
2911
|
+
rows = await self._fetchall(
|
|
2912
|
+
"SELECT status, COUNT(*) AS n FROM queue WHERE stage=? GROUP BY status",
|
|
2913
|
+
(Stage.OUTBOUND.value,),
|
|
2914
|
+
)
|
|
2915
|
+
return {r["status"]: int(r["n"]) for r in rows}
|
|
2916
|
+
|
|
2917
|
+
async def in_pipeline_depth(self) -> int:
|
|
2918
|
+
rows = await self._fetchall(
|
|
2919
|
+
"SELECT COUNT(*) AS n FROM queue WHERE stage IN (?,?,?) AND status IN (?,?)",
|
|
2920
|
+
(
|
|
2921
|
+
Stage.INGRESS.value,
|
|
2922
|
+
Stage.ROUTED.value,
|
|
2923
|
+
Stage.OUTBOUND.value,
|
|
2924
|
+
OutboxStatus.PENDING.value,
|
|
2925
|
+
OutboxStatus.INFLIGHT.value,
|
|
2926
|
+
),
|
|
2927
|
+
)
|
|
2928
|
+
return int(rows[0]["n"]) if rows else 0
|
|
2929
|
+
|
|
2930
|
+
async def db_status(self) -> DbStatus:
|
|
2931
|
+
recovery = await self._fetchone(
|
|
2932
|
+
"SELECT recovery_model_desc AS m FROM sys.databases WHERE name = DB_NAME()"
|
|
2933
|
+
)
|
|
2934
|
+
size = await self._fetchone(
|
|
2935
|
+
"SELECT CAST(SUM(size) AS BIGINT) * 8 * 1024 AS b FROM sys.database_files"
|
|
2936
|
+
)
|
|
2937
|
+
return DbStatus(
|
|
2938
|
+
path=self.path,
|
|
2939
|
+
size_bytes=int(size["b"]) if size and size["b"] is not None else 0,
|
|
2940
|
+
disk_free_bytes=0, # not readily available for a remote SQL Server
|
|
2941
|
+
journal_mode=str(recovery["m"]) if recovery and recovery["m"] else "",
|
|
2942
|
+
messages=await self._count("messages"),
|
|
2943
|
+
events=await self._count("message_events"),
|
|
2944
|
+
audit=await self._count("audit_log"),
|
|
2945
|
+
)
|
|
2946
|
+
|
|
2947
|
+
async def integrity_check(self) -> tuple[bool, str]:
|
|
2948
|
+
# A connectivity/consistency probe; deep checks (DBCC CHECKDB) are an out-of-band DBA task.
|
|
2949
|
+
await self._fetchone("SELECT 1 AS ok")
|
|
2950
|
+
return True, "ok (SQL Server: run DBCC CHECKDB out-of-band for deep checks)"
|
|
2951
|
+
|
|
2952
|
+
async def _count(self, table: str) -> int:
|
|
2953
|
+
row = await self._fetchone(f"SELECT COUNT(*) AS n FROM {table}") # table is a constant
|
|
2954
|
+
return int(row["n"]) if row else 0
|
|
2955
|
+
|
|
2956
|
+
async def connection_metrics(
|
|
2957
|
+
self, *, since: float, now: float | None = None, rate_window: float = 60.0
|
|
2958
|
+
) -> ConnectionMetrics:
|
|
2959
|
+
now = time.time() if now is None else now
|
|
2960
|
+
rate_since = now - rate_window
|
|
2961
|
+
|
|
2962
|
+
count_rows = await self._fetchall(
|
|
2963
|
+
"SELECT channel_id, COUNT(*) AS [read],"
|
|
2964
|
+
" SUM(CASE WHEN status=? THEN 1 ELSE 0 END) AS errored"
|
|
2965
|
+
" FROM messages WHERE received_at>=? GROUP BY channel_id",
|
|
2966
|
+
(MessageStatus.ERROR.value, since),
|
|
2967
|
+
)
|
|
2968
|
+
counts = {r["channel_id"]: (r["read"], r["errored"]) for r in count_rows}
|
|
2969
|
+
last_rows = await self._fetchall(
|
|
2970
|
+
"SELECT channel_id, MAX(received_at) AS last_at FROM messages GROUP BY channel_id"
|
|
2971
|
+
)
|
|
2972
|
+
inbound: dict[str, InboundMetrics] = {}
|
|
2973
|
+
for r in last_rows:
|
|
2974
|
+
read, errored = counts.pop(r["channel_id"], (0, 0))
|
|
2975
|
+
inbound[r["channel_id"]] = InboundMetrics(
|
|
2976
|
+
read=int(read), errored=int(errored or 0), last_at=r["last_at"]
|
|
2977
|
+
)
|
|
2978
|
+
for cid, (read, errored) in counts.items():
|
|
2979
|
+
inbound[cid] = InboundMetrics(read=int(read), errored=int(errored or 0), last_at=None)
|
|
2980
|
+
|
|
2981
|
+
dest_rows = await self._fetchall(
|
|
2982
|
+
"SELECT channel_id, destination_name,"
|
|
2983
|
+
" SUM(CASE WHEN status IN (?,?) THEN 1 ELSE 0 END) AS queue_depth,"
|
|
2984
|
+
" SUM(CASE WHEN status=? AND updated_at>=? THEN 1 ELSE 0 END) AS written,"
|
|
2985
|
+
" SUM(CASE WHEN status=? AND updated_at>=? THEN 1 ELSE 0 END) AS dead,"
|
|
2986
|
+
" MIN(CASE WHEN status=? THEN created_at END) AS oldest_pending_at,"
|
|
2987
|
+
" SUM(CASE WHEN status=? AND updated_at>=? THEN 1 ELSE 0 END) AS recent_done,"
|
|
2988
|
+
" MAX(CASE WHEN status=? THEN updated_at END) AS last_done_at"
|
|
2989
|
+
" FROM queue WHERE stage=? GROUP BY channel_id, destination_name",
|
|
2990
|
+
(
|
|
2991
|
+
OutboxStatus.PENDING.value,
|
|
2992
|
+
OutboxStatus.INFLIGHT.value,
|
|
2993
|
+
OutboxStatus.DONE.value,
|
|
2994
|
+
since,
|
|
2995
|
+
OutboxStatus.DEAD.value,
|
|
2996
|
+
since,
|
|
2997
|
+
OutboxStatus.PENDING.value,
|
|
2998
|
+
OutboxStatus.DONE.value,
|
|
2999
|
+
rate_since,
|
|
3000
|
+
OutboxStatus.DONE.value,
|
|
3001
|
+
Stage.OUTBOUND.value,
|
|
3002
|
+
),
|
|
3003
|
+
)
|
|
3004
|
+
destinations: dict[tuple[str, str], DestinationMetrics] = {}
|
|
3005
|
+
for r in dest_rows:
|
|
3006
|
+
destinations[(r["channel_id"], r["destination_name"])] = DestinationMetrics(
|
|
3007
|
+
queue_depth=int(r["queue_depth"] or 0),
|
|
3008
|
+
written=int(r["written"] or 0),
|
|
3009
|
+
dead=int(r["dead"] or 0),
|
|
3010
|
+
oldest_pending_at=r["oldest_pending_at"],
|
|
3011
|
+
recent_done=int(r["recent_done"] or 0),
|
|
3012
|
+
last_done_at=r["last_done_at"],
|
|
3013
|
+
)
|
|
3014
|
+
return ConnectionMetrics(inbound=inbound, destinations=destinations)
|