agentlighthouse-core 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentlighthouse_core-0.4.0.dist-info/METADATA +358 -0
- agentlighthouse_core-0.4.0.dist-info/RECORD +78 -0
- agentlighthouse_core-0.4.0.dist-info/WHEEL +4 -0
- agentlighthouse_core-0.4.0.dist-info/entry_points.txt +2 -0
- agentlighthouse_core-0.4.0.dist-info/licenses/LICENSE +21 -0
- al_core/__init__.py +12 -0
- al_core/_bundled/configs/audit.yaml +12 -0
- al_core/_bundled/configs/balanced.yaml +36 -0
- al_core/_bundled/configs/strict.yaml +32 -0
- al_core/_bundled/dashboard/assets/index-CN2LU6nV.css +1 -0
- al_core/_bundled/dashboard/assets/index-DX6cZzRi.js +40 -0
- al_core/_bundled/dashboard/index.html +15 -0
- al_core/_bundled/policies/default-deny.yaml +37 -0
- al_core/a2a/__init__.py +25 -0
- al_core/a2a/envelope.py +60 -0
- al_core/a2a/mediator.py +221 -0
- al_core/app.py +427 -0
- al_core/audit/__init__.py +149 -0
- al_core/audit/db.py +481 -0
- al_core/audit/pg.py +350 -0
- al_core/audit/pool.py +260 -0
- al_core/audit/siem.py +97 -0
- al_core/capability/__init__.py +28 -0
- al_core/capability/budget.py +56 -0
- al_core/capability/gate.py +334 -0
- al_core/capability/hitl.py +331 -0
- al_core/capability/policy.py +270 -0
- al_core/capability/store.py +209 -0
- al_core/capability/taint.py +79 -0
- al_core/cli/__init__.py +1 -0
- al_core/cli/main.py +1275 -0
- al_core/config/__init__.py +436 -0
- al_core/config/watch.py +51 -0
- al_core/detect/__init__.py +23 -0
- al_core/detect/base.py +131 -0
- al_core/detect/bip39_words.py +267 -0
- al_core/detect/budget.py +92 -0
- al_core/detect/detect_secrets_adapter.py +110 -0
- al_core/detect/engine.py +270 -0
- al_core/detect/scanners.py +454 -0
- al_core/egress/__init__.py +27 -0
- al_core/egress/nftables.py +59 -0
- al_core/egress/selftest.py +94 -0
- al_core/embed.py +51 -0
- al_core/gateway/__init__.py +29 -0
- al_core/gateway/content.py +154 -0
- al_core/gateway/decision.py +124 -0
- al_core/gateway/dnspin.py +131 -0
- al_core/gateway/forward.py +261 -0
- al_core/gateway/policy.py +66 -0
- al_core/gateway/ssrf.py +178 -0
- al_core/gateway/vkeys.py +205 -0
- al_core/gateway/web.py +801 -0
- al_core/identity/__init__.py +209 -0
- al_core/jsonstore.py +73 -0
- al_core/keys/__init__.py +141 -0
- al_core/killswitch.py +130 -0
- al_core/learn/__init__.py +17 -0
- al_core/learn/bundle.py +76 -0
- al_core/learn/miner.py +156 -0
- al_core/learn/scanner.py +77 -0
- al_core/learn/testgen.py +107 -0
- al_core/mcp/__init__.py +25 -0
- al_core/mcp/chain.py +147 -0
- al_core/mcp/descriptors.py +111 -0
- al_core/mcp/mediator.py +150 -0
- al_core/mcp/proxy.py +161 -0
- al_core/mcp/session.py +537 -0
- al_core/memory/__init__.py +15 -0
- al_core/memory/guard.py +330 -0
- al_core/memory/store.py +80 -0
- al_core/normalize/__init__.py +254 -0
- al_core/principal.py +59 -0
- al_core/receipt/__init__.py +192 -0
- al_core/resources.py +33 -0
- al_core/runtime.py +471 -0
- al_core/skills/__init__.py +30 -0
- al_core/skills/guard.py +218 -0
al_core/audit/pg.py
ADDED
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
"""Postgres mirror — the concurrent-writer path (Phase 6).
|
|
2
|
+
|
|
3
|
+
Same contract as :class:`~al_core.audit.db.SqliteMirror`: a **query surface**,
|
|
4
|
+
never a source of truth (the append-only JSONL stays canonical). The reason it
|
|
5
|
+
exists is concurrency, not scale:
|
|
6
|
+
|
|
7
|
+
* SQLite serializes writers behind one process-local lock. That is fine for one
|
|
8
|
+
mediator, and it is exactly what a multi-writer deployment cannot rely on —
|
|
9
|
+
two al-core replicas sharing a mirror would race the ``quotas`` table, and a
|
|
10
|
+
raced budget check is a **denial-of-wallet hole**: both writers read "99 of
|
|
11
|
+
100 used", both allow, and the tenant spends 101.
|
|
12
|
+
* Postgres gives us a real lock. ``quota_try_consume`` materializes the row,
|
|
13
|
+
takes ``SELECT ... FOR UPDATE`` on it, and only then decides — so N writers
|
|
14
|
+
hitting one budget concurrently allow *exactly* the budget, never one more.
|
|
15
|
+
|
|
16
|
+
Requires the ``postgres`` extra (``psycopg[binary]``). Import stays lazy so a
|
|
17
|
+
SQLite-only deployment never pays for it.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
from typing import Any, Iterable, Iterator
|
|
24
|
+
|
|
25
|
+
from .db import actor_prefix, org_of
|
|
26
|
+
|
|
27
|
+
_SCHEMA = """
|
|
28
|
+
CREATE TABLE IF NOT EXISTS events (
|
|
29
|
+
seq BIGINT PRIMARY KEY,
|
|
30
|
+
ts TEXT NOT NULL,
|
|
31
|
+
actor TEXT NOT NULL,
|
|
32
|
+
action TEXT NOT NULL,
|
|
33
|
+
target TEXT NOT NULL,
|
|
34
|
+
verdict TEXT NOT NULL,
|
|
35
|
+
block_reason TEXT,
|
|
36
|
+
prev_hash TEXT NOT NULL,
|
|
37
|
+
record_hash TEXT NOT NULL,
|
|
38
|
+
receipt_json TEXT NOT NULL
|
|
39
|
+
);
|
|
40
|
+
CREATE INDEX IF NOT EXISTS idx_events_actor ON events(actor);
|
|
41
|
+
CREATE INDEX IF NOT EXISTS idx_events_action ON events(action);
|
|
42
|
+
CREATE INDEX IF NOT EXISTS idx_events_verdict ON events(verdict);
|
|
43
|
+
|
|
44
|
+
CREATE TABLE IF NOT EXISTS quotas (
|
|
45
|
+
quota_key TEXT PRIMARY KEY,
|
|
46
|
+
window_start TEXT NOT NULL,
|
|
47
|
+
count BIGINT NOT NULL DEFAULT 0,
|
|
48
|
+
bytes BIGINT NOT NULL DEFAULT 0
|
|
49
|
+
);
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _org_filter(org: str | None) -> tuple[list[str], list[Any]]:
|
|
54
|
+
if org is None:
|
|
55
|
+
return [], []
|
|
56
|
+
if org == "system":
|
|
57
|
+
return ["actor NOT LIKE 'spiffe://%%'"], []
|
|
58
|
+
prefix = actor_prefix(org)
|
|
59
|
+
if prefix is None:
|
|
60
|
+
return ["FALSE"], []
|
|
61
|
+
return ["substr(actor, 1, %s) = %s"], [len(prefix), prefix]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class PostgresMirror:
|
|
65
|
+
"""Multi-writer mirror. Method-for-method compatible with SqliteMirror for
|
|
66
|
+
everything the control plane and governance layer read."""
|
|
67
|
+
|
|
68
|
+
def __init__(self, dsn: str) -> None:
|
|
69
|
+
import psycopg # lazy: only a postgres deployment needs the driver
|
|
70
|
+
|
|
71
|
+
self._psycopg = psycopg
|
|
72
|
+
self._dsn = dsn
|
|
73
|
+
self._conn = psycopg.connect(dsn, autocommit=True)
|
|
74
|
+
with self._conn.cursor() as cur:
|
|
75
|
+
cur.execute(_SCHEMA)
|
|
76
|
+
|
|
77
|
+
# -- writes ---------------------------------------------------------------
|
|
78
|
+
|
|
79
|
+
def upsert_event(self, receipt: dict[str, Any]) -> None:
|
|
80
|
+
with self._conn.cursor() as cur:
|
|
81
|
+
cur.execute(
|
|
82
|
+
"INSERT INTO events (seq, ts, actor, action, target, verdict, "
|
|
83
|
+
" block_reason, prev_hash, record_hash, receipt_json) "
|
|
84
|
+
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) "
|
|
85
|
+
"ON CONFLICT (seq) DO UPDATE SET "
|
|
86
|
+
" ts=EXCLUDED.ts, actor=EXCLUDED.actor, action=EXCLUDED.action, "
|
|
87
|
+
" target=EXCLUDED.target, verdict=EXCLUDED.verdict, "
|
|
88
|
+
" block_reason=EXCLUDED.block_reason, prev_hash=EXCLUDED.prev_hash, "
|
|
89
|
+
" record_hash=EXCLUDED.record_hash, receipt_json=EXCLUDED.receipt_json",
|
|
90
|
+
(receipt["seq"], receipt["ts"], receipt["actor"], receipt["action"],
|
|
91
|
+
receipt["target"], receipt["verdict"], receipt.get("block_reason"),
|
|
92
|
+
receipt["prev_hash"], receipt["record_hash"],
|
|
93
|
+
json.dumps(receipt, separators=(",", ":"), ensure_ascii=False)),
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
def reconcile(self, receipts: Iterable[dict[str, Any]]) -> int:
|
|
97
|
+
n = 0
|
|
98
|
+
for r in receipts:
|
|
99
|
+
self.upsert_event(r)
|
|
100
|
+
n += 1
|
|
101
|
+
return n
|
|
102
|
+
|
|
103
|
+
# -- quotas (THE reason this backend exists) --------------------------------
|
|
104
|
+
|
|
105
|
+
def quota_try_consume(
|
|
106
|
+
self,
|
|
107
|
+
quota_key: str,
|
|
108
|
+
window_start: str,
|
|
109
|
+
*,
|
|
110
|
+
count_delta: int = 0,
|
|
111
|
+
bytes_delta: int = 0,
|
|
112
|
+
max_count: int | None = None,
|
|
113
|
+
max_bytes: int | None = None,
|
|
114
|
+
) -> tuple[bool, int, int]:
|
|
115
|
+
"""Atomic check-and-increment across *processes*.
|
|
116
|
+
|
|
117
|
+
The row is materialized, then locked with ``FOR UPDATE`` for the whole
|
|
118
|
+
decide-and-write. Concurrent writers therefore queue on the row rather
|
|
119
|
+
than racing it: a budget of N admits exactly N, never N+1.
|
|
120
|
+
"""
|
|
121
|
+
with self._conn.transaction(), self._conn.cursor() as cur:
|
|
122
|
+
cur.execute(
|
|
123
|
+
"INSERT INTO quotas (quota_key, window_start, count, bytes) "
|
|
124
|
+
"VALUES (%s, %s, 0, 0) ON CONFLICT (quota_key) DO NOTHING",
|
|
125
|
+
(quota_key, window_start),
|
|
126
|
+
)
|
|
127
|
+
cur.execute(
|
|
128
|
+
"SELECT window_start, count, bytes FROM quotas "
|
|
129
|
+
"WHERE quota_key=%s FOR UPDATE",
|
|
130
|
+
(quota_key,),
|
|
131
|
+
)
|
|
132
|
+
row = cur.fetchone()
|
|
133
|
+
in_window = row is not None and row[0] == window_start
|
|
134
|
+
cur_count = row[1] if in_window else 0
|
|
135
|
+
cur_bytes = row[2] if in_window else 0
|
|
136
|
+
new_count, new_bytes = cur_count + count_delta, cur_bytes + bytes_delta
|
|
137
|
+
if (max_count is not None and new_count > max_count) or (
|
|
138
|
+
max_bytes is not None and new_bytes > max_bytes
|
|
139
|
+
):
|
|
140
|
+
return False, cur_count, cur_bytes
|
|
141
|
+
cur.execute(
|
|
142
|
+
"UPDATE quotas SET window_start=%s, count=%s, bytes=%s "
|
|
143
|
+
"WHERE quota_key=%s",
|
|
144
|
+
(window_start, new_count, new_bytes, quota_key),
|
|
145
|
+
)
|
|
146
|
+
return True, new_count, new_bytes
|
|
147
|
+
|
|
148
|
+
def quota_usage(self, quota_key: str, window_start: str) -> tuple[int, int]:
|
|
149
|
+
with self._conn.cursor() as cur:
|
|
150
|
+
cur.execute(
|
|
151
|
+
"SELECT window_start, count, bytes FROM quotas WHERE quota_key=%s",
|
|
152
|
+
(quota_key,),
|
|
153
|
+
)
|
|
154
|
+
row = cur.fetchone()
|
|
155
|
+
if row is None or row[0] != window_start:
|
|
156
|
+
return 0, 0
|
|
157
|
+
return int(row[1]), int(row[2])
|
|
158
|
+
|
|
159
|
+
# -- reads (same surface + same org scoping as SQLite) ----------------------
|
|
160
|
+
|
|
161
|
+
def _rows(self, sql: str, params: list[Any]) -> list[Any]:
|
|
162
|
+
with self._conn.cursor() as cur:
|
|
163
|
+
cur.execute(sql, params)
|
|
164
|
+
return cur.fetchall()
|
|
165
|
+
|
|
166
|
+
def count_events(self, org: str | None = None) -> int:
|
|
167
|
+
clauses, params = _org_filter(org)
|
|
168
|
+
where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
|
|
169
|
+
return int(self._rows(f"SELECT COUNT(*) FROM events {where}", params)[0][0])
|
|
170
|
+
|
|
171
|
+
def latest_seq(self) -> int | None:
|
|
172
|
+
row = self._rows("SELECT MAX(seq) FROM events", [])[0][0]
|
|
173
|
+
return int(row) if row is not None else None
|
|
174
|
+
|
|
175
|
+
def last_config_change(self) -> dict[str, Any] | None:
|
|
176
|
+
rows = self._rows(
|
|
177
|
+
"SELECT receipt_json FROM events WHERE action='config_change' "
|
|
178
|
+
"ORDER BY seq DESC LIMIT 1", [])
|
|
179
|
+
return json.loads(rows[0][0]) if rows else None
|
|
180
|
+
|
|
181
|
+
def _group_count(self, column: str, extra: str = "",
|
|
182
|
+
org: str | None = None) -> dict[str, int]:
|
|
183
|
+
clauses, params = _org_filter(org)
|
|
184
|
+
if extra:
|
|
185
|
+
clauses.append(extra)
|
|
186
|
+
where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
|
|
187
|
+
rows = self._rows(
|
|
188
|
+
f"SELECT {column}, COUNT(*) FROM events {where} "
|
|
189
|
+
f"GROUP BY {column} ORDER BY 2 DESC", params)
|
|
190
|
+
return {r[0]: int(r[1]) for r in rows}
|
|
191
|
+
|
|
192
|
+
def verdict_counts(self, org: str | None = None) -> dict[str, int]:
|
|
193
|
+
return self._group_count("verdict", org=org)
|
|
194
|
+
|
|
195
|
+
def action_counts(self, org: str | None = None) -> dict[str, int]:
|
|
196
|
+
return self._group_count("action", org=org)
|
|
197
|
+
|
|
198
|
+
def block_reason_counts(self, org: str | None = None) -> dict[str, int]:
|
|
199
|
+
return self._group_count("block_reason", "block_reason IS NOT NULL", org=org)
|
|
200
|
+
|
|
201
|
+
def actor_counts(self, org: str | None = None) -> dict[str, int]:
|
|
202
|
+
return self._group_count("actor", org=org)
|
|
203
|
+
|
|
204
|
+
def org_counts(self, org: str | None = None) -> dict[str, int]:
|
|
205
|
+
out: dict[str, int] = {}
|
|
206
|
+
for actor, n in self.actor_counts(org).items():
|
|
207
|
+
out[org_of(actor)] = out.get(org_of(actor), 0) + n
|
|
208
|
+
return out
|
|
209
|
+
|
|
210
|
+
def recent_filtered(
|
|
211
|
+
self, *, limit: int = 50, actor: str | None = None, action: str | None = None,
|
|
212
|
+
verdict: str | None = None, org: str | None = None,
|
|
213
|
+
) -> list[dict[str, Any]]:
|
|
214
|
+
clauses, params = _org_filter(org)
|
|
215
|
+
for col, val in (("actor", actor), ("action", action), ("verdict", verdict)):
|
|
216
|
+
if val is not None:
|
|
217
|
+
clauses.append(f"{col}=%s")
|
|
218
|
+
params.append(val)
|
|
219
|
+
where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
|
|
220
|
+
params.append(max(1, min(limit, 500)))
|
|
221
|
+
rows = self._rows(
|
|
222
|
+
f"SELECT receipt_json FROM events {where} ORDER BY seq DESC LIMIT %s", params)
|
|
223
|
+
return [json.loads(r[0]) for r in rows]
|
|
224
|
+
|
|
225
|
+
def recent(self, limit: int = 50) -> list[dict[str, Any]]:
|
|
226
|
+
return self.recent_filtered(limit=limit)
|
|
227
|
+
|
|
228
|
+
def receipt_by_seq(self, seq: int, org: str | None = None) -> dict[str, Any] | None:
|
|
229
|
+
clauses, params = _org_filter(org)
|
|
230
|
+
clauses.append("seq=%s")
|
|
231
|
+
params.append(seq)
|
|
232
|
+
rows = self._rows(
|
|
233
|
+
"SELECT receipt_json FROM events WHERE " + " AND ".join(clauses), params)
|
|
234
|
+
return json.loads(rows[0][0]) if rows else None
|
|
235
|
+
|
|
236
|
+
def search(
|
|
237
|
+
self, *, query: str | None = None, since: str | None = None,
|
|
238
|
+
until: str | None = None, actor: str | None = None, action: str | None = None,
|
|
239
|
+
verdict: str | None = None, org: str | None = None, limit: int = 100,
|
|
240
|
+
) -> list[dict[str, Any]]:
|
|
241
|
+
clauses, params = _org_filter(org)
|
|
242
|
+
for col, val in (("actor", actor), ("action", action), ("verdict", verdict)):
|
|
243
|
+
if val is not None:
|
|
244
|
+
clauses.append(f"{col}=%s")
|
|
245
|
+
params.append(val)
|
|
246
|
+
if since is not None:
|
|
247
|
+
clauses.append("ts >= %s")
|
|
248
|
+
params.append(since)
|
|
249
|
+
if until is not None:
|
|
250
|
+
clauses.append("ts <= %s")
|
|
251
|
+
params.append(until)
|
|
252
|
+
if query:
|
|
253
|
+
clauses.append("(actor ILIKE %s OR target ILIKE %s "
|
|
254
|
+
"OR COALESCE(block_reason,'') ILIKE %s)")
|
|
255
|
+
like = f"%{query}%"
|
|
256
|
+
params.extend([like, like, like])
|
|
257
|
+
where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
|
|
258
|
+
params.append(max(1, min(limit, 500)))
|
|
259
|
+
rows = self._rows(
|
|
260
|
+
f"SELECT receipt_json FROM events {where} ORDER BY seq DESC LIMIT %s", params)
|
|
261
|
+
return [json.loads(r[0]) for r in rows]
|
|
262
|
+
|
|
263
|
+
def iter_receipts(
|
|
264
|
+
self, *, since: str | None = None, until: str | None = None,
|
|
265
|
+
org: str | None = None, batch: int = 1000,
|
|
266
|
+
) -> Iterator[dict[str, Any]]:
|
|
267
|
+
"""Every receipt in the window, newest first — uncapped (see the
|
|
268
|
+
SQLite twin): keyset-paginated on ``seq``."""
|
|
269
|
+
base, base_params = _org_filter(org)
|
|
270
|
+
if since is not None:
|
|
271
|
+
base.append("ts >= %s")
|
|
272
|
+
base_params.append(since)
|
|
273
|
+
if until is not None:
|
|
274
|
+
base.append("ts <= %s")
|
|
275
|
+
base_params.append(until)
|
|
276
|
+
last_seq: int | None = None
|
|
277
|
+
while True:
|
|
278
|
+
clauses, params = list(base), list(base_params)
|
|
279
|
+
if last_seq is not None:
|
|
280
|
+
clauses.append("seq < %s")
|
|
281
|
+
params.append(last_seq)
|
|
282
|
+
where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
|
|
283
|
+
params.append(max(1, batch))
|
|
284
|
+
rows = self._rows(
|
|
285
|
+
f"SELECT seq, receipt_json FROM events {where} "
|
|
286
|
+
"ORDER BY seq DESC LIMIT %s", params)
|
|
287
|
+
if not rows:
|
|
288
|
+
return
|
|
289
|
+
for r in rows:
|
|
290
|
+
yield json.loads(r[1])
|
|
291
|
+
last_seq = rows[-1][0]
|
|
292
|
+
|
|
293
|
+
def time_series(self, *, buckets: int = 24, span_hours: int = 24,
|
|
294
|
+
now: str | None = None, org: str | None = None) -> list[dict[str, Any]]:
|
|
295
|
+
from datetime import datetime, timedelta
|
|
296
|
+
|
|
297
|
+
clauses, params = _org_filter(org)
|
|
298
|
+
where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
|
|
299
|
+
if now is None:
|
|
300
|
+
row = self._rows(f"SELECT MAX(ts) FROM events {where}", params)[0][0]
|
|
301
|
+
now = row if row else None
|
|
302
|
+
if now is None:
|
|
303
|
+
return []
|
|
304
|
+
|
|
305
|
+
def parse(s: str) -> datetime:
|
|
306
|
+
return datetime.fromisoformat(s.replace("Z", "+00:00"))
|
|
307
|
+
|
|
308
|
+
end = parse(now)
|
|
309
|
+
start = end - timedelta(hours=span_hours)
|
|
310
|
+
rows = self._rows(
|
|
311
|
+
f"SELECT ts, verdict FROM events {where}", params)
|
|
312
|
+
width = (end - start) / buckets
|
|
313
|
+
series = [{"bucket_start": (start + width * i).isoformat().replace("+00:00", "Z"),
|
|
314
|
+
"total": 0, "allow": 0, "block": 0, "strip": 0, "warn": 0, "ask": 0}
|
|
315
|
+
for i in range(buckets)]
|
|
316
|
+
for ts, verdict in rows:
|
|
317
|
+
try:
|
|
318
|
+
point = parse(ts)
|
|
319
|
+
except ValueError:
|
|
320
|
+
continue
|
|
321
|
+
if point < start or point > end:
|
|
322
|
+
continue
|
|
323
|
+
idx = max(0, min(buckets - 1, int((point - start) / width)))
|
|
324
|
+
series[idx]["total"] += 1
|
|
325
|
+
if verdict in series[idx]:
|
|
326
|
+
series[idx][verdict] += 1
|
|
327
|
+
return series
|
|
328
|
+
|
|
329
|
+
def stats(self, org: str | None = None) -> dict[str, Any]:
|
|
330
|
+
return {
|
|
331
|
+
"events": self.count_events(org),
|
|
332
|
+
"latest_seq": self.latest_seq(),
|
|
333
|
+
"verdicts": self.verdict_counts(org),
|
|
334
|
+
"actions": self.action_counts(org),
|
|
335
|
+
"block_reasons": self.block_reason_counts(org),
|
|
336
|
+
"actors": self.actor_counts(org),
|
|
337
|
+
"orgs": self.org_counts(org),
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
def close(self) -> None:
|
|
341
|
+
self._conn.close()
|
|
342
|
+
|
|
343
|
+
def __enter__(self) -> "PostgresMirror":
|
|
344
|
+
return self
|
|
345
|
+
|
|
346
|
+
def __exit__(self, *exc: object) -> None:
|
|
347
|
+
self.close()
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
__all__ = ["PostgresMirror"]
|
al_core/audit/pool.py
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
"""Evidence pool — one query surface over several planes' ledgers.
|
|
2
|
+
|
|
3
|
+
The compose topology gives every plane its **own** ledger with its own single
|
|
4
|
+
writer (invariant #7): the gateway (data plane) writes the receipts operators
|
|
5
|
+
actually care about, the control plane writes only its own actions (kill
|
|
6
|
+
switch, config). The dashboard lives on the control plane, so without help it
|
|
7
|
+
would show the control ledger and nothing else — which is exactly the bug an
|
|
8
|
+
operator reports as "my traffic never shows up".
|
|
9
|
+
|
|
10
|
+
The pool fixes that read-side only. Each extra plane is attached as a
|
|
11
|
+
:class:`RemotePlane`: a **read-only** SQLite connection over that plane's live
|
|
12
|
+
mirror (shared volume) plus its public key and canonical JSONL for chain
|
|
13
|
+
verification. Nothing here can write another plane's chain or mirror — the
|
|
14
|
+
one-writer invariant holds by construction, not by discipline.
|
|
15
|
+
|
|
16
|
+
Every receipt returned by the pool is tagged with a ``plane`` key (presentation
|
|
17
|
+
only — the tag is added after the signed body is loaded, so signatures still
|
|
18
|
+
verify over the original fields).
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import logging
|
|
24
|
+
import sqlite3
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import Any
|
|
27
|
+
|
|
28
|
+
from al_verify.verify import load_public_key, verify_chain
|
|
29
|
+
|
|
30
|
+
from . import JsonlLedger
|
|
31
|
+
from .db import SqliteMirror
|
|
32
|
+
|
|
33
|
+
log = logging.getLogger("al.audit.pool")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _tag(receipt: dict[str, Any], plane: str) -> dict[str, Any]:
|
|
37
|
+
receipt["plane"] = plane
|
|
38
|
+
return receipt
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class RemotePlane:
|
|
42
|
+
"""Another process's evidence, opened read-only over a shared volume.
|
|
43
|
+
|
|
44
|
+
The mirror may not exist yet (the writer has not booted) or may be replaced
|
|
45
|
+
(volume recreated) — every access re-opens on failure and degrades to
|
|
46
|
+
"no events" rather than taking the dashboard down with it.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
def __init__(self, plane: str, data_dir: str | Path,
|
|
50
|
+
pubkey_path: str | Path | None = None) -> None:
|
|
51
|
+
self.plane = plane
|
|
52
|
+
self.data_dir = Path(data_dir)
|
|
53
|
+
self.db_path = self.data_dir / "al.sqlite"
|
|
54
|
+
self.jsonl_path = self.data_dir / "ledger.jsonl"
|
|
55
|
+
self.pubkey_path = (Path(pubkey_path) if pubkey_path is not None
|
|
56
|
+
else self.data_dir / "keys" / "mediator_ed25519.pub")
|
|
57
|
+
self._mirror: SqliteMirror | None = None
|
|
58
|
+
|
|
59
|
+
def mirror(self) -> SqliteMirror | None:
|
|
60
|
+
if self._mirror is None:
|
|
61
|
+
try:
|
|
62
|
+
self._mirror = SqliteMirror(self.db_path, read_only=True)
|
|
63
|
+
except sqlite3.Error as exc:
|
|
64
|
+
log.debug("plane %r mirror not readable yet: %s", self.plane, exc)
|
|
65
|
+
return None
|
|
66
|
+
return self._mirror
|
|
67
|
+
|
|
68
|
+
def query(self, method: str, /, *args: Any, **kwargs: Any) -> Any:
|
|
69
|
+
"""Run one mirror query; on error drop the connection and report empty.
|
|
70
|
+
|
|
71
|
+
A vanished/recreated database must degrade to "no events from this
|
|
72
|
+
plane", never to a dashboard 500."""
|
|
73
|
+
mirror = self.mirror()
|
|
74
|
+
if mirror is None:
|
|
75
|
+
return None
|
|
76
|
+
try:
|
|
77
|
+
return getattr(mirror, method)(*args, **kwargs)
|
|
78
|
+
except sqlite3.Error as exc:
|
|
79
|
+
log.warning("plane %r query %s failed (%s) — reopening next call",
|
|
80
|
+
self.plane, method, exc)
|
|
81
|
+
try:
|
|
82
|
+
mirror.close()
|
|
83
|
+
except Exception: # noqa: BLE001 — already broken
|
|
84
|
+
pass
|
|
85
|
+
self._mirror = None
|
|
86
|
+
return None
|
|
87
|
+
|
|
88
|
+
def public_key(self): # noqa: ANN201 — Ed25519PublicKey
|
|
89
|
+
return load_public_key(self.pubkey_path.read_bytes())
|
|
90
|
+
|
|
91
|
+
def chain_status(self) -> dict[str, Any]:
|
|
92
|
+
"""Standalone chain verification of this plane's canonical JSONL.
|
|
93
|
+
|
|
94
|
+
Fail-closed either way: whatever we cannot check is never reported as
|
|
95
|
+
verified. But *why* it is unverified matters. A chain whose signatures
|
|
96
|
+
do not check out means tampering — the loudest alarm this product has.
|
|
97
|
+
A pubkey we cannot read (never pointed at the native ``keys/`` dir, or
|
|
98
|
+
the plane has not booted) means we could not look. Reporting the second
|
|
99
|
+
as the first would make that alarm cry wolf on a config typo, and an
|
|
100
|
+
operator who sees one false BROKEN stops believing the real one.
|
|
101
|
+
``reason`` keeps them apart: ``unavailable`` = could not check,
|
|
102
|
+
``invalid`` = checked and it failed.
|
|
103
|
+
"""
|
|
104
|
+
try:
|
|
105
|
+
records = JsonlLedger(self.jsonl_path).read_all()
|
|
106
|
+
except OSError as exc:
|
|
107
|
+
return self._unverified("unavailable", f"ledger unreadable: {exc}")
|
|
108
|
+
if not records: # plane not booted yet — nothing to verify
|
|
109
|
+
return {"verified": True, "length": 0, "reason": None, "error": None}
|
|
110
|
+
try:
|
|
111
|
+
key = self.public_key()
|
|
112
|
+
except Exception as exc: # noqa: BLE001 — missing/malformed key = cannot check
|
|
113
|
+
return self._unverified(
|
|
114
|
+
"unavailable",
|
|
115
|
+
f"no usable public key at {self.pubkey_path}: {exc}")
|
|
116
|
+
try:
|
|
117
|
+
length = verify_chain(records, key)
|
|
118
|
+
except Exception as exc: # noqa: BLE001 — the check ran and it failed
|
|
119
|
+
return self._unverified("invalid", str(exc))
|
|
120
|
+
return {"verified": True, "length": length, "reason": None, "error": None}
|
|
121
|
+
|
|
122
|
+
@staticmethod
|
|
123
|
+
def _unverified(reason: str, error: str) -> dict[str, Any]:
|
|
124
|
+
return {"verified": False, "length": 0, "reason": reason, "error": error}
|
|
125
|
+
|
|
126
|
+
def close(self) -> None:
|
|
127
|
+
if self._mirror is not None:
|
|
128
|
+
self._mirror.close()
|
|
129
|
+
self._mirror = None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class EvidencePool:
|
|
133
|
+
"""Merged, org-scoped, plane-tagged reads over the local ledger plus any
|
|
134
|
+
attached remote planes. The local plane stays first — ties in merged
|
|
135
|
+
ordering resolve toward it deterministically."""
|
|
136
|
+
|
|
137
|
+
def __init__(self, local_plane: str, local_db: Any,
|
|
138
|
+
remotes: list[RemotePlane] | None = None) -> None:
|
|
139
|
+
self.local_plane = local_plane
|
|
140
|
+
self._local = local_db
|
|
141
|
+
self._remotes = remotes or []
|
|
142
|
+
|
|
143
|
+
@property
|
|
144
|
+
def planes(self) -> list[str]:
|
|
145
|
+
return [self.local_plane, *(r.plane for r in self._remotes)]
|
|
146
|
+
|
|
147
|
+
def _each(self, method: str, /, *args: Any, **kwargs: Any) -> dict[str, Any]:
|
|
148
|
+
"""{plane: result} for one query across all planes (None on failure)."""
|
|
149
|
+
out = {self.local_plane: getattr(self._local, method)(*args, **kwargs)}
|
|
150
|
+
for remote in self._remotes:
|
|
151
|
+
out[remote.plane] = remote.query(method, *args, **kwargs)
|
|
152
|
+
return out
|
|
153
|
+
|
|
154
|
+
# -- merged reads --------------------------------------------------------
|
|
155
|
+
|
|
156
|
+
def _merged_receipts(self, method: str, /, *, limit: int,
|
|
157
|
+
**filters: Any) -> list[dict[str, Any]]:
|
|
158
|
+
merged: list[dict[str, Any]] = []
|
|
159
|
+
for plane, receipts in self._each(method, limit=limit, **filters).items():
|
|
160
|
+
merged.extend(_tag(r, plane) for r in receipts or [])
|
|
161
|
+
# Newest first; ts strings are RFC 3339 (lexicographically ordered).
|
|
162
|
+
# plane/seq tiebreak keeps the order stable across polls.
|
|
163
|
+
merged.sort(key=lambda r: (r.get("ts", ""), r.get("plane", ""),
|
|
164
|
+
r.get("seq", 0)), reverse=True)
|
|
165
|
+
return merged[:limit]
|
|
166
|
+
|
|
167
|
+
def recent_filtered(self, *, limit: int = 50, **filters: Any) -> list[dict[str, Any]]:
|
|
168
|
+
return self._merged_receipts("recent_filtered", limit=limit, **filters)
|
|
169
|
+
|
|
170
|
+
def search(self, *, limit: int = 100, **filters: Any) -> list[dict[str, Any]]:
|
|
171
|
+
return self._merged_receipts("search", limit=limit, **filters)
|
|
172
|
+
|
|
173
|
+
def stats(self, org: str | None = None) -> dict[str, Any]:
|
|
174
|
+
"""Merged one-shot summary; per-plane event counts under ``planes``."""
|
|
175
|
+
def merge_counts(dsts: dict[str, int], src: dict[str, int] | None) -> None:
|
|
176
|
+
for k, v in (src or {}).items():
|
|
177
|
+
dsts[k] = dsts.get(k, 0) + v
|
|
178
|
+
|
|
179
|
+
events = 0
|
|
180
|
+
planes: dict[str, int] = {}
|
|
181
|
+
merged: dict[str, dict[str, int]] = {
|
|
182
|
+
"verdicts": {}, "actions": {}, "block_reasons": {},
|
|
183
|
+
"actors": {}, "orgs": {},
|
|
184
|
+
}
|
|
185
|
+
for plane, stats in self._each("stats", org).items():
|
|
186
|
+
if stats is None:
|
|
187
|
+
planes[plane] = 0
|
|
188
|
+
continue
|
|
189
|
+
events += stats.get("events", 0)
|
|
190
|
+
planes[plane] = stats.get("events", 0)
|
|
191
|
+
for key, dst in merged.items():
|
|
192
|
+
merge_counts(dst, stats.get(key))
|
|
193
|
+
return {"events": events, "planes": planes, **merged}
|
|
194
|
+
|
|
195
|
+
def org_counts(self, org: str | None = None) -> dict[str, int]:
|
|
196
|
+
out: dict[str, int] = {}
|
|
197
|
+
for counts in self._each("org_counts", org).values():
|
|
198
|
+
for k, v in (counts or {}).items():
|
|
199
|
+
out[k] = out.get(k, 0) + v
|
|
200
|
+
return out
|
|
201
|
+
|
|
202
|
+
def time_series(self, *, buckets: int = 24, span_hours: int = 24,
|
|
203
|
+
org: str | None = None) -> list[dict[str, Any]]:
|
|
204
|
+
"""Bucket-aligned sum across planes (one shared ``now`` anchors both)."""
|
|
205
|
+
latest = [r[0]["ts"] for r in
|
|
206
|
+
self._each("recent_filtered", limit=1, org=org).values() if r]
|
|
207
|
+
if not latest:
|
|
208
|
+
return []
|
|
209
|
+
now = max(latest)
|
|
210
|
+
merged: list[dict[str, Any]] | None = None
|
|
211
|
+
for series in self._each("time_series", buckets=buckets,
|
|
212
|
+
span_hours=span_hours, now=now, org=org).values():
|
|
213
|
+
if not series:
|
|
214
|
+
continue
|
|
215
|
+
if merged is None:
|
|
216
|
+
merged = [dict(b) for b in series]
|
|
217
|
+
continue
|
|
218
|
+
for dst, src in zip(merged, series):
|
|
219
|
+
for key, val in src.items():
|
|
220
|
+
if isinstance(val, int):
|
|
221
|
+
dst[key] = dst.get(key, 0) + val
|
|
222
|
+
return merged or []
|
|
223
|
+
|
|
224
|
+
def receipt_by_seq(self, seq: int, org: str | None = None,
|
|
225
|
+
plane: str | None = None) -> dict[str, Any] | None:
|
|
226
|
+
"""One receipt by (plane, seq). ``plane=None`` = the local plane, so a
|
|
227
|
+
single-ledger deployment keeps its old URL shape."""
|
|
228
|
+
plane = plane or self.local_plane
|
|
229
|
+
if plane == self.local_plane:
|
|
230
|
+
receipt = self._local.receipt_by_seq(seq, org)
|
|
231
|
+
else:
|
|
232
|
+
remote = self._remote(plane)
|
|
233
|
+
receipt = remote.query("receipt_by_seq", seq, org) if remote else None
|
|
234
|
+
return _tag(receipt, plane) if receipt else None
|
|
235
|
+
|
|
236
|
+
def public_key_for(self, plane: str | None, local_key): # noqa: ANN201
|
|
237
|
+
plane = plane or self.local_plane
|
|
238
|
+
if plane == self.local_plane:
|
|
239
|
+
return local_key
|
|
240
|
+
remote = self._remote(plane)
|
|
241
|
+
if remote is None:
|
|
242
|
+
raise KeyError(f"unknown plane: {plane!r}")
|
|
243
|
+
return remote.public_key()
|
|
244
|
+
|
|
245
|
+
def chains(self, local_chain: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
|
246
|
+
"""Per-plane chain verification status ({plane: healthz-shaped dict})."""
|
|
247
|
+
out = {self.local_plane: local_chain}
|
|
248
|
+
for remote in self._remotes:
|
|
249
|
+
out[remote.plane] = remote.chain_status()
|
|
250
|
+
return out
|
|
251
|
+
|
|
252
|
+
def _remote(self, plane: str) -> RemotePlane | None:
|
|
253
|
+
return next((r for r in self._remotes if r.plane == plane), None)
|
|
254
|
+
|
|
255
|
+
def close(self) -> None:
|
|
256
|
+
for remote in self._remotes:
|
|
257
|
+
remote.close()
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
__all__ = ["EvidencePool", "RemotePlane"]
|