aether-quant 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aether_quant-0.0.1.dist-info/METADATA +7 -0
- aether_quant-0.0.1.dist-info/RECORD +99 -0
- aether_quant-0.0.1.dist-info/WHEEL +5 -0
- aether_quant-0.0.1.dist-info/entry_points.txt +2 -0
- aether_quant-0.0.1.dist-info/licenses/LICENSE +133 -0
- aether_quant-0.0.1.dist-info/top_level.txt +13 -0
- analyzer/README.md +69 -0
- analyzer/__init__.py +17 -0
- analyzer/market_analyzer.py +521 -0
- aq_cli.py +3633 -0
- audit/README.md +47 -0
- audit/__init__.py +38 -0
- audit/hash_chain.py +52 -0
- audit/postgres_audit.py +61 -0
- audit/postgres_worker.py +402 -0
- audit/redis_queue.py +124 -0
- audit/status_export.py +62 -0
- data_pipeline/README.md +257 -0
- data_pipeline/__init__.py +6 -0
- data_pipeline/bar_synthesis.py +97 -0
- data_pipeline/dividend_backfill.py +243 -0
- data_pipeline/factor_file_backfill.py +276 -0
- data_pipeline/fetch.py +206 -0
- data_pipeline/fred_backfill.py +435 -0
- data_pipeline/ib_backfill.py +287 -0
- data_pipeline/v2_manifest.py +75 -0
- data_pipeline/yfinance_backfill.py +501 -0
- evaluation/README.md +152 -0
- evaluation/__init__.py +73 -0
- evaluation/ablation.py +310 -0
- evaluation/benchmark_comparison.py +195 -0
- evaluation/confidence_threshold_calibration.py +157 -0
- evaluation/factor_neutralization_check.py +94 -0
- evaluation/feature_parity.py +321 -0
- evaluation/feature_reconciliation.py +286 -0
- evaluation/impact_model.py +77 -0
- evaluation/kill_switch_replay.py +186 -0
- evaluation/limit_fill_simulator.py +152 -0
- evaluation/model_predictions.py +192 -0
- evaluation/monte_carlo.py +161 -0
- evaluation/public_benchmarks.py +117 -0
- evaluation/rank_book_simulator.py +494 -0
- evaluation/rank_ic_core.py +204 -0
- evaluation/rank_signal_calibration.py +690 -0
- evaluation/rolling_ic_gate_calibration.py +190 -0
- evaluation/rolling_ic_gate_replay.py +138 -0
- execution/README.md +281 -0
- execution/__init__.py +80 -0
- execution/config_cache.py +48 -0
- execution/cost_model.py +213 -0
- execution/lean_config_render.py +131 -0
- execution/live_credentials.py +38 -0
- execution/live_credentials_io.py +52 -0
- execution/order_gate.py +388 -0
- execution/paper_readiness.py +181 -0
- execution/paper_readiness_io.py +54 -0
- execution/paper_readiness_report.py +103 -0
- execution/paper_readiness_scheduler.py +137 -0
- execution/reconciliation.py +160 -0
- execution/runtime_config_io.py +38 -0
- execution/secret_scan.py +73 -0
- features/README.md +77 -0
- features/__init__.py +137 -0
- features/alt_data_features.py +154 -0
- features/bond_features.py +275 -0
- features/cross_asset_sensitivity.py +79 -0
- features/derivatives_macro_features.py +72 -0
- features/macro_features.py +79 -0
- features/options_greeks.py +329 -0
- features/sector_map.py +53 -0
- features/technical_indicators.py +185 -0
- generate_backtest_report.py +311 -0
- generate_evaluation_report.py +712 -0
- inference/README.md +105 -0
- inference/__init__.py +36 -0
- inference/exported_model.py +929 -0
- inference/parallel_inference.py +160 -0
- inference/strategy_selector_inference.py +75 -0
- json_safety.py +130 -0
- portfolio/README.md +447 -0
- portfolio/__init__.py +41 -0
- portfolio/book_construction.py +786 -0
- portfolio/book_neutrality.py +178 -0
- portfolio/hrp_allocation.py +159 -0
- portfolio/options_arbitrage_detector.py +232 -0
- portfolio/options_assignment_risk.py +119 -0
- portfolio/options_margin_sizing.py +333 -0
- portfolio/options_strategy.py +1999 -0
- portfolio/rank_signal.py +219 -0
- portfolio/rolling_ic_gate.py +183 -0
- risk/README.md +275 -0
- risk/__init__.py +38 -0
- risk/asset_class_router.py +584 -0
- risk/forex_risk.py +158 -0
- risk/futures_risk.py +202 -0
- risk/kill_switch.py +261 -0
- risk/manual_override.py +118 -0
- risk/position_sizing.py +303 -0
- risk/rl_sizing.py +189 -0
audit/README.md
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# audit
|
|
2
|
+
|
|
3
|
+
Owns the **tamper-evident audit log** — a hash-chained record of
|
|
4
|
+
security-relevant events (credential loads, live-mode transitions, and the
|
|
5
|
+
per-bar order-placement path), added in V2-18 (see
|
|
6
|
+
`development/Problems.md` #42, hardened further in #48).
|
|
7
|
+
|
|
8
|
+
This package is a deliberate structural mirror of `experience/`: same
|
|
9
|
+
Redis-Stream → async-worker → PostgreSQL pipeline, same fire-and-forget
|
|
10
|
+
publish contract, so audit events can be emitted from `main.py`'s hot path
|
|
11
|
+
and from `execution/`'s credential/live-mode call sites without ever risking
|
|
12
|
+
a real trade or blocking a live process on a Postgres round-trip. What makes
|
|
13
|
+
it *audit* rather than *experience* is the hash chain: each row's hash covers
|
|
14
|
+
its own content plus the previous row's hash, so altering or deleting any
|
|
15
|
+
historical row breaks every hash after it (the same construction git commits
|
|
16
|
+
and blockchains use, applied to one linear table).
|
|
17
|
+
|
|
18
|
+
Members:
|
|
19
|
+
|
|
20
|
+
- `hash_chain.py` — pure tamper-evidence logic, no I/O. `compute_entry_hash()`
|
|
21
|
+
(`sha256(prev_hash || event_type || created_at || canonical-JSON payload)`)
|
|
22
|
+
and `verify_chain()`. `GENESIS_HASH` is a fixed all-zero value so an empty
|
|
23
|
+
`prev_hash` can never be confused with "not yet verified".
|
|
24
|
+
- `redis_queue.py` — the fire-and-forget Redis Stream publisher (`aether:audit`),
|
|
25
|
+
mirroring `experience/redis_queue.py`'s never-raises contract exactly.
|
|
26
|
+
- `postgres_worker.py` — drains the Redis Stream via `XREADGROUP` consumer-group
|
|
27
|
+
semantics and batch-inserts into the `audit_log` table, computing each row's
|
|
28
|
+
hash-chain link **at durable-write order** (not push order), so out-of-order
|
|
29
|
+
Redis delivery can never produce a chain that disagrees with on-disk row
|
|
30
|
+
order. Idempotent embedded DDL (`ensure_schema()`), no Alembic — same design
|
|
31
|
+
as `experience/postgres_worker.py`. V5.4.7 (#120): a poison-row batch failure
|
|
32
|
+
persists the clean prefix and dead-letters the poison row PLUS every later
|
|
33
|
+
row (their hashes chain off a parent that never landed - inserting them
|
|
34
|
+
would fork the chain); a Postgres outage still re-raises with everything pending.
|
|
35
|
+
- `postgres_audit.py` — read-only query helpers over `audit_log`, shared by the
|
|
36
|
+
`aq audit-log` CLI and `monitoring/api_server.py`'s `GET /api/audit-log`
|
|
37
|
+
route so the CLI and webui can never drift on row shape. Never inserts.
|
|
38
|
+
- `status_export.py` — sole writer of `visualization/grafana/audit_log.json`,
|
|
39
|
+
refreshed by `postgres_worker.py` after every persisted batch (real-time-ish,
|
|
40
|
+
not a periodic cron export — a stale audit trail defeats its own purpose),
|
|
41
|
+
since `main.py` only ever connects to Redis, not Postgres.
|
|
42
|
+
|
|
43
|
+
Verify the chain end-to-end with `aq audit-log --verify` (re-hashes every row
|
|
44
|
+
and reports the first break, if any). See `development/infrastructure.md`'s
|
|
45
|
+
audit section for the worker's Compose service and `development/Problems.md`
|
|
46
|
+
#42/#48 for the full design rationale and the reconciliation/`init: true`
|
|
47
|
+
hardening.
|
audit/__init__.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Tamper-evident audit trail for order placement, credential loads, and
|
|
2
|
+
live-mode transitions (development/Problems.md #42's deferred item).
|
|
3
|
+
|
|
4
|
+
Same three-piece shape as experience/ (the only other Redis-Stream-backed
|
|
5
|
+
package here): redis_queue.py (pure event builder + fire-and-forget
|
|
6
|
+
publisher, called from main.py/execution/ - never blocks trading),
|
|
7
|
+
postgres_worker.py (drains the stream into a durable, hash-chained Postgres
|
|
8
|
+
table), postgres_audit.py (read-only query helpers shared by the CLI and the
|
|
9
|
+
webui API route). hash_chain.py is the one pure piece unique to this
|
|
10
|
+
package - the tamper-evidence logic itself.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from .hash_chain import GENESIS_HASH, compute_entry_hash, verify_chain
|
|
16
|
+
from .postgres_audit import fetch_all_events_ordered, fetch_recent_events
|
|
17
|
+
from .postgres_worker import PostgresWorker, ensure_schema, event_to_row
|
|
18
|
+
from .redis_queue import CREDENTIAL_LOAD, EVENT_TYPES, LIVE_MODE_TRANSITION, ORDER_PLACEMENT, AuditQueue, build_audit_event
|
|
19
|
+
from .status_export import build_audit_status_view, write_status_file
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"AuditQueue",
|
|
23
|
+
"build_audit_event",
|
|
24
|
+
"CREDENTIAL_LOAD",
|
|
25
|
+
"EVENT_TYPES",
|
|
26
|
+
"LIVE_MODE_TRANSITION",
|
|
27
|
+
"ORDER_PLACEMENT",
|
|
28
|
+
"compute_entry_hash",
|
|
29
|
+
"verify_chain",
|
|
30
|
+
"GENESIS_HASH",
|
|
31
|
+
"ensure_schema",
|
|
32
|
+
"event_to_row",
|
|
33
|
+
"PostgresWorker",
|
|
34
|
+
"fetch_recent_events",
|
|
35
|
+
"fetch_all_events_ordered",
|
|
36
|
+
"build_audit_status_view",
|
|
37
|
+
"write_status_file",
|
|
38
|
+
]
|
audit/hash_chain.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Pure tamper-evidence logic for the audit log (development/Problems.md #42).
|
|
2
|
+
|
|
3
|
+
Each row's hash covers its own content plus the previous row's hash, so
|
|
4
|
+
altering or deleting any historical row breaks every hash after it - the
|
|
5
|
+
same construction git commits / blockchains use, just for one linear table
|
|
6
|
+
instead of a DAG. No I/O here on purpose: audit/postgres_worker.py computes
|
|
7
|
+
each new row's hash at insert time (fetching the current chain tail first),
|
|
8
|
+
and audit/postgres_audit.py's callers (the CLI's --verify, the API route)
|
|
9
|
+
re-verify an already-fetched list of rows through verify_chain() below.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
|
|
17
|
+
# The hash of the very first entry in the chain (no predecessor to link to) -
|
|
18
|
+
# a fixed, well-known genesis value rather than None/empty-string, so an
|
|
19
|
+
# empty prev_hash column can never be confused with "chain not yet verified".
|
|
20
|
+
GENESIS_HASH = "0" * 64
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def compute_entry_hash(prev_hash: str, event_type: str, created_at: str, payload: dict) -> str:
|
|
24
|
+
"""sha256(prev_hash || event_type || created_at || canonical-JSON payload).
|
|
25
|
+
|
|
26
|
+
`created_at` must be a string (ISO 8601), not a datetime - callers own
|
|
27
|
+
formatting so this stays pure and independent of any particular
|
|
28
|
+
Postgres driver's timestamp type. `payload` is serialized with sorted
|
|
29
|
+
keys and no extra whitespace so the same logical event always hashes
|
|
30
|
+
identically regardless of dict insertion order.
|
|
31
|
+
"""
|
|
32
|
+
canonical_payload = json.dumps(payload, sort_keys=True, separators=(",", ":"))
|
|
33
|
+
digest_input = "|".join([prev_hash, event_type, created_at, canonical_payload])
|
|
34
|
+
return hashlib.sha256(digest_input.encode("utf-8")).hexdigest()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def verify_chain(rows: list[dict]) -> tuple[bool, int | None]:
|
|
38
|
+
"""Walks `rows` (each needing prev_hash/hash/event_type/created_at/payload
|
|
39
|
+
keys, oldest-first) recomputing every hash and comparing. Returns
|
|
40
|
+
(all_valid, first_broken_index) - first_broken_index is None when
|
|
41
|
+
all_valid is True, else the 0-based index of the first row whose
|
|
42
|
+
prev_hash doesn't match the prior row's hash, or whose own hash doesn't
|
|
43
|
+
match its recomputed value. An empty list is trivially valid."""
|
|
44
|
+
expected_prev = GENESIS_HASH
|
|
45
|
+
for index, row in enumerate(rows):
|
|
46
|
+
if row["prev_hash"] != expected_prev:
|
|
47
|
+
return False, index
|
|
48
|
+
recomputed = compute_entry_hash(row["prev_hash"], row["event_type"], row["created_at"], row["payload"])
|
|
49
|
+
if recomputed != row["hash"]:
|
|
50
|
+
return False, index
|
|
51
|
+
expected_prev = row["hash"]
|
|
52
|
+
return True, None
|
audit/postgres_audit.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Read-only query helpers over the audit_log table (development/Problems.md #42).
|
|
2
|
+
|
|
3
|
+
Shared by `aq audit-log` (aq_cli.py) and monitoring/api_server.py's
|
|
4
|
+
`GET /api/audit-log` route - one place owns the SELECT shape so the CLI and
|
|
5
|
+
the webui can never drift apart on what a row looks like. Write path lives
|
|
6
|
+
in audit/postgres_worker.py (the Redis-Stream-draining worker); this module
|
|
7
|
+
never inserts anything.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
_COLUMNS = (
|
|
16
|
+
"event_id",
|
|
17
|
+
"created_at",
|
|
18
|
+
"event_type",
|
|
19
|
+
"actor",
|
|
20
|
+
"prev_hash",
|
|
21
|
+
"hash",
|
|
22
|
+
"payload",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _row_to_dict(row: tuple) -> dict[str, Any]:
|
|
27
|
+
return dict(zip(_COLUMNS, row))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def fetch_recent_events(
|
|
31
|
+
conn,
|
|
32
|
+
event_type: str | None = None,
|
|
33
|
+
since: datetime | None = None,
|
|
34
|
+
limit: int = 100,
|
|
35
|
+
) -> list[dict]:
|
|
36
|
+
"""Newest-first page of audit_log rows, optionally filtered. `since`
|
|
37
|
+
defaults to the Unix epoch (no filter) rather than requiring callers to
|
|
38
|
+
special-case None, matching retraining/postgres_registry.py's
|
|
39
|
+
fetch_recent_retraining_events() convention."""
|
|
40
|
+
if since is None:
|
|
41
|
+
since = datetime.fromtimestamp(0, tz=timezone.utc)
|
|
42
|
+
query = f"SELECT {', '.join(_COLUMNS)} FROM audit_log WHERE created_at > %(since)s"
|
|
43
|
+
params: dict[str, Any] = {"since": since, "limit": limit}
|
|
44
|
+
if event_type is not None:
|
|
45
|
+
query += " AND event_type = %(event_type)s"
|
|
46
|
+
params["event_type"] = event_type
|
|
47
|
+
query += " ORDER BY id DESC LIMIT %(limit)s;"
|
|
48
|
+
with conn.cursor() as cur:
|
|
49
|
+
cur.execute(query, params)
|
|
50
|
+
rows = cur.fetchall()
|
|
51
|
+
return [_row_to_dict(row) for row in rows]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def fetch_all_events_ordered(conn) -> list[dict]:
|
|
55
|
+
"""Every audit_log row, oldest-first - the shape audit.hash_chain.verify_chain()
|
|
56
|
+
expects. Only ever used by `aq audit-log --verify`; not paginated
|
|
57
|
+
(verification is meaningless over a partial chain)."""
|
|
58
|
+
with conn.cursor() as cur:
|
|
59
|
+
cur.execute(f"SELECT {', '.join(_COLUMNS)} FROM audit_log ORDER BY id ASC;")
|
|
60
|
+
rows = cur.fetchall()
|
|
61
|
+
return [_row_to_dict(row) for row in rows]
|
audit/postgres_worker.py
ADDED
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
"""PostgreSQL persistence worker for the audit log (development/Problems.md #42).
|
|
2
|
+
|
|
3
|
+
Reads from the Redis Stream ``aether:audit`` via XREADGROUP consumer group
|
|
4
|
+
semantics (identical mechanics to experience/postgres_worker.py) and
|
|
5
|
+
batch-inserts each event into the ``audit_log`` table, computing each row's
|
|
6
|
+
hash-chain link (audit/hash_chain.py) at insert time - chaining happens in
|
|
7
|
+
durable-write order, not push order, so out-of-order Redis delivery can
|
|
8
|
+
never produce a chain that doesn't match on-disk row order.
|
|
9
|
+
|
|
10
|
+
Design decisions (mirrors experience/postgres_worker.py exactly):
|
|
11
|
+
- Sync-only: no asyncio; event volume does not justify async complexity.
|
|
12
|
+
- Embedded DDL: no Alembic; ensure_schema() is idempotent and called on startup.
|
|
13
|
+
- ON CONFLICT (event_id) DO NOTHING: safe idempotent re-delivery.
|
|
14
|
+
- Messages are NOT acked until PG commit succeeds — safe at-least-once delivery.
|
|
15
|
+
- Malformed JSON → dead-letter stream, immediate XACK, log WARNING.
|
|
16
|
+
- _redis_client and _pg_conn constructor params for test injection.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import argparse
|
|
22
|
+
import json
|
|
23
|
+
import logging
|
|
24
|
+
import os
|
|
25
|
+
import time
|
|
26
|
+
from typing import Any
|
|
27
|
+
|
|
28
|
+
from .hash_chain import GENESIS_HASH, compute_entry_hash
|
|
29
|
+
from .status_export import build_audit_status_view, write_status_file
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
# ---------------------------------------------------------------------------
|
|
34
|
+
# Schema DDL — embedded constant, no migration files
|
|
35
|
+
# ---------------------------------------------------------------------------
|
|
36
|
+
|
|
37
|
+
_CREATE_TABLE_SQL = """
|
|
38
|
+
CREATE TABLE IF NOT EXISTS audit_log (
|
|
39
|
+
id BIGSERIAL PRIMARY KEY,
|
|
40
|
+
event_id UUID UNIQUE NOT NULL,
|
|
41
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
42
|
+
ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
43
|
+
event_type VARCHAR(30) NOT NULL,
|
|
44
|
+
actor VARCHAR(30) NOT NULL,
|
|
45
|
+
prev_hash CHAR(64) NOT NULL,
|
|
46
|
+
hash CHAR(64) NOT NULL,
|
|
47
|
+
payload JSONB NOT NULL
|
|
48
|
+
);
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
_CREATE_INDEXES_SQL = [
|
|
52
|
+
"CREATE INDEX IF NOT EXISTS ix_audit_created_at ON audit_log (created_at);",
|
|
53
|
+
"CREATE INDEX IF NOT EXISTS ix_audit_event_type ON audit_log (event_type);",
|
|
54
|
+
"CREATE INDEX IF NOT EXISTS ix_audit_payload ON audit_log USING GIN (payload);",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
_INSERT_SQL = """
|
|
58
|
+
INSERT INTO audit_log
|
|
59
|
+
(event_id, created_at, event_type, actor, prev_hash, hash, payload)
|
|
60
|
+
VALUES
|
|
61
|
+
(%(event_id)s, %(created_at)s, %(event_type)s, %(actor)s,
|
|
62
|
+
%(prev_hash)s, %(hash)s, %(payload)s)
|
|
63
|
+
ON CONFLICT (event_id) DO NOTHING;
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
_LATEST_HASH_SQL = "SELECT hash FROM audit_log ORDER BY id DESC LIMIT 1;"
|
|
67
|
+
|
|
68
|
+
# ---------------------------------------------------------------------------
|
|
69
|
+
# Pure transformation — testable without any I/O
|
|
70
|
+
# ---------------------------------------------------------------------------
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def event_to_row(event: dict, prev_hash: str) -> dict[str, Any]:
|
|
74
|
+
"""Extract INSERT params from an audit event dict plus the chain-tail
|
|
75
|
+
hash it links to. Pure function - computes this row's own hash via
|
|
76
|
+
audit.hash_chain.compute_entry_hash(), never mutates `event`."""
|
|
77
|
+
entry_hash = compute_entry_hash(prev_hash, event["event_type"], event["created_at"], event["payload"])
|
|
78
|
+
return {
|
|
79
|
+
"event_id": event["event_id"],
|
|
80
|
+
"created_at": event["created_at"],
|
|
81
|
+
"event_type": event["event_type"],
|
|
82
|
+
"actor": event.get("actor", "system"),
|
|
83
|
+
"prev_hash": prev_hash,
|
|
84
|
+
"hash": entry_hash,
|
|
85
|
+
"payload": json.dumps(event["payload"]),
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# ---------------------------------------------------------------------------
|
|
90
|
+
# Schema helper
|
|
91
|
+
# ---------------------------------------------------------------------------
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def ensure_schema(conn) -> None:
|
|
95
|
+
"""Create audit_log table and indexes if they do not exist. Idempotent."""
|
|
96
|
+
with conn.cursor() as cur:
|
|
97
|
+
cur.execute(_CREATE_TABLE_SQL)
|
|
98
|
+
for idx_sql in _CREATE_INDEXES_SQL:
|
|
99
|
+
cur.execute(idx_sql)
|
|
100
|
+
conn.commit()
|
|
101
|
+
logger.info("ensure_schema: audit_log ready.")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def fetch_latest_hash(conn) -> str:
|
|
105
|
+
"""The chain tail: the most recently inserted row's hash, or GENESIS_HASH
|
|
106
|
+
if the table is empty. Callers hold this fixed across one run_once()
|
|
107
|
+
batch and thread it forward row-by-row (see PostgresWorker.run_once())
|
|
108
|
+
so a batch of N events chains correctly among themselves before any of
|
|
109
|
+
them are committed."""
|
|
110
|
+
with conn.cursor() as cur:
|
|
111
|
+
cur.execute(_LATEST_HASH_SQL)
|
|
112
|
+
row = cur.fetchone()
|
|
113
|
+
return row[0] if row else GENESIS_HASH
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# ---------------------------------------------------------------------------
|
|
117
|
+
# Worker
|
|
118
|
+
# ---------------------------------------------------------------------------
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class PostgresWorker:
|
|
122
|
+
"""Drains aether:audit Redis Stream into PostgreSQL audit_log.
|
|
123
|
+
|
|
124
|
+
Parameters — identical shape to experience.postgres_worker.PostgresWorker;
|
|
125
|
+
see that class's docstring for the full parameter list. Differs only in
|
|
126
|
+
stream_name/group_name defaults and in computing+persisting the
|
|
127
|
+
hash-chain link for every row (see event_to_row()/fetch_latest_hash()).
|
|
128
|
+
"""
|
|
129
|
+
|
|
130
|
+
def __init__(
|
|
131
|
+
self,
|
|
132
|
+
*,
|
|
133
|
+
redis_url: str = "redis://localhost:6380/0",
|
|
134
|
+
postgres_dsn: str,
|
|
135
|
+
stream_name: str = "aether:audit",
|
|
136
|
+
group_name: str = "aether-audit-workers",
|
|
137
|
+
consumer_name: str = "worker-1",
|
|
138
|
+
batch_size: int = 100,
|
|
139
|
+
deadletter_stream: str = "aether:audit:deadletter",
|
|
140
|
+
backoff_max: int = 60,
|
|
141
|
+
_redis_client=None,
|
|
142
|
+
_pg_conn=None,
|
|
143
|
+
) -> None:
|
|
144
|
+
self.stream_name = stream_name
|
|
145
|
+
self.group_name = group_name
|
|
146
|
+
self.consumer_name = consumer_name
|
|
147
|
+
self.batch_size = batch_size
|
|
148
|
+
self.deadletter_stream = deadletter_stream
|
|
149
|
+
self.backoff_max = backoff_max
|
|
150
|
+
|
|
151
|
+
if _redis_client is not None:
|
|
152
|
+
self._redis = _redis_client
|
|
153
|
+
else:
|
|
154
|
+
import redis as redis_lib
|
|
155
|
+
url = os.environ.get("AETHER_REDIS_URL", redis_url)
|
|
156
|
+
self._redis = redis_lib.from_url(url, socket_connect_timeout=5, socket_timeout=5)
|
|
157
|
+
# V5.4.7 (#120): warn instead of crash-looping when Redis is
|
|
158
|
+
# down at container start; run()'s backoff loop retries.
|
|
159
|
+
try:
|
|
160
|
+
self._redis.ping()
|
|
161
|
+
logger.info("AuditPostgresWorker: Redis connected at %s", url)
|
|
162
|
+
except Exception as exc:
|
|
163
|
+
logger.warning("AuditPostgresWorker: Redis not reachable yet (%s) - will retry in run loop.", exc)
|
|
164
|
+
|
|
165
|
+
if _pg_conn is not None:
|
|
166
|
+
self._conn = _pg_conn
|
|
167
|
+
else:
|
|
168
|
+
import psycopg
|
|
169
|
+
dsn = os.environ.get("AETHER_POSTGRES_DSN", postgres_dsn)
|
|
170
|
+
self._conn = psycopg.connect(dsn, autocommit=False)
|
|
171
|
+
logger.info("AuditPostgresWorker: PostgreSQL connected.")
|
|
172
|
+
|
|
173
|
+
ensure_schema(self._conn)
|
|
174
|
+
self._ensure_consumer_group()
|
|
175
|
+
|
|
176
|
+
def _ensure_consumer_group(self) -> None:
|
|
177
|
+
try:
|
|
178
|
+
self._redis.xgroup_create(
|
|
179
|
+
self.stream_name, self.group_name, id="0", mkstream=True
|
|
180
|
+
)
|
|
181
|
+
logger.info("Consumer group '%s' created on '%s'.", self.group_name, self.stream_name)
|
|
182
|
+
except Exception as exc:
|
|
183
|
+
if "BUSYGROUP" in str(exc):
|
|
184
|
+
logger.debug("Consumer group '%s' already exists.", self.group_name)
|
|
185
|
+
else:
|
|
186
|
+
raise
|
|
187
|
+
|
|
188
|
+
def _reconnect_pg(self, postgres_dsn: str) -> None:
|
|
189
|
+
import psycopg
|
|
190
|
+
try:
|
|
191
|
+
self._conn.close()
|
|
192
|
+
except Exception:
|
|
193
|
+
pass
|
|
194
|
+
dsn = os.environ.get("AETHER_POSTGRES_DSN", postgres_dsn)
|
|
195
|
+
self._conn = psycopg.connect(dsn, autocommit=False)
|
|
196
|
+
ensure_schema(self._conn)
|
|
197
|
+
logger.info("AuditPostgresWorker: PostgreSQL reconnected.")
|
|
198
|
+
|
|
199
|
+
def run_once(self) -> int:
|
|
200
|
+
"""Read one batch, hash-chain it, persist, ack. Returns count of
|
|
201
|
+
events persisted. Raises on PostgreSQL failure — messages stay
|
|
202
|
+
pending for re-delivery."""
|
|
203
|
+
results = self._redis.xreadgroup(
|
|
204
|
+
self.group_name,
|
|
205
|
+
self.consumer_name,
|
|
206
|
+
{self.stream_name: ">"},
|
|
207
|
+
count=self.batch_size,
|
|
208
|
+
# block omitted (not 0 - that means "block forever" in Redis,
|
|
209
|
+
# not "don't block") so this returns immediately; run() provides
|
|
210
|
+
# the idle sleep.
|
|
211
|
+
)
|
|
212
|
+
if not results:
|
|
213
|
+
return 0
|
|
214
|
+
|
|
215
|
+
_, messages = results[0]
|
|
216
|
+
rows: list[dict] = []
|
|
217
|
+
ids_good: list = []
|
|
218
|
+
ids_deadletter: list = []
|
|
219
|
+
raw_by_id: dict = {}
|
|
220
|
+
running_hash = fetch_latest_hash(self._conn)
|
|
221
|
+
|
|
222
|
+
for msg_id, fields in messages:
|
|
223
|
+
raw = fields.get(b"payload") or fields.get("payload", b"")
|
|
224
|
+
raw_by_id[msg_id] = raw if isinstance(raw, (bytes, bytearray)) else str(raw).encode("utf-8")
|
|
225
|
+
try:
|
|
226
|
+
event = json.loads(raw)
|
|
227
|
+
row = event_to_row(event, running_hash)
|
|
228
|
+
rows.append(row)
|
|
229
|
+
running_hash = row["hash"] # chain the NEXT event in this batch to THIS one
|
|
230
|
+
ids_good.append(msg_id)
|
|
231
|
+
except Exception as exc:
|
|
232
|
+
logger.warning("Malformed message %s → dead-letter. Error: %s", msg_id, exc)
|
|
233
|
+
self._redis.xadd(
|
|
234
|
+
self.deadletter_stream,
|
|
235
|
+
{"payload": raw, "error": str(exc), "original_id": str(msg_id)},
|
|
236
|
+
)
|
|
237
|
+
ids_deadletter.append(msg_id)
|
|
238
|
+
|
|
239
|
+
if ids_deadletter:
|
|
240
|
+
self._redis.xack(self.stream_name, self.group_name, *ids_deadletter)
|
|
241
|
+
|
|
242
|
+
# V5.4.7 (Problems.md #120): a single poison row used to fail the
|
|
243
|
+
# whole executemany, roll back every good row, and leave the batch
|
|
244
|
+
# pending forever (xreadgroup(">") never re-delivers) - wedging
|
|
245
|
+
# the pipeline on one bad payload. Unlike the experience worker,
|
|
246
|
+
# this chain CANNOT skip a failed middle row: every row's hash
|
|
247
|
+
# chains to its predecessor's, so once one row cannot be inserted
|
|
248
|
+
# every LATER row in the batch would fork the tamper-evident chain.
|
|
249
|
+
# Fallback therefore inserts per-row IN ORDER and stops at the
|
|
250
|
+
# first failure - that row and everything after it dead-letters
|
|
251
|
+
# (payloads preserved verbatim), everything before it commits.
|
|
252
|
+
if rows:
|
|
253
|
+
try:
|
|
254
|
+
with self._conn.cursor() as cur:
|
|
255
|
+
cur.executemany(_INSERT_SQL, rows)
|
|
256
|
+
self._conn.commit()
|
|
257
|
+
self._redis.xack(self.stream_name, self.group_name, *ids_good)
|
|
258
|
+
logger.info("AuditPostgresWorker: persisted %d events.", len(rows))
|
|
259
|
+
except Exception as batch_exc:
|
|
260
|
+
# V5.4.7 (#120) - poison row vs postgres down: first-row
|
|
261
|
+
# failure means the connection/statement, not the data -
|
|
262
|
+
# re-raise so everything stays pending (old contract). A
|
|
263
|
+
# batch whose head inserts fine takes the per-row path,
|
|
264
|
+
# which for this hash-chained stream STOPS at the first
|
|
265
|
+
# poison row and dead-letters it plus every LATER row
|
|
266
|
+
# (their hashes chain off a parent that never landed -
|
|
267
|
+
# inserting them would fork the tamper-evident chain).
|
|
268
|
+
try:
|
|
269
|
+
with self._conn.cursor() as cur:
|
|
270
|
+
cur.execute(_INSERT_SQL, rows[0])
|
|
271
|
+
self._conn.commit()
|
|
272
|
+
except Exception:
|
|
273
|
+
self._conn.rollback()
|
|
274
|
+
raise
|
|
275
|
+
self._redis.xack(self.stream_name, self.group_name, ids_good[0])
|
|
276
|
+
persisted = 1
|
|
277
|
+
for msg_id, row in zip(ids_good[1:], rows[1:]):
|
|
278
|
+
try:
|
|
279
|
+
with self._conn.cursor() as cur:
|
|
280
|
+
cur.execute(_INSERT_SQL, row)
|
|
281
|
+
self._conn.commit()
|
|
282
|
+
self._redis.xack(self.stream_name, self.group_name, msg_id)
|
|
283
|
+
persisted += 1
|
|
284
|
+
except Exception as row_exc:
|
|
285
|
+
self._conn.rollback()
|
|
286
|
+
logger.warning(
|
|
287
|
+
"Poison row %s → dead-letter; stopping - later rows "
|
|
288
|
+
"chain off this one's hash. Error: %s", msg_id, row_exc
|
|
289
|
+
)
|
|
290
|
+
break
|
|
291
|
+
for msg_id in ids_good[persisted:]:
|
|
292
|
+
raw = raw_by_id.get(msg_id)
|
|
293
|
+
self._redis.xadd(
|
|
294
|
+
self.deadletter_stream,
|
|
295
|
+
{
|
|
296
|
+
"payload": raw if raw is not None else b"",
|
|
297
|
+
"error": f"chained after poison row: {batch_exc}",
|
|
298
|
+
"original_id": str(msg_id),
|
|
299
|
+
},
|
|
300
|
+
)
|
|
301
|
+
self._redis.xack(self.stream_name, self.group_name, msg_id)
|
|
302
|
+
logger.info(
|
|
303
|
+
"AuditPostgresWorker: per-row fallback persisted %d/%d events.",
|
|
304
|
+
persisted, len(rows),
|
|
305
|
+
)
|
|
306
|
+
return persisted
|
|
307
|
+
# Refresh the webui/API dashboard snapshot right after a real
|
|
308
|
+
# persist (not on every idle poll) - a stale audit trail
|
|
309
|
+
# defeats its own purpose, so this stays close to real-time
|
|
310
|
+
# rather than a periodic cron export. Best-effort: a dashboard
|
|
311
|
+
# export failure must never lose already-committed audit rows.
|
|
312
|
+
try:
|
|
313
|
+
write_status_file(build_audit_status_view(self._conn))
|
|
314
|
+
except Exception as exc:
|
|
315
|
+
logger.warning("AuditPostgresWorker: dashboard export failed: %s", exc)
|
|
316
|
+
|
|
317
|
+
return len(rows)
|
|
318
|
+
|
|
319
|
+
def run(self, postgres_dsn: str = "") -> None:
|
|
320
|
+
"""Continuous loop with exponential backoff on errors."""
|
|
321
|
+
backoff = 1
|
|
322
|
+
logger.info("AuditPostgresWorker: entering run loop.")
|
|
323
|
+
while True:
|
|
324
|
+
try:
|
|
325
|
+
count = self.run_once()
|
|
326
|
+
if count == 0:
|
|
327
|
+
time.sleep(1) # idle poll interval
|
|
328
|
+
else:
|
|
329
|
+
backoff = 1
|
|
330
|
+
except KeyboardInterrupt:
|
|
331
|
+
logger.info("AuditPostgresWorker: shutdown requested.")
|
|
332
|
+
break
|
|
333
|
+
except Exception as exc:
|
|
334
|
+
logger.error("Worker error — %s. Retrying in %ds.", exc, backoff)
|
|
335
|
+
time.sleep(backoff)
|
|
336
|
+
backoff = min(backoff * 2, self.backoff_max)
|
|
337
|
+
if postgres_dsn and "connection" in str(exc).lower():
|
|
338
|
+
try:
|
|
339
|
+
self._reconnect_pg(postgres_dsn)
|
|
340
|
+
except Exception as reconn_exc:
|
|
341
|
+
logger.error("Reconnect failed: %s", reconn_exc)
|
|
342
|
+
|
|
343
|
+
def close(self) -> None:
|
|
344
|
+
try:
|
|
345
|
+
self._redis.close()
|
|
346
|
+
except Exception:
|
|
347
|
+
pass
|
|
348
|
+
try:
|
|
349
|
+
self._conn.close()
|
|
350
|
+
except Exception:
|
|
351
|
+
pass
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
# ---------------------------------------------------------------------------
|
|
355
|
+
# Entry point
|
|
356
|
+
# ---------------------------------------------------------------------------
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def main() -> None:
|
|
360
|
+
logging.basicConfig(
|
|
361
|
+
level=logging.INFO,
|
|
362
|
+
format="%(asctime)s [%(levelname)s] %(name)s — %(message)s",
|
|
363
|
+
)
|
|
364
|
+
parser = argparse.ArgumentParser(description="Aether Quant audit-log persistence worker")
|
|
365
|
+
parser.add_argument("--once", action="store_true", help="Read one batch and exit")
|
|
366
|
+
parser.add_argument(
|
|
367
|
+
"--stream",
|
|
368
|
+
default=os.environ.get("AETHER_AUDIT_STREAM", "aether:audit"),
|
|
369
|
+
)
|
|
370
|
+
parser.add_argument(
|
|
371
|
+
"--group",
|
|
372
|
+
default=os.environ.get("AETHER_AUDIT_GROUP", "aether-audit-workers"),
|
|
373
|
+
)
|
|
374
|
+
parser.add_argument(
|
|
375
|
+
"--batch-size",
|
|
376
|
+
type=int,
|
|
377
|
+
default=int(os.environ.get("AETHER_AUDIT_BATCH_SIZE", "100")),
|
|
378
|
+
)
|
|
379
|
+
args = parser.parse_args()
|
|
380
|
+
|
|
381
|
+
redis_url = os.environ.get("AETHER_REDIS_URL", "redis://localhost:6380/0")
|
|
382
|
+
postgres_dsn = os.environ.get("AETHER_POSTGRES_DSN", "")
|
|
383
|
+
|
|
384
|
+
worker = PostgresWorker(
|
|
385
|
+
redis_url=redis_url,
|
|
386
|
+
postgres_dsn=postgres_dsn,
|
|
387
|
+
stream_name=args.stream,
|
|
388
|
+
group_name=args.group,
|
|
389
|
+
batch_size=args.batch_size,
|
|
390
|
+
)
|
|
391
|
+
try:
|
|
392
|
+
if args.once:
|
|
393
|
+
count = worker.run_once()
|
|
394
|
+
logger.info("--once: persisted %d events.", count)
|
|
395
|
+
else:
|
|
396
|
+
worker.run(postgres_dsn=postgres_dsn)
|
|
397
|
+
finally:
|
|
398
|
+
worker.close()
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
if __name__ == "__main__":
|
|
402
|
+
main()
|