aether-quant 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aether_quant-0.0.1.dist-info/METADATA +7 -0
- aether_quant-0.0.1.dist-info/RECORD +99 -0
- aether_quant-0.0.1.dist-info/WHEEL +5 -0
- aether_quant-0.0.1.dist-info/entry_points.txt +2 -0
- aether_quant-0.0.1.dist-info/licenses/LICENSE +133 -0
- aether_quant-0.0.1.dist-info/top_level.txt +13 -0
- analyzer/README.md +69 -0
- analyzer/__init__.py +17 -0
- analyzer/market_analyzer.py +521 -0
- aq_cli.py +3633 -0
- audit/README.md +47 -0
- audit/__init__.py +38 -0
- audit/hash_chain.py +52 -0
- audit/postgres_audit.py +61 -0
- audit/postgres_worker.py +402 -0
- audit/redis_queue.py +124 -0
- audit/status_export.py +62 -0
- data_pipeline/README.md +257 -0
- data_pipeline/__init__.py +6 -0
- data_pipeline/bar_synthesis.py +97 -0
- data_pipeline/dividend_backfill.py +243 -0
- data_pipeline/factor_file_backfill.py +276 -0
- data_pipeline/fetch.py +206 -0
- data_pipeline/fred_backfill.py +435 -0
- data_pipeline/ib_backfill.py +287 -0
- data_pipeline/v2_manifest.py +75 -0
- data_pipeline/yfinance_backfill.py +501 -0
- evaluation/README.md +152 -0
- evaluation/__init__.py +73 -0
- evaluation/ablation.py +310 -0
- evaluation/benchmark_comparison.py +195 -0
- evaluation/confidence_threshold_calibration.py +157 -0
- evaluation/factor_neutralization_check.py +94 -0
- evaluation/feature_parity.py +321 -0
- evaluation/feature_reconciliation.py +286 -0
- evaluation/impact_model.py +77 -0
- evaluation/kill_switch_replay.py +186 -0
- evaluation/limit_fill_simulator.py +152 -0
- evaluation/model_predictions.py +192 -0
- evaluation/monte_carlo.py +161 -0
- evaluation/public_benchmarks.py +117 -0
- evaluation/rank_book_simulator.py +494 -0
- evaluation/rank_ic_core.py +204 -0
- evaluation/rank_signal_calibration.py +690 -0
- evaluation/rolling_ic_gate_calibration.py +190 -0
- evaluation/rolling_ic_gate_replay.py +138 -0
- execution/README.md +281 -0
- execution/__init__.py +80 -0
- execution/config_cache.py +48 -0
- execution/cost_model.py +213 -0
- execution/lean_config_render.py +131 -0
- execution/live_credentials.py +38 -0
- execution/live_credentials_io.py +52 -0
- execution/order_gate.py +388 -0
- execution/paper_readiness.py +181 -0
- execution/paper_readiness_io.py +54 -0
- execution/paper_readiness_report.py +103 -0
- execution/paper_readiness_scheduler.py +137 -0
- execution/reconciliation.py +160 -0
- execution/runtime_config_io.py +38 -0
- execution/secret_scan.py +73 -0
- features/README.md +77 -0
- features/__init__.py +137 -0
- features/alt_data_features.py +154 -0
- features/bond_features.py +275 -0
- features/cross_asset_sensitivity.py +79 -0
- features/derivatives_macro_features.py +72 -0
- features/macro_features.py +79 -0
- features/options_greeks.py +329 -0
- features/sector_map.py +53 -0
- features/technical_indicators.py +185 -0
- generate_backtest_report.py +311 -0
- generate_evaluation_report.py +712 -0
- inference/README.md +105 -0
- inference/__init__.py +36 -0
- inference/exported_model.py +929 -0
- inference/parallel_inference.py +160 -0
- inference/strategy_selector_inference.py +75 -0
- json_safety.py +130 -0
- portfolio/README.md +447 -0
- portfolio/__init__.py +41 -0
- portfolio/book_construction.py +786 -0
- portfolio/book_neutrality.py +178 -0
- portfolio/hrp_allocation.py +159 -0
- portfolio/options_arbitrage_detector.py +232 -0
- portfolio/options_assignment_risk.py +119 -0
- portfolio/options_margin_sizing.py +333 -0
- portfolio/options_strategy.py +1999 -0
- portfolio/rank_signal.py +219 -0
- portfolio/rolling_ic_gate.py +183 -0
- risk/README.md +275 -0
- risk/__init__.py +38 -0
- risk/asset_class_router.py +584 -0
- risk/forex_risk.py +158 -0
- risk/futures_risk.py +202 -0
- risk/kill_switch.py +261 -0
- risk/manual_override.py +118 -0
- risk/position_sizing.py +303 -0
- risk/rl_sizing.py +189 -0
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""Opt-in per-symbol multiprocessing for main.py::on_data()'s Pass 1 heavy
|
|
2
|
+
inference (baseline + experts + multitask + expert-multitask + sequence
|
|
3
|
+
model calls) - see main.py's phase_v2.inference_parallelism.enabled flag
|
|
4
|
+
(default off).
|
|
5
|
+
|
|
6
|
+
Honest framing, not oversold: per-symbol inference is now fast (~4.8ms
|
|
7
|
+
mean per symbol, post the weight-caching/batching pass - see
|
|
8
|
+
development/Problems.md) precisely because the previous, much larger cost
|
|
9
|
+
(Python/NumPy per-call dispatch overhead, a Python loop inside
|
|
10
|
+
_conv1d_causal) got fixed WITHOUT multiprocessing. IPC/pickling overhead
|
|
11
|
+
for submitting work to and receiving results from a separate process may
|
|
12
|
+
easily exceed any parallel win at this point, for a universe this size
|
|
13
|
+
(~30 symbols). Shipped default-off; the real judge is a real `lean
|
|
14
|
+
backtest .` run (Windows' ProcessPoolExecutor uses the `spawn` start
|
|
15
|
+
method, which re-bootstraps a new interpreter per worker - untested
|
|
16
|
+
territory inside Lean's own embedded-Python runtime, which is not a
|
|
17
|
+
standard `python.exe` process tree main.py was ever designed to be
|
|
18
|
+
spawned from. main.py's own pool-creation code treats ANY failure here,
|
|
19
|
+
including a hang on first use, as a signal to permanently fall back to
|
|
20
|
+
the always-correct sequential path for the rest of the run.)
|
|
21
|
+
|
|
22
|
+
Workers hold their OWN copy of every model export (loaded once via
|
|
23
|
+
ProcessPoolExecutor's `initializer`, not re-sent on every call - sending
|
|
24
|
+
model weights, now real NumPy arrays post the weight-caching pass,
|
|
25
|
+
through IPC on every single bar would defeat the whole purpose). Only
|
|
26
|
+
small per-symbol data (a flat feature vector, an optional sequence-window
|
|
27
|
+
history) crosses the process boundary per call. Never a bound method on
|
|
28
|
+
`self` (the QCAlgorithm instance holds Lean/.NET interop objects that are
|
|
29
|
+
almost certainly unpicklable) - this module has zero Lean/main.py
|
|
30
|
+
dependency, by design.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
# Populated once per worker process by init_worker() below - never sent
|
|
36
|
+
# across the process boundary again after that, and never touched by the
|
|
37
|
+
# main process itself.
|
|
38
|
+
_worker_model_exports: dict | None = None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def windows_parallelism_slowdown_warning(platform: str) -> str | None:
|
|
42
|
+
"""Returns the Windows-specific inference-parallelism slowdown warning
|
|
43
|
+
for main.py to self.Debug() once, right after the pool starts
|
|
44
|
+
successfully - None on every other platform, since this has only ever
|
|
45
|
+
been measured on Windows (development/Problems.md #65: Windows'
|
|
46
|
+
spawn-based ProcessPoolExecutor measured dramatically slower than
|
|
47
|
+
sequential - single-digit-ms sequential bars vs. multi-second, in one
|
|
48
|
+
run five-figure-ms, pooled bars). Never claims anything about
|
|
49
|
+
Linux/fork, which this project has never measured."""
|
|
50
|
+
if platform != "win32":
|
|
51
|
+
return None
|
|
52
|
+
return (
|
|
53
|
+
"phase_v2.inference_parallelism.enabled is active on Windows - "
|
|
54
|
+
"development/Problems.md #65 measured this ProcessPoolExecutor "
|
|
55
|
+
"pool as dramatically slower than sequential on Windows' spawn "
|
|
56
|
+
"start method (single-digit-ms sequential bars vs. multi-second, "
|
|
57
|
+
"in one run five-figure-ms, pooled bars). Not measured on "
|
|
58
|
+
"Linux/fork. Consider setting phase_v2.inference_parallelism.enabled "
|
|
59
|
+
"back to false unless your own measurement shows otherwise."
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def init_worker(model_exports: dict) -> None:
|
|
64
|
+
"""ProcessPoolExecutor `initializer` - runs once when each worker
|
|
65
|
+
process starts, storing this run's model exports (baseline, experts,
|
|
66
|
+
multitask, expert-multitask, sequence, plus their precomputed
|
|
67
|
+
BatchedLayerStackCache/BatchedMultitaskLayerStackCache instances) in
|
|
68
|
+
that worker's own global state."""
|
|
69
|
+
global _worker_model_exports
|
|
70
|
+
_worker_model_exports = model_exports
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _padded_sequence(history: list[list[float]], window_size: int) -> list[list[float]]:
|
|
74
|
+
"""Same left-zero-padding logic main.py::_run_sequence_model() already
|
|
75
|
+
applies - duplicated here (a handful of lines) rather than shared,
|
|
76
|
+
since this module must have zero main.py/Lean dependency and the
|
|
77
|
+
logic is small enough that a shared-helper extraction isn't worth the
|
|
78
|
+
added indirection."""
|
|
79
|
+
if not history:
|
|
80
|
+
return []
|
|
81
|
+
input_width = len(history[0])
|
|
82
|
+
padding_needed = window_size - len(history)
|
|
83
|
+
return [[0.0] * input_width for _ in range(max(0, padding_needed))] + list(history)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def run_symbol_inference(
|
|
87
|
+
inputs: list[float],
|
|
88
|
+
sequence_history: list[list[float]] | None,
|
|
89
|
+
sequence_window_size: int,
|
|
90
|
+
) -> dict:
|
|
91
|
+
"""The actual per-symbol work submitted to the pool - reads this
|
|
92
|
+
worker's own _worker_model_exports (set once by init_worker()) and
|
|
93
|
+
runs the exact same 5-call cluster main.py::on_data()'s Pass 1 always
|
|
94
|
+
ran inline: baseline, sequence (if history given), experts (batched
|
|
95
|
+
when possible), multitask, expert-multitask (batched when possible).
|
|
96
|
+
|
|
97
|
+
Returns a plain dict of plain values - always picklable back to the
|
|
98
|
+
main process, matching the same graceful-degradation contract
|
|
99
|
+
(missing/failed pieces -> None/empty, never a raised exception
|
|
100
|
+
propagating out) every other inference call site in this codebase
|
|
101
|
+
already has."""
|
|
102
|
+
from inference.exported_model import (
|
|
103
|
+
run_exported_model,
|
|
104
|
+
run_exported_multitask_model,
|
|
105
|
+
run_exported_multitask_models_batched,
|
|
106
|
+
run_exported_models_batched,
|
|
107
|
+
run_exported_sequence_multitask_model,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
exports = _worker_model_exports or {}
|
|
111
|
+
expert_names: list[str] = exports.get("expert_names", [])
|
|
112
|
+
|
|
113
|
+
result: dict = {
|
|
114
|
+
"baseline_probability": None,
|
|
115
|
+
"sequence_result": None,
|
|
116
|
+
"expert_probabilities": dict.fromkeys(expert_names),
|
|
117
|
+
"multitask_result": None,
|
|
118
|
+
"expert_multitask_magnitudes": dict.fromkeys(expert_names),
|
|
119
|
+
"expert_multitask_volatilities": dict.fromkeys(expert_names),
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
baseline = exports.get("baseline")
|
|
123
|
+
if baseline:
|
|
124
|
+
try:
|
|
125
|
+
result["baseline_probability"] = run_exported_model(baseline, inputs)
|
|
126
|
+
except Exception:
|
|
127
|
+
pass
|
|
128
|
+
|
|
129
|
+
sequence = exports.get("sequence")
|
|
130
|
+
if sequence and sequence_history:
|
|
131
|
+
try:
|
|
132
|
+
padded = _padded_sequence(sequence_history, sequence_window_size)
|
|
133
|
+
result["sequence_result"] = run_exported_sequence_multitask_model(sequence, padded)
|
|
134
|
+
except Exception:
|
|
135
|
+
pass
|
|
136
|
+
|
|
137
|
+
expert_exports = exports.get("experts", {})
|
|
138
|
+
expert_list = [expert_exports.get(name) for name in expert_names]
|
|
139
|
+
if any(expert_list):
|
|
140
|
+
expert_results = run_exported_models_batched(expert_list, inputs, stack_cache=exports.get("expert_stack_cache"))
|
|
141
|
+
result["expert_probabilities"] = dict(zip(expert_names, expert_results))
|
|
142
|
+
|
|
143
|
+
multitask = exports.get("multitask")
|
|
144
|
+
if multitask:
|
|
145
|
+
try:
|
|
146
|
+
result["multitask_result"] = run_exported_multitask_model(multitask, inputs)
|
|
147
|
+
except Exception:
|
|
148
|
+
pass
|
|
149
|
+
|
|
150
|
+
expert_multitask_exports = exports.get("expert_multitask", {})
|
|
151
|
+
expert_multitask_list = [expert_multitask_exports.get(name) for name in expert_names]
|
|
152
|
+
if any(expert_multitask_list):
|
|
153
|
+
emt_results = run_exported_multitask_models_batched(
|
|
154
|
+
expert_multitask_list, inputs, stack_cache=exports.get("expert_multitask_stack_cache")
|
|
155
|
+
)
|
|
156
|
+
for name, one_result in zip(expert_names, emt_results):
|
|
157
|
+
result["expert_multitask_magnitudes"][name] = one_result.get("magnitude") if one_result else None
|
|
158
|
+
result["expert_multitask_volatilities"][name] = one_result.get("volatility") if one_result else None
|
|
159
|
+
|
|
160
|
+
return result
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Runtime inference for the learned multi-leg strategy-selector model
|
|
2
|
+
(V4.7, development/Problems.md #29's own framing - a model that picks
|
|
3
|
+
which enabled strategy to prefer, replacing/augmenting
|
|
4
|
+
portfolio/options_strategy.py::order_enabled_strategies()'s static
|
|
5
|
+
risk-tier-preference reordering).
|
|
6
|
+
|
|
7
|
+
Pure Python, no numpy/sklearn at runtime - same "trained offline
|
|
8
|
+
(train_strategy_selector.py), loaded as a plain JSON dict, degrades
|
|
9
|
+
node-by-node (here: strategy-by-strategy) to an empty score dict when
|
|
10
|
+
missing/malformed" split topology/learned_topology.py already establishes
|
|
11
|
+
for the topology model. Never raises -
|
|
12
|
+
risk/asset_class_router.py::route_multi_leg_option_sizing() falls back to
|
|
13
|
+
today's exact order_enabled_strategies() ordering whenever this module
|
|
14
|
+
returns {} (its only output when no trained model exists, which is the
|
|
15
|
+
ONLY state this codebase can reach today - see train_strategy_selector.py's
|
|
16
|
+
own module docstring for why: zero real option positions have ever
|
|
17
|
+
traded, so there is no per-strategy realized-outcome data to train from
|
|
18
|
+
yet).
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
FEATURE_KEYS = ("regime_risk_score", "regime_trend_score", "topology_correlation_strength")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def build_strategy_selector_features(base_features: dict, topology: dict) -> dict[str, float]:
|
|
27
|
+
"""Pure, degrade-to-0.0 feature extraction reusing values main.py
|
|
28
|
+
already computes every bar (base_features["regime_signal_risk_score"/
|
|
29
|
+
"regime_signal_trend_score"], topology["correlation_strength"]) -
|
|
30
|
+
deliberately NOT a new computation path, so this feature vector stays
|
|
31
|
+
in lockstep with whatever the main probability model itself already
|
|
32
|
+
sees, and needs no additional per-bar state. Never raises on a
|
|
33
|
+
missing key. Key names here (bare, e.g. "regime_risk_score") match
|
|
34
|
+
train_strategy_selector.py's own extraction from a persisted
|
|
35
|
+
option_strategy_outcome event's regime/topology sub-payloads (which
|
|
36
|
+
use the SAME bare key names main.py's regime_payload/topology_payload
|
|
37
|
+
dicts do, before base_features's own "regime_signal_"/"topology_"
|
|
38
|
+
prefixing) - the trainer and this runtime scorer must always agree on
|
|
39
|
+
FEATURE_KEYS; this module owns that contract for the runtime side."""
|
|
40
|
+
return {
|
|
41
|
+
"regime_risk_score": float(base_features.get("regime_signal_risk_score", 0.0) or 0.0),
|
|
42
|
+
"regime_trend_score": float(base_features.get("regime_signal_trend_score", 0.0) or 0.0),
|
|
43
|
+
"topology_correlation_strength": float((topology or {}).get("correlation_strength", 0.0) or 0.0),
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def score_strategies(model: dict | None, features: dict[str, float]) -> dict[str, float]:
|
|
48
|
+
"""model is train_strategy_selector.py's own strategy_selector_model.json
|
|
49
|
+
payload: {"strategy_names": [...], "scorers": {name: {"weights":
|
|
50
|
+
{feature_key: w}, "bias": b}}, "feature_keys": [...]}. Returns {}
|
|
51
|
+
(never raises) when model is None/empty/malformed - the exact
|
|
52
|
+
byte-identical-default trigger
|
|
53
|
+
risk/asset_class_router.py::route_multi_leg_option_sizing() relies on
|
|
54
|
+
(falls back to order_enabled_strategies() whenever this is falsy).
|
|
55
|
+
Otherwise a plain linear score per strategy_name: bias + sum(weight *
|
|
56
|
+
feature_value) - deliberately simple (this is a brand-new,
|
|
57
|
+
realistically-always-under-threshold data stream, per
|
|
58
|
+
train_strategy_selector.py's own docstring - not worth a more complex
|
|
59
|
+
runtime scorer until real training data exists to justify one)."""
|
|
60
|
+
if not model:
|
|
61
|
+
return {}
|
|
62
|
+
scorers = model.get("scorers")
|
|
63
|
+
if not scorers:
|
|
64
|
+
return {}
|
|
65
|
+
|
|
66
|
+
scores: dict[str, float] = {}
|
|
67
|
+
for strategy_name, scorer in scorers.items():
|
|
68
|
+
try:
|
|
69
|
+
weights = scorer.get("weights", {})
|
|
70
|
+
bias = float(scorer.get("bias", 0.0))
|
|
71
|
+
score = bias + sum(float(weights.get(key, 0.0)) * float(value) for key, value in features.items())
|
|
72
|
+
except (TypeError, ValueError, AttributeError):
|
|
73
|
+
continue
|
|
74
|
+
scores[strategy_name] = score
|
|
75
|
+
return scores
|
json_safety.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Shared JSON-safety helpers (V5.4.7, development/Problems.md #120).
|
|
2
|
+
|
|
3
|
+
Three failure modes kept biting every JSON producer in this repo
|
|
4
|
+
independently:
|
|
5
|
+
|
|
6
|
+
1. **Non-finite floats** — `json.dumps` happily emits bare `NaN`/`Infinity`
|
|
7
|
+
(Python-tolerant, but PostgreSQL JSONB rejects them and browser
|
|
8
|
+
`JSON.parse` throws), so one poisoned event wedged the whole Redis →
|
|
9
|
+
Postgres pipeline (Problems.md #120) and a NaN state value broke the
|
|
10
|
+
webui's response parsing.
|
|
11
|
+
2. **numpy scalar types** — `np.float64`/`np.int64` raise `TypeError` under
|
|
12
|
+
`json.dumps`, silently dropping events whose payloads came straight from
|
|
13
|
+
pandas.
|
|
14
|
+
3. **Non-atomic writes** — truncate-then-`write_text` leaves a torn file
|
|
15
|
+
whenever a reader races the writer or the writer dies mid-write; every
|
|
16
|
+
reader of these files (`monitoring/api_server.py`, both workers) then
|
|
17
|
+
either 500s or reads half a document.
|
|
18
|
+
|
|
19
|
+
One module, three pure functions, stdlib-only (plus numpy when present),
|
|
20
|
+
so any layer can import it without cycles: the producers sanitize before
|
|
21
|
+
dumps, the writers go through `atomic_write_json`, and the lenient reader
|
|
22
|
+
degrades to None instead of raising.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import json
|
|
28
|
+
import math
|
|
29
|
+
import os
|
|
30
|
+
import tempfile
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _is_numpy_number(value) -> bool:
|
|
35
|
+
"""True for np.integer/np.floating without importing numpy at module
|
|
36
|
+
load (kept optional on purpose - this module must import everywhere)."""
|
|
37
|
+
type_name = type(value).__module__
|
|
38
|
+
if type_name != "numpy":
|
|
39
|
+
return False
|
|
40
|
+
return hasattr(value, "item")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def sanitize_json_finite(value):
|
|
44
|
+
"""Recursively converts a structure into something `json.dumps(...,
|
|
45
|
+
allow_nan=False)` can always serialize:
|
|
46
|
+
|
|
47
|
+
- non-finite floats (NaN / +/-inf) -> None ("value unavailable",
|
|
48
|
+
the repo-wide degrade convention)
|
|
49
|
+
- numpy integer/float scalars -> plain int/float (via .item())
|
|
50
|
+
- dict keys are passed through unchanged (they must already be strings)
|
|
51
|
+
|
|
52
|
+
Tuples/lists/dicts are rebuilt; everything else returns unchanged."""
|
|
53
|
+
if isinstance(value, float):
|
|
54
|
+
return value if math.isfinite(value) else None
|
|
55
|
+
if isinstance(value, bool) or isinstance(value, int) or value is None or isinstance(value, str):
|
|
56
|
+
return value
|
|
57
|
+
if _is_numpy_number(value):
|
|
58
|
+
as_python = value.item()
|
|
59
|
+
if isinstance(as_python, float) and not math.isfinite(as_python):
|
|
60
|
+
return None
|
|
61
|
+
return as_python
|
|
62
|
+
if isinstance(value, dict):
|
|
63
|
+
return {key: sanitize_json_finite(item) for key, item in value.items()}
|
|
64
|
+
if isinstance(value, (list, tuple)):
|
|
65
|
+
return [sanitize_json_finite(item) for item in value]
|
|
66
|
+
return value
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def dumps_json_safe(payload) -> str:
|
|
70
|
+
"""sanitize + dumps with allow_nan=False - the single serialization
|
|
71
|
+
path shared by the Redis producers and the status-file writers, so a
|
|
72
|
+
non-finite value can never reach Postgres JSONB or the webui again."""
|
|
73
|
+
return json.dumps(sanitize_json_finite(payload), allow_nan=False)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def atomic_write_json(path: Path, payload, *, indent: int | None = None) -> None:
|
|
77
|
+
"""Writes payload to path via a same-directory temp file + os.replace.
|
|
78
|
+
|
|
79
|
+
os.replace is atomic within a filesystem, so a concurrent reader sees
|
|
80
|
+
either the complete old file or the complete new file - never a torn
|
|
81
|
+
one - and a writer crash mid-write leaves the previous generation
|
|
82
|
+
intact. The temp file is cleaned up on failure."""
|
|
83
|
+
path = Path(path)
|
|
84
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
85
|
+
text = json.dumps(sanitize_json_finite(payload), allow_nan=False, indent=indent)
|
|
86
|
+
fd, tmp_name = tempfile.mkstemp(dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp")
|
|
87
|
+
try:
|
|
88
|
+
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
89
|
+
handle.write(text)
|
|
90
|
+
handle.flush()
|
|
91
|
+
os.fsync(handle.fileno())
|
|
92
|
+
os.replace(tmp_name, path)
|
|
93
|
+
except Exception:
|
|
94
|
+
try:
|
|
95
|
+
os.unlink(tmp_name)
|
|
96
|
+
except OSError:
|
|
97
|
+
pass
|
|
98
|
+
raise
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def atomic_write_text(path: Path, text: str) -> None:
|
|
102
|
+
"""atomic_write_json's twin for plain-text content (CSV caches etc.) -
|
|
103
|
+
same same-directory temp file + os.replace contract."""
|
|
104
|
+
path = Path(path)
|
|
105
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
106
|
+
fd, tmp_name = tempfile.mkstemp(dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp")
|
|
107
|
+
try:
|
|
108
|
+
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
109
|
+
handle.write(text)
|
|
110
|
+
handle.flush()
|
|
111
|
+
os.fsync(handle.fileno())
|
|
112
|
+
os.replace(tmp_name, path)
|
|
113
|
+
except Exception:
|
|
114
|
+
try:
|
|
115
|
+
os.unlink(tmp_name)
|
|
116
|
+
except OSError:
|
|
117
|
+
pass
|
|
118
|
+
raise
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def load_json_lenient(path: Path):
|
|
122
|
+
"""Reads and parses path, returning None on ANY failure (missing,
|
|
123
|
+
unreadable, mid-write torn, malformed). Every caller treats None as
|
|
124
|
+
'not available right now' and degrades - the monitoring readers'
|
|
125
|
+
documented contract, now actually true while a writer is mid-replace."""
|
|
126
|
+
try:
|
|
127
|
+
with Path(path).open("r", encoding="utf-8") as handle:
|
|
128
|
+
return json.load(handle)
|
|
129
|
+
except Exception:
|
|
130
|
+
return None
|