shadowbox 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shadowbox/__init__.py +3 -0
- shadowbox/api.py +136 -0
- shadowbox/cards.py +59 -0
- shadowbox/cli.py +222 -0
- shadowbox/compare.py +103 -0
- shadowbox/data/__init__.py +1 -0
- shadowbox/data/cards/cache-poison.yaml +7 -0
- shadowbox/data/cards/db-down.yaml +7 -0
- shadowbox/data/cards/latency-500ms.yaml +8 -0
- shadowbox/data/cards/queue-overflow.yaml +6 -0
- shadowbox/data/cards/slow-dependency.yaml +8 -0
- shadowbox/data/cards/traffic-10x.yaml +6 -0
- shadowbox/data/cards/zone-loss.yaml +8 -0
- shadowbox/data/example/docker-compose.yaml +13 -0
- shadowbox/data/example/model.yaml +33 -0
- shadowbox/data/example/scenarios/db-failure.yaml +12 -0
- shadowbox/dsl.py +94 -0
- shadowbox/engine.py +220 -0
- shadowbox/errors.py +38 -0
- shadowbox/importers/__init__.py +3 -0
- shadowbox/importers/compose.py +109 -0
- shadowbox/metrics.py +49 -0
- shadowbox/model.py +74 -0
- shadowbox/report.py +48 -0
- shadowbox/store.py +85 -0
- shadowbox-0.3.0.dist-info/METADATA +130 -0
- shadowbox-0.3.0.dist-info/RECORD +29 -0
- shadowbox-0.3.0.dist-info/WHEEL +4 -0
- shadowbox-0.3.0.dist-info/entry_points.txt +2 -0
shadowbox/__init__.py
ADDED
shadowbox/api.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""FastAPI surface over the headless core (M4a local server, M4b Worker later).
|
|
2
|
+
|
|
3
|
+
Runs synchronously and returns the completed simulation (201) with a
|
|
4
|
+
`status` field the Worker async path will reuse for queued/running states.
|
|
5
|
+
Local-only: no authentication; bind to localhost unless you know why not.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from dataclasses import asdict as _asdict
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from fastapi import FastAPI, HTTPException
|
|
13
|
+
from fastapi.responses import JSONResponse
|
|
14
|
+
from pydantic import BaseModel, ValidationError
|
|
15
|
+
|
|
16
|
+
from shadowbox import engine as engine_mod
|
|
17
|
+
from shadowbox.errors import ShadowBoxError
|
|
18
|
+
from shadowbox.metrics import summarize
|
|
19
|
+
from shadowbox.model import Scenario, SystemModel
|
|
20
|
+
from shadowbox.report import build_report
|
|
21
|
+
from shadowbox.store import Store
|
|
22
|
+
|
|
23
|
+
MAX_EVENT_LIMIT = 1000
|
|
24
|
+
|
|
25
|
+
app = FastAPI(title="ShadowBox", version="0.1.0")
|
|
26
|
+
store = Store(Path("shadowbox.db"))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ScenarioIn(BaseModel):
|
|
30
|
+
scenario: dict[str, Any]
|
|
31
|
+
seed: int = 42
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _as_422(code: str, message: str) -> JSONResponse:
|
|
35
|
+
return JSONResponse(status_code=422, content={"code": code, "detail": message})
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@app.post("/api/v1/models", status_code=201)
|
|
39
|
+
def create_model(body: dict[str, Any]) -> dict[str, str]:
|
|
40
|
+
try:
|
|
41
|
+
SystemModel.model_validate(body)
|
|
42
|
+
except ValidationError as exc:
|
|
43
|
+
return _as_422("E_SCHEMA", str(exc)) # type: ignore[return-value]
|
|
44
|
+
return {"id": store.save_model(body)}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@app.get("/api/v1/models/{model_id}")
|
|
48
|
+
def get_model(model_id: str) -> dict[str, Any]:
|
|
49
|
+
body = store.get_model(model_id)
|
|
50
|
+
if body is None:
|
|
51
|
+
raise HTTPException(status_code=404, detail="model not found")
|
|
52
|
+
return {"id": model_id, "model": body}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@app.post("/api/v1/simulations", status_code=201)
|
|
56
|
+
def create_simulation(model_id: str, payload: ScenarioIn) -> dict[str, Any]:
|
|
57
|
+
body = store.get_model(model_id)
|
|
58
|
+
if body is None:
|
|
59
|
+
raise HTTPException(status_code=404, detail="model not found")
|
|
60
|
+
try:
|
|
61
|
+
system = SystemModel.model_validate(body)
|
|
62
|
+
scenario = Scenario.model_validate(payload.scenario.get("scenario", payload.scenario))
|
|
63
|
+
result = engine_mod.simulate(system, scenario, payload.seed)
|
|
64
|
+
metrics = summarize(system, result, scenario.duration_s)
|
|
65
|
+
report = build_report(
|
|
66
|
+
system.model_dump(mode="json", by_alias=True),
|
|
67
|
+
scenario.model_dump(mode="json"),
|
|
68
|
+
payload.seed,
|
|
69
|
+
metrics,
|
|
70
|
+
result.events_processed,
|
|
71
|
+
)
|
|
72
|
+
sample = [_asdict(s) for s in result.sample]
|
|
73
|
+
report["sample"] = sample
|
|
74
|
+
except ShadowBoxError as exc:
|
|
75
|
+
return _as_422(exc.code, str(exc)) # type: ignore[return-value]
|
|
76
|
+
sim_id = store.save_simulation(model_id, payload.scenario, payload.seed, report)
|
|
77
|
+
return {"id": sim_id, "status": "completed", "metrics_hash": report["metrics_hash"]}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@app.get("/api/v1/simulations/{sim_id}")
|
|
81
|
+
def get_simulation(sim_id: str) -> dict[str, Any]:
|
|
82
|
+
found = store.get_simulation(sim_id)
|
|
83
|
+
if found is None:
|
|
84
|
+
raise HTTPException(status_code=404, detail="simulation not found")
|
|
85
|
+
report = found["report"]
|
|
86
|
+
assert isinstance(report, dict)
|
|
87
|
+
return {
|
|
88
|
+
"id": sim_id,
|
|
89
|
+
"status": found["status"],
|
|
90
|
+
"metrics_hash": report.get("metrics_hash"),
|
|
91
|
+
"metrics": report.get("metrics"),
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@app.get("/api/v1/simulations/{sim_id}/events")
|
|
96
|
+
def get_events(sim_id: str, limit: int = 100, cursor: int = 0) -> dict[str, Any]:
|
|
97
|
+
found = store.get_simulation(sim_id)
|
|
98
|
+
if found is None:
|
|
99
|
+
raise HTTPException(status_code=404, detail="simulation not found")
|
|
100
|
+
if limit < 1 or limit > MAX_EVENT_LIMIT:
|
|
101
|
+
raise HTTPException(status_code=422, detail=f"limit must be 1..{MAX_EVENT_LIMIT}")
|
|
102
|
+
if cursor < 0:
|
|
103
|
+
raise HTTPException(status_code=422, detail="cursor must be >= 0")
|
|
104
|
+
report = found["report"]
|
|
105
|
+
assert isinstance(report, dict)
|
|
106
|
+
sample = report.get("sample", [])
|
|
107
|
+
assert isinstance(sample, list)
|
|
108
|
+
page = sample[cursor : cursor + limit]
|
|
109
|
+
nxt = cursor + len(page)
|
|
110
|
+
return {
|
|
111
|
+
"events": page,
|
|
112
|
+
"next_cursor": nxt if nxt < len(sample) else None,
|
|
113
|
+
"total_sampled": len(sample),
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@app.get("/api/v1/simulations/{sim_id}/metrics")
|
|
118
|
+
def get_metrics(sim_id: str) -> dict[str, Any]:
|
|
119
|
+
found = store.get_simulation(sim_id)
|
|
120
|
+
if found is None:
|
|
121
|
+
raise HTTPException(status_code=404, detail="simulation not found")
|
|
122
|
+
report = found["report"]
|
|
123
|
+
assert isinstance(report, dict)
|
|
124
|
+
metrics = report.get("metrics")
|
|
125
|
+
assert isinstance(metrics, dict)
|
|
126
|
+
return metrics
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@app.get("/api/v1/simulations/{sim_id}/report")
|
|
130
|
+
def get_report(sim_id: str) -> dict[str, Any]:
|
|
131
|
+
found = store.get_simulation(sim_id)
|
|
132
|
+
if found is None:
|
|
133
|
+
raise HTTPException(status_code=404, detail="simulation not found")
|
|
134
|
+
report = found["report"]
|
|
135
|
+
assert isinstance(report, dict)
|
|
136
|
+
return report
|
shadowbox/cards.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Bundled example and chaos cards, readable from an installed wheel (M6).
|
|
2
|
+
|
|
3
|
+
Canonical files live under `src/shadowbox/data/`; `init` copies them to the
|
|
4
|
+
user's directory so the tool works without a repo clone.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from importlib import resources
|
|
8
|
+
from importlib.resources.abc import Traversable
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from shadowbox.errors import ExistsError
|
|
12
|
+
|
|
13
|
+
_DATA = resources.files("shadowbox.data")
|
|
14
|
+
CARDS_DIR = _DATA / "cards"
|
|
15
|
+
EXAMPLE_DIR = _DATA / "example"
|
|
16
|
+
|
|
17
|
+
EXAMPLE_FILES = ("model.yaml", "docker-compose.yaml", "scenarios/db-failure.yaml")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def list_cards() -> list[str]:
|
|
21
|
+
"""Sorted chaos-card names shipped in the package."""
|
|
22
|
+
return sorted(p.name for p in CARDS_DIR.iterdir() if p.name.endswith(".yaml"))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def read_card(name: str) -> str:
|
|
26
|
+
"""Card YAML text; name with or without `.yaml` suffix."""
|
|
27
|
+
filename = name if name.endswith(".yaml") else f"{name}.yaml"
|
|
28
|
+
return (CARDS_DIR / filename).read_text(encoding="utf-8")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _copy_tree(source: Traversable, target: Path, written: list[str], force: bool) -> None:
|
|
32
|
+
for entry in sorted(source.iterdir(), key=lambda p: p.name):
|
|
33
|
+
dest = target / entry.name
|
|
34
|
+
if entry.is_dir():
|
|
35
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
36
|
+
_copy_tree(entry, dest, written, force)
|
|
37
|
+
elif entry.is_file():
|
|
38
|
+
if dest.exists() and not force:
|
|
39
|
+
raise ExistsError(f"{dest} exists (use --force to overwrite)")
|
|
40
|
+
dest.write_text(entry.read_text(encoding="utf-8"), encoding="utf-8")
|
|
41
|
+
written.append(str(dest))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def write_tree(out: Path, force: bool = False, include_cards: bool = True) -> list[str]:
|
|
45
|
+
"""Copy example (plus cards unless excluded) into `out`; returns written paths."""
|
|
46
|
+
written: list[str] = []
|
|
47
|
+
for filename in EXAMPLE_FILES:
|
|
48
|
+
dest = out / filename
|
|
49
|
+
if dest.exists() and not force:
|
|
50
|
+
raise ExistsError(f"{dest} exists (use --force to overwrite)")
|
|
51
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
52
|
+
text = (_DATA / "example" / filename).read_text(encoding="utf-8")
|
|
53
|
+
dest.write_text(text, encoding="utf-8")
|
|
54
|
+
written.append(str(dest))
|
|
55
|
+
if include_cards:
|
|
56
|
+
cards_out = out / "cards"
|
|
57
|
+
cards_out.mkdir(parents=True, exist_ok=True)
|
|
58
|
+
_copy_tree(CARDS_DIR, cards_out, written, force)
|
|
59
|
+
return written
|
shadowbox/cli.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""Typer CLI: validate (M0) and simulate (M1); compare/report land in M3."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from importlib.metadata import PackageNotFoundError
|
|
5
|
+
from importlib.metadata import version as pkg_version
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
import uvicorn
|
|
10
|
+
import yaml
|
|
11
|
+
from rich.console import Console
|
|
12
|
+
|
|
13
|
+
from shadowbox import engine as engine_mod
|
|
14
|
+
from shadowbox.cards import list_cards, write_tree
|
|
15
|
+
from shadowbox.compare import (
|
|
16
|
+
DEFAULT_THRESHOLDS,
|
|
17
|
+
compare_reports,
|
|
18
|
+
parse_thresholds,
|
|
19
|
+
seed_warning,
|
|
20
|
+
)
|
|
21
|
+
from shadowbox.dsl import load_model, load_scenario
|
|
22
|
+
from shadowbox.errors import SchemaError, ShadowBoxError
|
|
23
|
+
from shadowbox.importers.compose import import_compose
|
|
24
|
+
from shadowbox.metrics import summarize
|
|
25
|
+
from shadowbox.report import build_report
|
|
26
|
+
from shadowbox.store import Store
|
|
27
|
+
|
|
28
|
+
app = typer.Typer(no_args_is_help=True)
|
|
29
|
+
console = Console()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@app.command()
|
|
33
|
+
def validate(
|
|
34
|
+
model: Path = typer.Argument(..., help="Path to model.yaml"),
|
|
35
|
+
scenario: Path = typer.Option(None, "--scenario", help="Path to scenario YAML"),
|
|
36
|
+
) -> None:
|
|
37
|
+
"""Validate a model and optionally a scenario (exit 0 ok, 3 invalid)."""
|
|
38
|
+
try:
|
|
39
|
+
system = load_model(model)
|
|
40
|
+
if scenario is not None:
|
|
41
|
+
load_scenario(scenario, system)
|
|
42
|
+
except ShadowBoxError as exc:
|
|
43
|
+
console.print(f"[red]{exc.code}[/red]: {exc}")
|
|
44
|
+
raise typer.Exit(code=3) from exc
|
|
45
|
+
faults = f", {len(system.connections)} connections" if system.connections else ""
|
|
46
|
+
console.print(f"[green]valid[/green]: {len(system.components)} components{faults}")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@app.command()
|
|
50
|
+
def version() -> None:
|
|
51
|
+
"""Print the package version (also keeps `validate` as a named subcommand)."""
|
|
52
|
+
try:
|
|
53
|
+
number = pkg_version("shadowbox")
|
|
54
|
+
except PackageNotFoundError:
|
|
55
|
+
number = "0.0.0+local"
|
|
56
|
+
console.print(f"shadowbox {number}")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@app.command(name="import")
|
|
60
|
+
def import_model(
|
|
61
|
+
from_path: Path = typer.Option(..., "--from", help="Path to docker-compose.yaml"),
|
|
62
|
+
out: Path = typer.Option(Path("model.yaml"), "--out", help="Imported model output path"),
|
|
63
|
+
) -> None:
|
|
64
|
+
"""Import a compose file into a validated model (all fields estimated)."""
|
|
65
|
+
try:
|
|
66
|
+
model, warnings = import_compose(from_path)
|
|
67
|
+
except ShadowBoxError as exc:
|
|
68
|
+
console.print(f"[red]{exc.code}[/red]: {exc}")
|
|
69
|
+
raise typer.Exit(code=3) from exc
|
|
70
|
+
out.write_text(
|
|
71
|
+
yaml.safe_dump(model.model_dump(mode="json", by_alias=True), sort_keys=False),
|
|
72
|
+
encoding="utf-8",
|
|
73
|
+
)
|
|
74
|
+
for warning in warnings:
|
|
75
|
+
console.print(f"[yellow]warn[/yellow]: {warning}")
|
|
76
|
+
console.print(f"[green]imported[/green]: {len(model.components)} components -> {out}")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@app.command()
|
|
80
|
+
def simulate(
|
|
81
|
+
model: Path = typer.Argument(..., help="Path to model.yaml"),
|
|
82
|
+
scenario: Path = typer.Option(..., "--scenario", help="Path to scenario YAML"),
|
|
83
|
+
seed: int = typer.Option(42, "--seed", help="Deterministic RNG seed"),
|
|
84
|
+
out: Path = typer.Option(Path("report.json"), "--out", help="Report output path"),
|
|
85
|
+
format: str = typer.Option("json", "--format", help="json or text"),
|
|
86
|
+
) -> None:
|
|
87
|
+
"""Run a deterministic simulation and write the report envelope."""
|
|
88
|
+
try:
|
|
89
|
+
system = load_model(model)
|
|
90
|
+
ordered = load_scenario(scenario, system)
|
|
91
|
+
result = engine_mod.simulate(system, ordered, seed)
|
|
92
|
+
metrics = summarize(system, result, ordered.duration_s)
|
|
93
|
+
report = build_report(
|
|
94
|
+
system.model_dump(mode="json", by_alias=True),
|
|
95
|
+
ordered.model_dump(mode="json"),
|
|
96
|
+
seed,
|
|
97
|
+
metrics,
|
|
98
|
+
result.events_processed,
|
|
99
|
+
)
|
|
100
|
+
except ShadowBoxError as exc:
|
|
101
|
+
console.print(f"[red]{exc.code}[/red]: {exc}")
|
|
102
|
+
raise typer.Exit(code=3) from exc
|
|
103
|
+
if format == "text":
|
|
104
|
+
out.write_text(_as_text(report), encoding="utf-8")
|
|
105
|
+
else:
|
|
106
|
+
out.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
|
|
107
|
+
latency = metrics["latency_ms"]
|
|
108
|
+
assert isinstance(latency, dict)
|
|
109
|
+
error_rate = metrics["error_rate"]
|
|
110
|
+
assert isinstance(error_rate, float)
|
|
111
|
+
p99 = latency["p99"]
|
|
112
|
+
assert isinstance(p99, (int, float))
|
|
113
|
+
metrics_hash = report["metrics_hash"]
|
|
114
|
+
assert isinstance(metrics_hash, str)
|
|
115
|
+
console.print(
|
|
116
|
+
f"[green]done[/green]: {result.succeeded}/{result.total} ok, "
|
|
117
|
+
f"error_rate={error_rate:.3f}, p99={p99:.0f}ms, "
|
|
118
|
+
f"hash={metrics_hash[:12]} -> {out}"
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _as_text(report: dict[str, object]) -> str:
|
|
123
|
+
metrics = report["metrics"]
|
|
124
|
+
assert isinstance(metrics, dict)
|
|
125
|
+
latency = metrics["latency_ms"]
|
|
126
|
+
assert isinstance(latency, dict)
|
|
127
|
+
lines = [
|
|
128
|
+
f"engine: {report['engine']}",
|
|
129
|
+
f"seed: {report['seed']}",
|
|
130
|
+
f"metrics_hash: {report['metrics_hash']}",
|
|
131
|
+
f"throughput_rps: {metrics['throughput_rps']}",
|
|
132
|
+
f"error_rate: {metrics['error_rate']}",
|
|
133
|
+
f"latency_ms p50/p95/p99: {latency['p50']}/{latency['p95']}/{latency['p99']}",
|
|
134
|
+
f"timeouts: {metrics['timeouts']}, cascade_depth: {metrics['cascade_depth']}",
|
|
135
|
+
f"confidence: {report['confidence']} ({report['calibration_source']})",
|
|
136
|
+
]
|
|
137
|
+
return "\n".join(lines) + "\n"
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _read_report(path: Path) -> dict[str, object]:
|
|
141
|
+
try:
|
|
142
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
143
|
+
except (OSError, ValueError) as exc:
|
|
144
|
+
raise SchemaError(f"cannot read report {path}: {exc}") from exc
|
|
145
|
+
if not isinstance(data, dict) or "metrics" not in data:
|
|
146
|
+
raise SchemaError(f"{path}: not a shadowbox report")
|
|
147
|
+
return data
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@app.command()
|
|
151
|
+
def report(
|
|
152
|
+
report_path: Path = typer.Argument(..., help="Path to report.json"),
|
|
153
|
+
format: str = typer.Option("json", "--format", help="json or text"),
|
|
154
|
+
) -> None:
|
|
155
|
+
"""Render an existing report file (exit 0 ok, 3 invalid)."""
|
|
156
|
+
try:
|
|
157
|
+
found = _read_report(report_path)
|
|
158
|
+
except ShadowBoxError as exc:
|
|
159
|
+
console.print(f"[red]{exc.code}[/red]: {exc}")
|
|
160
|
+
raise typer.Exit(code=3) from exc
|
|
161
|
+
if format == "text":
|
|
162
|
+
console.print(_as_text(found), end="")
|
|
163
|
+
else:
|
|
164
|
+
console.print_json(json.dumps(found, indent=2, sort_keys=True))
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
@app.command()
|
|
168
|
+
def compare(
|
|
169
|
+
a: Path = typer.Option(..., "--a", help="Baseline report.json"),
|
|
170
|
+
b: Path = typer.Option(..., "--b", help="Candidate report.json"),
|
|
171
|
+
threshold: str = typer.Option(DEFAULT_THRESHOLDS, "--threshold", help="p99:+10%,..."),
|
|
172
|
+
) -> None:
|
|
173
|
+
"""Compare two reports (exit 0 pass/improvement, 2 regression, 3 invalid)."""
|
|
174
|
+
try:
|
|
175
|
+
base = _read_report(a)
|
|
176
|
+
candidate = _read_report(b)
|
|
177
|
+
result = compare_reports(base, candidate, parse_thresholds(threshold))
|
|
178
|
+
except ShadowBoxError as exc:
|
|
179
|
+
console.print(f"[red]{exc.code}[/red]: {exc}")
|
|
180
|
+
raise typer.Exit(code=3) from exc
|
|
181
|
+
warning = seed_warning(base, candidate)
|
|
182
|
+
if warning is not None:
|
|
183
|
+
console.print(f"[yellow]warn[/yellow]: {warning}")
|
|
184
|
+
for metric in sorted(result.deltas):
|
|
185
|
+
console.print(f"{metric}: {result.deltas[metric]:+.2f}")
|
|
186
|
+
if result.breaches:
|
|
187
|
+
console.print(f"[red]regression[/red]: breached {', '.join(sorted(result.breaches))}")
|
|
188
|
+
raise typer.Exit(code=2)
|
|
189
|
+
console.print(f"[green]{result.verdict}[/green]")
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
@app.command()
|
|
193
|
+
def serve(
|
|
194
|
+
host: str = typer.Option("127.0.0.1", "--host", help="Bind host"),
|
|
195
|
+
port: int = typer.Option(8000, "--port", help="Bind port"),
|
|
196
|
+
db: Path = typer.Option(Path("shadowbox.db"), "--db", help="SQLite file"),
|
|
197
|
+
) -> None:
|
|
198
|
+
"""Run the local API server (blocks; Ctrl-C to stop)."""
|
|
199
|
+
from shadowbox import api as api_mod
|
|
200
|
+
|
|
201
|
+
api_mod.store = Store(db)
|
|
202
|
+
uvicorn.run(api_mod.app, host=host, port=port)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
@app.command(name="init")
|
|
206
|
+
def init_project(
|
|
207
|
+
out: Path = typer.Option(Path("."), "--out", help="Target directory"),
|
|
208
|
+
force: bool = typer.Option(False, "--force", help="Overwrite existing files"),
|
|
209
|
+
cards: bool = typer.Option(True, "--cards/--no-cards", help="Include chaos cards"),
|
|
210
|
+
) -> None:
|
|
211
|
+
"""Scaffold example model plus chaos cards (works without a repo clone)."""
|
|
212
|
+
try:
|
|
213
|
+
written = write_tree(out, force, include_cards=cards)
|
|
214
|
+
except ShadowBoxError as exc:
|
|
215
|
+
console.print(f"[red]{exc.code}[/red]: {exc}")
|
|
216
|
+
raise typer.Exit(code=3) from exc
|
|
217
|
+
console.print(f"[green]initialized[/green]: {len(written)} files -> {out}")
|
|
218
|
+
console.print(f"cards available: {', '.join(n.replace('.yaml', '') for n in list_cards())}")
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
if __name__ == "__main__":
|
|
222
|
+
app()
|
shadowbox/compare.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Scenario comparison: numeric deltas plus a threshold verdict (M3)."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
from shadowbox.errors import SchemaError
|
|
7
|
+
|
|
8
|
+
_THRESHOLD_RE = re.compile(r"^(p50|p90|p95|p99|error_rate|throughput):([+-])(\d+(?:\.\d+)?)(%|pp)$")
|
|
9
|
+
DEFAULT_THRESHOLDS = "p99:+10%,error_rate:+1pp,throughput:-10%"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class Threshold:
|
|
14
|
+
metric: str
|
|
15
|
+
limit: float # positive magnitude; unit implied by metric
|
|
16
|
+
percent: bool
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class Comparison:
|
|
21
|
+
deltas: dict[str, float]
|
|
22
|
+
breaches: list[str]
|
|
23
|
+
verdict: str # pass | regression | improvement
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def parse_thresholds(raw: str) -> list[Threshold]:
|
|
27
|
+
"""Parse `p99:+10%,error_rate:+1pp,throughput:-10%` (pp only for error_rate)."""
|
|
28
|
+
parsed: list[Threshold] = []
|
|
29
|
+
for chunk in raw.split(","):
|
|
30
|
+
match = _THRESHOLD_RE.match(chunk.strip())
|
|
31
|
+
if match is None:
|
|
32
|
+
raise SchemaError(f"invalid threshold {chunk!r}; expected metric:±value%|pp")
|
|
33
|
+
metric, _, value, unit = match.groups()
|
|
34
|
+
if metric == "error_rate" and unit != "pp":
|
|
35
|
+
raise SchemaError("error_rate threshold must use pp units")
|
|
36
|
+
if metric != "error_rate" and unit != "%":
|
|
37
|
+
raise SchemaError(f"{metric} threshold must use % units")
|
|
38
|
+
parsed.append(Threshold(metric, float(value), unit == "%"))
|
|
39
|
+
if not parsed:
|
|
40
|
+
raise SchemaError("at least one threshold required")
|
|
41
|
+
return parsed
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _latency(report: dict[str, object], key: str) -> float:
|
|
45
|
+
metrics = report["metrics"]
|
|
46
|
+
assert isinstance(metrics, dict)
|
|
47
|
+
latency = metrics["latency_ms"]
|
|
48
|
+
assert isinstance(latency, dict)
|
|
49
|
+
value = latency[key]
|
|
50
|
+
assert isinstance(value, (int, float))
|
|
51
|
+
return float(value)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _metric(report: dict[str, object], metric: str) -> float:
|
|
55
|
+
metrics = report["metrics"]
|
|
56
|
+
assert isinstance(metrics, dict)
|
|
57
|
+
if metric == "error_rate":
|
|
58
|
+
value = metrics["error_rate"]
|
|
59
|
+
elif metric == "throughput":
|
|
60
|
+
value = metrics["throughput_rps"]
|
|
61
|
+
else:
|
|
62
|
+
return _latency(report, metric)
|
|
63
|
+
assert isinstance(value, (int, float))
|
|
64
|
+
return float(value)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _pct_change(base: float, candidate: float) -> float:
|
|
68
|
+
if base == 0:
|
|
69
|
+
return 0.0 if candidate == 0 else float("inf")
|
|
70
|
+
return (candidate - base) / abs(base) * 100.0
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def compare_reports(
|
|
74
|
+
base: dict[str, object], candidate: dict[str, object], thresholds: list[Threshold]
|
|
75
|
+
) -> Comparison:
|
|
76
|
+
"""Diff candidate vs base; latency/error up is bad, throughput down is bad."""
|
|
77
|
+
deltas: dict[str, float] = {}
|
|
78
|
+
for threshold in thresholds:
|
|
79
|
+
a = _metric(base, threshold.metric)
|
|
80
|
+
b = _metric(candidate, threshold.metric)
|
|
81
|
+
deltas[threshold.metric] = (b - a) * 100.0 if not threshold.percent else _pct_change(a, b)
|
|
82
|
+
breaches = [
|
|
83
|
+
t.metric
|
|
84
|
+
for t in thresholds
|
|
85
|
+
if (deltas[t.metric] > t.limit if t.metric != "throughput" else deltas[t.metric] < -t.limit)
|
|
86
|
+
]
|
|
87
|
+
if breaches:
|
|
88
|
+
verdict = "regression"
|
|
89
|
+
elif any(
|
|
90
|
+
(deltas[t.metric] < 0 if t.metric != "throughput" else deltas[t.metric] > 0)
|
|
91
|
+
for t in thresholds
|
|
92
|
+
):
|
|
93
|
+
verdict = "improvement"
|
|
94
|
+
else:
|
|
95
|
+
verdict = "pass"
|
|
96
|
+
return Comparison(deltas, breaches, verdict)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def seed_warning(base: dict[str, object], candidate: dict[str, object]) -> str | None:
|
|
100
|
+
"""Warn when reports are not comparable (different seeds); confidence drops."""
|
|
101
|
+
if base.get("seed") != candidate.get("seed"):
|
|
102
|
+
return "seeds differ; treat verdict with low confidence (E_SEED_MISMATCH)"
|
|
103
|
+
return None
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Bundled data: example model plus chaos cards (ships inside the wheel)."""
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Card: cache poisoned (unavailable) 10s. Every request visits cache: expect ~1000 failed (10s x 100rps).
|
|
2
|
+
scenario:
|
|
3
|
+
name: cache-poison
|
|
4
|
+
duration_s: 60
|
|
5
|
+
workload: { rate_rps: 100, arrival: deterministic }
|
|
6
|
+
faults:
|
|
7
|
+
- { target: cache, type: unavailable, start_s: 20, duration_s: 10 }
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Card: database +500ms for the whole run (505ms > 500ms db timeout).
|
|
2
|
+
# Expect near-total timeouts; boundary requests past the window edge may still succeed.
|
|
3
|
+
scenario:
|
|
4
|
+
name: latency-500ms
|
|
5
|
+
duration_s: 60
|
|
6
|
+
workload: { rate_rps: 100, arrival: deterministic }
|
|
7
|
+
faults:
|
|
8
|
+
- { target: database, type: latency, extra_ms: 500, start_s: 0, duration_s: 60 }
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Card: slow cache dependency (+100ms, under the 200ms cache timeout).
|
|
2
|
+
# Expect +~100ms latency delta vs baseline, no errors.
|
|
3
|
+
scenario:
|
|
4
|
+
name: slow-dependency
|
|
5
|
+
duration_s: 60
|
|
6
|
+
workload: { rate_rps: 100, arrival: deterministic }
|
|
7
|
+
faults:
|
|
8
|
+
- { target: cache, type: latency, extra_ms: 100, start_s: 0, duration_s: 60 }
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Card: availability-zone loss takes cache+database down 20s. Expect near-total failure in window.
|
|
2
|
+
scenario:
|
|
3
|
+
name: zone-loss
|
|
4
|
+
duration_s: 60
|
|
5
|
+
workload: { rate_rps: 100, arrival: deterministic }
|
|
6
|
+
faults:
|
|
7
|
+
- { target: cache, type: unavailable, start_s: 20, duration_s: 20 }
|
|
8
|
+
- { target: database, type: unavailable, start_s: 20, duration_s: 20 }
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Canonical M0 fixture: ecommerce checkout (api + cache + db).
|
|
2
|
+
components:
|
|
3
|
+
- id: api
|
|
4
|
+
type: service
|
|
5
|
+
capacity: 50
|
|
6
|
+
latency_ms: { base: 20, jitter_ms: 5 }
|
|
7
|
+
timeout_ms: 1000
|
|
8
|
+
queue_size: 200
|
|
9
|
+
queue_policy: drop
|
|
10
|
+
|
|
11
|
+
- id: cache
|
|
12
|
+
type: cache
|
|
13
|
+
capacity: 100
|
|
14
|
+
latency_ms: { base: 2, jitter_ms: 1 }
|
|
15
|
+
timeout_ms: 200
|
|
16
|
+
queue_size: 500
|
|
17
|
+
queue_policy: drop
|
|
18
|
+
|
|
19
|
+
- id: database
|
|
20
|
+
type: database
|
|
21
|
+
capacity: 20
|
|
22
|
+
latency_ms: { base: 5, jitter_ms: 1 }
|
|
23
|
+
timeout_ms: 500
|
|
24
|
+
queue_size: 100
|
|
25
|
+
queue_policy: fifo
|
|
26
|
+
|
|
27
|
+
connections:
|
|
28
|
+
- from: api
|
|
29
|
+
to: cache
|
|
30
|
+
- from: api
|
|
31
|
+
to: database
|
|
32
|
+
- from: cache
|
|
33
|
+
to: database
|