simulsi 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- simulsi/__init__.py +70 -0
- simulsi/__main__.py +5 -0
- simulsi/_version.py +1 -0
- simulsi/analysis/__init__.py +24 -0
- simulsi/analysis/comparison.py +154 -0
- simulsi/analysis/report.py +46 -0
- simulsi/analysis/sensitivity.py +387 -0
- simulsi/benchmarks.py +128 -0
- simulsi/cli/__init__.py +3 -0
- simulsi/cli/main.py +486 -0
- simulsi/cli/templates.py +96 -0
- simulsi/config/__init__.py +19 -0
- simulsi/config/schema.py +263 -0
- simulsi/core/__init__.py +13 -0
- simulsi/core/checkpoint.py +159 -0
- simulsi/core/clock.py +87 -0
- simulsi/core/model.py +441 -0
- simulsi/core/simulation.py +670 -0
- simulsi/core/trace.py +79 -0
- simulsi/cost/__init__.py +3 -0
- simulsi/cost/model.py +193 -0
- simulsi/entities/__init__.py +3 -0
- simulsi/entities/entity.py +120 -0
- simulsi/errors.py +50 -0
- simulsi/events/__init__.py +3 -0
- simulsi/events/event.py +221 -0
- simulsi/experiments/__init__.py +17 -0
- simulsi/experiments/experiment.py +548 -0
- simulsi/experiments/montecarlo.py +261 -0
- simulsi/experiments/provenance.py +61 -0
- simulsi/introspection/__init__.py +21 -0
- simulsi/introspection/graph.py +210 -0
- simulsi/metrics/__init__.py +3 -0
- simulsi/metrics/collectors.py +348 -0
- simulsi/models/__init__.py +3 -0
- simulsi/models/queueing.py +73 -0
- simulsi/optimization/__init__.py +3 -0
- simulsi/optimization/objective.py +131 -0
- simulsi/processes/__init__.py +31 -0
- simulsi/processes/disruption.py +165 -0
- simulsi/processes/process.py +346 -0
- simulsi/py.typed +0 -0
- simulsi/queues/__init__.py +4 -0
- simulsi/queues/discipline.py +99 -0
- simulsi/queues/queue.py +256 -0
- simulsi/randomness/__init__.py +47 -0
- simulsi/randomness/distributions.py +449 -0
- simulsi/randomness/stream.py +117 -0
- simulsi/resources/__init__.py +3 -0
- simulsi/resources/resource.py +519 -0
- simulsi/scenarios/__init__.py +10 -0
- simulsi/scenarios/scenario.py +132 -0
- simulsi/serialization/__init__.py +23 -0
- simulsi/serialization/io.py +128 -0
- simulsi/statistics/__init__.py +37 -0
- simulsi/statistics/core.py +418 -0
- simulsi/validation/__init__.py +3 -0
- simulsi/validation/validate.py +168 -0
- simulsi/visualization/__init__.py +27 -0
- simulsi/visualization/plots.py +423 -0
- simulsi/web/__init__.py +1 -0
- simulsi/web/server.py +338 -0
- simulsi/web/static/assets/index-CNMcVqxw.css +2 -0
- simulsi/web/static/assets/index-DDtrYZla.js +9 -0
- simulsi/web/static/index.html +13 -0
- simulsi-0.2.0.dist-info/METADATA +206 -0
- simulsi-0.2.0.dist-info/RECORD +71 -0
- simulsi-0.2.0.dist-info/WHEEL +4 -0
- simulsi-0.2.0.dist-info/entry_points.txt +2 -0
- simulsi-0.2.0.dist-info/licenses/LICENSE +21 -0
- simulsi-0.2.0.dist-info/licenses/NOTICE +11 -0
simulsi/__init__.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""SimulSI: Simulation Intelligence. Model the system. Simulate the future."""
|
|
2
|
+
|
|
3
|
+
from simulsi._version import __version__
|
|
4
|
+
from simulsi.analysis import Comparison, compare
|
|
5
|
+
from simulsi.core import Clock, EventLog, LogRecord, Simulation, SimulationResult
|
|
6
|
+
from simulsi.core.model import Model, Parameter, Params, model
|
|
7
|
+
from simulsi.entities import Entity
|
|
8
|
+
from simulsi.errors import (
|
|
9
|
+
CapacityError,
|
|
10
|
+
ConfigError,
|
|
11
|
+
EventStateError,
|
|
12
|
+
Interrupt,
|
|
13
|
+
ModelValidationError,
|
|
14
|
+
ResourceUsageError,
|
|
15
|
+
SchedulingError,
|
|
16
|
+
SimulsiError,
|
|
17
|
+
)
|
|
18
|
+
from simulsi.events import Event, EventStatus, Priority
|
|
19
|
+
from simulsi.experiments import Experiment, ExperimentResult, MonteCarloResult, monte_carlo
|
|
20
|
+
from simulsi.metrics import Metrics
|
|
21
|
+
from simulsi.processes import AllOf, AnyOf, Process, Signal, Timeout, Waitable
|
|
22
|
+
from simulsi.queues import Queue
|
|
23
|
+
from simulsi.randomness import RandomStream
|
|
24
|
+
from simulsi.resources import Preempted, Request, Resource
|
|
25
|
+
from simulsi.scenarios import Scenario, grid
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"AllOf",
|
|
29
|
+
"AnyOf",
|
|
30
|
+
"CapacityError",
|
|
31
|
+
"Clock",
|
|
32
|
+
"Comparison",
|
|
33
|
+
"ConfigError",
|
|
34
|
+
"Entity",
|
|
35
|
+
"Event",
|
|
36
|
+
"EventLog",
|
|
37
|
+
"EventStateError",
|
|
38
|
+
"EventStatus",
|
|
39
|
+
"Experiment",
|
|
40
|
+
"ExperimentResult",
|
|
41
|
+
"Interrupt",
|
|
42
|
+
"LogRecord",
|
|
43
|
+
"Metrics",
|
|
44
|
+
"Model",
|
|
45
|
+
"ModelValidationError",
|
|
46
|
+
"MonteCarloResult",
|
|
47
|
+
"Parameter",
|
|
48
|
+
"Params",
|
|
49
|
+
"Preempted",
|
|
50
|
+
"Priority",
|
|
51
|
+
"Process",
|
|
52
|
+
"Queue",
|
|
53
|
+
"RandomStream",
|
|
54
|
+
"Request",
|
|
55
|
+
"Resource",
|
|
56
|
+
"ResourceUsageError",
|
|
57
|
+
"Scenario",
|
|
58
|
+
"SchedulingError",
|
|
59
|
+
"Signal",
|
|
60
|
+
"Simulation",
|
|
61
|
+
"SimulationResult",
|
|
62
|
+
"SimulsiError",
|
|
63
|
+
"Timeout",
|
|
64
|
+
"Waitable",
|
|
65
|
+
"__version__",
|
|
66
|
+
"compare",
|
|
67
|
+
"grid",
|
|
68
|
+
"model",
|
|
69
|
+
"monte_carlo",
|
|
70
|
+
]
|
simulsi/__main__.py
ADDED
simulsi/_version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.0"
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from simulsi.analysis.comparison import Comparison, ComparisonRow, compare, compare_samples
|
|
2
|
+
from simulsi.analysis.report import format_table
|
|
3
|
+
from simulsi.analysis.sensitivity import (
|
|
4
|
+
SensitivityResult,
|
|
5
|
+
SensitivityRow,
|
|
6
|
+
correlation_sensitivity,
|
|
7
|
+
finite_difference,
|
|
8
|
+
one_at_a_time,
|
|
9
|
+
sobol_indices,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"Comparison",
|
|
14
|
+
"ComparisonRow",
|
|
15
|
+
"SensitivityResult",
|
|
16
|
+
"SensitivityRow",
|
|
17
|
+
"compare",
|
|
18
|
+
"compare_samples",
|
|
19
|
+
"correlation_sensitivity",
|
|
20
|
+
"finite_difference",
|
|
21
|
+
"format_table",
|
|
22
|
+
"one_at_a_time",
|
|
23
|
+
"sobol_indices",
|
|
24
|
+
]
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""Scenario comparison against a baseline."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
6
|
+
from dataclasses import asdict, dataclass
|
|
7
|
+
from typing import TYPE_CHECKING, Any
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
from simulsi.analysis.report import format_table
|
|
12
|
+
from simulsi.statistics.core import Difference, paired_difference, welch_difference
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from simulsi.experiments.experiment import ExperimentResult
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class ComparisonRow:
|
|
20
|
+
metric: str
|
|
21
|
+
baseline: str
|
|
22
|
+
scenario: str
|
|
23
|
+
baseline_mean: float
|
|
24
|
+
scenario_mean: float
|
|
25
|
+
absolute_difference: float
|
|
26
|
+
percentage_difference: float
|
|
27
|
+
ci_low: float
|
|
28
|
+
ci_high: float
|
|
29
|
+
p_value: float
|
|
30
|
+
method: str
|
|
31
|
+
n: int
|
|
32
|
+
significant: bool
|
|
33
|
+
|
|
34
|
+
def to_dict(self) -> dict[str, Any]:
|
|
35
|
+
return asdict(self)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Comparison:
|
|
39
|
+
"""Differences ``scenario - baseline`` for each metric, with confidence intervals.
|
|
40
|
+
|
|
41
|
+
The intervals quantify *simulation sampling error* only. A significant
|
|
42
|
+
difference means the model, as specified, responds to the change; it says
|
|
43
|
+
nothing about whether the real system would (that depends on model
|
|
44
|
+
validity). No multiple-comparison correction is applied.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
def __init__(self, rows: Sequence[ComparisonRow], confidence: float) -> None:
|
|
48
|
+
self.rows = list(rows)
|
|
49
|
+
self.confidence = confidence
|
|
50
|
+
|
|
51
|
+
def to_dicts(self) -> list[dict[str, Any]]:
|
|
52
|
+
return [r.to_dict() for r in self.rows]
|
|
53
|
+
|
|
54
|
+
def get(self, metric: str, scenario: str) -> ComparisonRow:
|
|
55
|
+
for r in self.rows:
|
|
56
|
+
if r.metric == metric and r.scenario == scenario:
|
|
57
|
+
return r
|
|
58
|
+
raise KeyError((metric, scenario))
|
|
59
|
+
|
|
60
|
+
def format(self) -> str:
|
|
61
|
+
return format_table(
|
|
62
|
+
[
|
|
63
|
+
{
|
|
64
|
+
"metric": r.metric,
|
|
65
|
+
"scenario": r.scenario,
|
|
66
|
+
"baseline": r.baseline_mean,
|
|
67
|
+
"value": r.scenario_mean,
|
|
68
|
+
"diff": r.absolute_difference,
|
|
69
|
+
"diff%": r.percentage_difference,
|
|
70
|
+
"ci_low": r.ci_low,
|
|
71
|
+
"ci_high": r.ci_high,
|
|
72
|
+
"sig": "*" if r.significant else "",
|
|
73
|
+
}
|
|
74
|
+
for r in self.rows
|
|
75
|
+
]
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
def __repr__(self) -> str:
|
|
79
|
+
return f"Comparison({len(self.rows)} rows, confidence={self.confidence})"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _row(metric: str, base: str, sc: str, d: Difference) -> ComparisonRow:
|
|
83
|
+
pct = d.relative_difference * 100
|
|
84
|
+
return ComparisonRow(
|
|
85
|
+
metric,
|
|
86
|
+
base,
|
|
87
|
+
sc,
|
|
88
|
+
d.mean_a,
|
|
89
|
+
d.mean_b,
|
|
90
|
+
d.difference,
|
|
91
|
+
pct,
|
|
92
|
+
d.ci_low,
|
|
93
|
+
d.ci_high,
|
|
94
|
+
d.p_value,
|
|
95
|
+
d.method,
|
|
96
|
+
min(d.n_a, d.n_b),
|
|
97
|
+
d.significant,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def compare(
|
|
102
|
+
result: ExperimentResult,
|
|
103
|
+
baseline: str = "baseline",
|
|
104
|
+
scenarios: Iterable[str] | None = None,
|
|
105
|
+
metrics: Iterable[str] | None = None,
|
|
106
|
+
*,
|
|
107
|
+
confidence: float = 0.95,
|
|
108
|
+
) -> Comparison:
|
|
109
|
+
"""Compare scenarios of one experiment with its ``baseline`` scenario.
|
|
110
|
+
|
|
111
|
+
When the experiment used common random numbers and both scenarios have the
|
|
112
|
+
same replication seeds, a paired-t interval is used (replication *i* of
|
|
113
|
+
each scenario saw the same random streams); otherwise Welch's t.
|
|
114
|
+
"""
|
|
115
|
+
if baseline not in result.scenarios:
|
|
116
|
+
raise KeyError(f"baseline {baseline!r} not in experiment scenarios {result.scenarios}")
|
|
117
|
+
others = [
|
|
118
|
+
s for s in (scenarios if scenarios is not None else result.scenarios) if s != baseline
|
|
119
|
+
]
|
|
120
|
+
names = list(metrics) if metrics is not None else result.metric_names
|
|
121
|
+
rows = []
|
|
122
|
+
for sc in others:
|
|
123
|
+
paired = result.metadata.common_random_numbers and result.seeds(sc) == result.seeds(
|
|
124
|
+
baseline
|
|
125
|
+
)
|
|
126
|
+
for m in names:
|
|
127
|
+
a, b = result.values(m, baseline), result.values(m, sc)
|
|
128
|
+
d = (
|
|
129
|
+
paired_difference(a, b, confidence)
|
|
130
|
+
if paired and len(a) == len(b)
|
|
131
|
+
else welch_difference(a, b, confidence)
|
|
132
|
+
)
|
|
133
|
+
rows.append(_row(m, baseline, sc, d))
|
|
134
|
+
return Comparison(rows, confidence)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def compare_samples(
|
|
138
|
+
baseline: Mapping[str, Sequence[float]],
|
|
139
|
+
scenario: Mapping[str, Sequence[float]],
|
|
140
|
+
*,
|
|
141
|
+
paired: bool = False,
|
|
142
|
+
confidence: float = 0.95,
|
|
143
|
+
baseline_name: str = "baseline",
|
|
144
|
+
scenario_name: str = "scenario",
|
|
145
|
+
) -> Comparison:
|
|
146
|
+
"""Compare two ``{metric: values}`` samples (e.g. collected outside an Experiment)."""
|
|
147
|
+
rows = []
|
|
148
|
+
for m in baseline:
|
|
149
|
+
if m not in scenario:
|
|
150
|
+
continue
|
|
151
|
+
a, b = np.asarray(baseline[m], dtype=float), np.asarray(scenario[m], dtype=float)
|
|
152
|
+
d = paired_difference(a, b, confidence) if paired else welch_difference(a, b, confidence)
|
|
153
|
+
rows.append(_row(m, baseline_name, scenario_name, d))
|
|
154
|
+
return Comparison(rows, confidence)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Plain-text tables for terminal output."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from collections.abc import Mapping, Sequence
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def fmt_value(v: Any) -> str:
|
|
11
|
+
if isinstance(v, bool):
|
|
12
|
+
return str(v)
|
|
13
|
+
if isinstance(v, int):
|
|
14
|
+
return str(v)
|
|
15
|
+
if isinstance(v, float):
|
|
16
|
+
if math.isnan(v):
|
|
17
|
+
return "-"
|
|
18
|
+
if math.isinf(v):
|
|
19
|
+
return "inf" if v > 0 else "-inf"
|
|
20
|
+
a = abs(v)
|
|
21
|
+
if a != 0 and (a >= 1e6 or a < 1e-3):
|
|
22
|
+
return f"{v:.3e}"
|
|
23
|
+
return f"{v:.4g}" if a < 1000 else f"{v:,.1f}"
|
|
24
|
+
return str(v)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def format_table(rows: Sequence[Mapping[str, Any]], columns: Sequence[str] | None = None) -> str:
|
|
28
|
+
if not rows:
|
|
29
|
+
return "(no rows)"
|
|
30
|
+
cols = list(columns) if columns is not None else list(rows[0].keys())
|
|
31
|
+
cells = [[fmt_value(r.get(c, "")) for c in cols] for r in rows]
|
|
32
|
+
widths = [max(len(c), *(len(row[i]) for row in cells)) for i, c in enumerate(cols)]
|
|
33
|
+
numeric = [
|
|
34
|
+
all(isinstance(r.get(c), int | float) and not isinstance(r.get(c), bool) for r in rows)
|
|
35
|
+
for c in cols
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
def line(values: Sequence[str]) -> str:
|
|
39
|
+
return " ".join(
|
|
40
|
+
v.rjust(w) if num else v.ljust(w)
|
|
41
|
+
for v, w, num in zip(values, widths, numeric, strict=True)
|
|
42
|
+
).rstrip()
|
|
43
|
+
|
|
44
|
+
out = [line(cols), line(["-" * w for w in widths])]
|
|
45
|
+
out += [line(row) for row in cells]
|
|
46
|
+
return "\n".join(out)
|
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
"""Sensitivity analysis: which inputs move which outputs, and by how much.
|
|
2
|
+
|
|
3
|
+
Methods implemented (all simple and well understood):
|
|
4
|
+
|
|
5
|
+
* :func:`one_at_a_time` - change one parameter at a time around a base
|
|
6
|
+
point, with common random numbers, and report the change in each output
|
|
7
|
+
plus a paired confidence interval and an elasticity.
|
|
8
|
+
* :func:`finite_difference` - local derivative ``d output / d parameter``
|
|
9
|
+
by central (or forward) differences with common random numbers.
|
|
10
|
+
* :func:`correlation_sensitivity` - global screening from Monte Carlo
|
|
11
|
+
samples: Pearson, Spearman rank correlation and standardised regression
|
|
12
|
+
coefficients (SRC, with the regression R^2 so you can tell whether a
|
|
13
|
+
linear summary is adequate).
|
|
14
|
+
|
|
15
|
+
* :func:`sobol_indices` - variance-based global sensitivity: first-order
|
|
16
|
+
and total-effect Sobol indices (Saltelli 2010 / Jansen estimators) with
|
|
17
|
+
bootstrap confidence intervals. Captures interactions and non-linear
|
|
18
|
+
effects at a cost of ``N * (d + 2)`` model evaluations.
|
|
19
|
+
|
|
20
|
+
Correlation measures are cheap screening tools that miss interactions and
|
|
21
|
+
non-monotonic effects; use Sobol indices when that matters.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import inspect
|
|
27
|
+
import math
|
|
28
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
29
|
+
from dataclasses import asdict, dataclass
|
|
30
|
+
from typing import Any, Literal
|
|
31
|
+
|
|
32
|
+
import numpy as np
|
|
33
|
+
from scipy import stats as _st
|
|
34
|
+
|
|
35
|
+
from simulsi.analysis.report import format_table
|
|
36
|
+
from simulsi.core.model import Model
|
|
37
|
+
from simulsi.experiments.montecarlo import MonteCarloResult, sample_inputs
|
|
38
|
+
from simulsi.randomness.distributions import DistributionLike
|
|
39
|
+
from simulsi.randomness.stream import RandomStream, derive_seed
|
|
40
|
+
from simulsi.statistics.core import paired_difference, t_half_width
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class SensitivityRow:
|
|
45
|
+
parameter: str
|
|
46
|
+
output: str
|
|
47
|
+
method: str
|
|
48
|
+
value: float
|
|
49
|
+
ci_low: float = math.nan
|
|
50
|
+
ci_high: float = math.nan
|
|
51
|
+
setting: str = ""
|
|
52
|
+
detail: str = ""
|
|
53
|
+
|
|
54
|
+
def to_dict(self) -> dict[str, Any]:
|
|
55
|
+
return asdict(self)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class SensitivityResult:
|
|
59
|
+
def __init__(
|
|
60
|
+
self, rows: Sequence[SensitivityRow], info: Mapping[str, Any] | None = None
|
|
61
|
+
) -> None:
|
|
62
|
+
self.rows = list(rows)
|
|
63
|
+
self.info = dict(info or {})
|
|
64
|
+
|
|
65
|
+
def ranking(self, output: str | None = None, method: str | None = None) -> list[SensitivityRow]:
|
|
66
|
+
"""Rows sorted by absolute effect size, largest first."""
|
|
67
|
+
rows = [
|
|
68
|
+
r
|
|
69
|
+
for r in self.rows
|
|
70
|
+
if (output is None or r.output == output) and (method is None or r.method == method)
|
|
71
|
+
]
|
|
72
|
+
return sorted(rows, key=lambda r: -abs(r.value) if not math.isnan(r.value) else 0.0)
|
|
73
|
+
|
|
74
|
+
def to_dicts(self) -> list[dict[str, Any]]:
|
|
75
|
+
return [r.to_dict() for r in self.rows]
|
|
76
|
+
|
|
77
|
+
def format(self) -> str:
|
|
78
|
+
return format_table(
|
|
79
|
+
[
|
|
80
|
+
{
|
|
81
|
+
"parameter": r.parameter,
|
|
82
|
+
"setting": r.setting,
|
|
83
|
+
"output": r.output,
|
|
84
|
+
"method": r.method,
|
|
85
|
+
"value": r.value,
|
|
86
|
+
"ci_low": r.ci_low,
|
|
87
|
+
"ci_high": r.ci_high,
|
|
88
|
+
}
|
|
89
|
+
for r in self.ranking()
|
|
90
|
+
]
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
def __repr__(self) -> str:
|
|
94
|
+
return f"SensitivityResult({len(self.rows)} rows)"
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _run_metrics(
|
|
98
|
+
model: Model, params: Mapping[str, Any], outputs: Sequence[str], reps: int, seed: int
|
|
99
|
+
) -> dict[str, np.ndarray[Any, Any]]:
|
|
100
|
+
vals: dict[str, list[float]] = {o: [] for o in outputs}
|
|
101
|
+
for r in range(reps):
|
|
102
|
+
m = model.simulate(params, seed=derive_seed(seed, "replication", r)).metrics
|
|
103
|
+
for o in outputs:
|
|
104
|
+
if o not in m:
|
|
105
|
+
raise KeyError(f"model produced no metric {o!r}")
|
|
106
|
+
vals[o].append(m[o])
|
|
107
|
+
return {o: np.asarray(v, dtype=float) for o, v in vals.items()}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def one_at_a_time(
|
|
111
|
+
model: Model,
|
|
112
|
+
changes: Mapping[str, Sequence[Any]],
|
|
113
|
+
outputs: Sequence[str],
|
|
114
|
+
*,
|
|
115
|
+
base: Mapping[str, Any] | None = None,
|
|
116
|
+
replications: int = 10,
|
|
117
|
+
seed: int = 0,
|
|
118
|
+
confidence: float = 0.95,
|
|
119
|
+
) -> SensitivityResult:
|
|
120
|
+
"""Vary each parameter in ``changes`` alone; report output change vs the base point.
|
|
121
|
+
|
|
122
|
+
``value`` is the mean change in the output; ``detail`` includes the
|
|
123
|
+
elasticity ``(dy/y) / (dx/x)`` when both base values are non-zero numbers.
|
|
124
|
+
"""
|
|
125
|
+
base_params = dict(model.resolve(base))
|
|
126
|
+
base_out = _run_metrics(model, base_params, outputs, replications, seed)
|
|
127
|
+
rows = []
|
|
128
|
+
for param, values in changes.items():
|
|
129
|
+
if param not in base_params:
|
|
130
|
+
raise KeyError(f"unknown parameter {param!r}")
|
|
131
|
+
x0 = base_params[param]
|
|
132
|
+
for v in values:
|
|
133
|
+
out = _run_metrics(model, {**(base or {}), param: v}, outputs, replications, seed)
|
|
134
|
+
for o in outputs:
|
|
135
|
+
d = paired_difference(base_out[o], out[o], confidence)
|
|
136
|
+
elasticity = math.nan
|
|
137
|
+
y0 = float(np.nanmean(base_out[o]))
|
|
138
|
+
if (
|
|
139
|
+
isinstance(x0, int | float)
|
|
140
|
+
and isinstance(v, int | float)
|
|
141
|
+
and x0
|
|
142
|
+
and y0
|
|
143
|
+
and v != x0
|
|
144
|
+
):
|
|
145
|
+
elasticity = (d.difference / y0) / ((v - x0) / x0)
|
|
146
|
+
rows.append(
|
|
147
|
+
SensitivityRow(
|
|
148
|
+
param,
|
|
149
|
+
o,
|
|
150
|
+
"oat",
|
|
151
|
+
d.difference,
|
|
152
|
+
d.ci_low,
|
|
153
|
+
d.ci_high,
|
|
154
|
+
f"{param}={v}",
|
|
155
|
+
f"base {param}={x0}; base output={y0:.4g}; elasticity={elasticity:.3g}",
|
|
156
|
+
)
|
|
157
|
+
)
|
|
158
|
+
return SensitivityResult(
|
|
159
|
+
rows, {"base": base_params, "replications": replications, "seed": seed}
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def finite_difference(
|
|
164
|
+
model: Model,
|
|
165
|
+
parameters: Sequence[str],
|
|
166
|
+
outputs: Sequence[str],
|
|
167
|
+
*,
|
|
168
|
+
base: Mapping[str, Any] | None = None,
|
|
169
|
+
relative_step: float = 0.05,
|
|
170
|
+
replications: int = 10,
|
|
171
|
+
seed: int = 0,
|
|
172
|
+
scheme: Literal["central", "forward"] = "central",
|
|
173
|
+
confidence: float = 0.95,
|
|
174
|
+
) -> SensitivityResult:
|
|
175
|
+
"""Local derivatives by finite differences with common random numbers.
|
|
176
|
+
|
|
177
|
+
Integer parameters use a step of at least 1. The CI comes from the
|
|
178
|
+
per-replication paired differences, so it reflects simulation noise only;
|
|
179
|
+
a step that is too small amplifies noise, too large adds bias.
|
|
180
|
+
"""
|
|
181
|
+
base_params = dict(model.resolve(base))
|
|
182
|
+
rows = []
|
|
183
|
+
base_out = (
|
|
184
|
+
_run_metrics(model, base_params, outputs, replications, seed)
|
|
185
|
+
if scheme == "forward"
|
|
186
|
+
else None
|
|
187
|
+
)
|
|
188
|
+
for p in parameters:
|
|
189
|
+
x0 = base_params[p]
|
|
190
|
+
if isinstance(x0, bool) or not isinstance(x0, int | float):
|
|
191
|
+
raise TypeError(f"parameter {p!r} is not numeric")
|
|
192
|
+
h: float = abs(x0) * relative_step or relative_step
|
|
193
|
+
if isinstance(x0, int):
|
|
194
|
+
h = max(1, round(h))
|
|
195
|
+
hi_params = {**(base or {}), p: x0 + h}
|
|
196
|
+
hi = _run_metrics(model, hi_params, outputs, replications, seed)
|
|
197
|
+
if scheme == "central":
|
|
198
|
+
lo = _run_metrics(model, {**(base or {}), p: x0 - h}, outputs, replications, seed)
|
|
199
|
+
span = 2 * h
|
|
200
|
+
else:
|
|
201
|
+
assert base_out is not None
|
|
202
|
+
lo, span = base_out, h
|
|
203
|
+
for o in outputs:
|
|
204
|
+
per_rep = (hi[o] - lo[o]) / span
|
|
205
|
+
mean = float(np.nanmean(per_rep))
|
|
206
|
+
hw = t_half_width(per_rep, confidence)
|
|
207
|
+
rows.append(
|
|
208
|
+
SensitivityRow(p, o, f"fd-{scheme}", mean, mean - hw, mean + hw, f"step={h:g}")
|
|
209
|
+
)
|
|
210
|
+
return SensitivityResult(rows, {"base": base_params, "relative_step": relative_step})
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def correlation_sensitivity(
|
|
214
|
+
mc: MonteCarloResult,
|
|
215
|
+
outputs: Sequence[str] | None = None,
|
|
216
|
+
*,
|
|
217
|
+
methods: Sequence[Literal["pearson", "spearman", "src"]] = ("pearson", "spearman", "src"),
|
|
218
|
+
) -> SensitivityResult:
|
|
219
|
+
"""Global screening from Monte Carlo samples of uncertain inputs."""
|
|
220
|
+
names = [k for k, v in mc.inputs.items() if np.issubdtype(np.asarray(v).dtype, np.number)]
|
|
221
|
+
if not names:
|
|
222
|
+
raise ValueError("no numeric sampled inputs to analyse")
|
|
223
|
+
X = np.column_stack([np.asarray(mc.inputs[k], dtype=float) for k in names])
|
|
224
|
+
rows = []
|
|
225
|
+
info: dict[str, Any] = {"r2": {}}
|
|
226
|
+
for o in outputs if outputs is not None else mc.output_names:
|
|
227
|
+
y = mc.outputs[o]
|
|
228
|
+
ok = ~np.isnan(y)
|
|
229
|
+
Xo, yo = X[ok], y[ok]
|
|
230
|
+
if len(yo) < 3 or np.std(yo) == 0:
|
|
231
|
+
continue
|
|
232
|
+
for j, name in enumerate(names):
|
|
233
|
+
x = Xo[:, j]
|
|
234
|
+
if np.std(x) == 0:
|
|
235
|
+
continue
|
|
236
|
+
if "pearson" in methods:
|
|
237
|
+
r, pv = _st.pearsonr(x, yo)
|
|
238
|
+
rows.append(
|
|
239
|
+
SensitivityRow(name, o, "pearson", float(r), detail=f"p={float(pv):.3g}")
|
|
240
|
+
)
|
|
241
|
+
if "spearman" in methods:
|
|
242
|
+
rho, pv = _st.spearmanr(x, yo)
|
|
243
|
+
rows.append(
|
|
244
|
+
SensitivityRow(name, o, "spearman", float(rho), detail=f"p={float(pv):.3g}")
|
|
245
|
+
)
|
|
246
|
+
if "src" in methods:
|
|
247
|
+
sd = Xo.std(axis=0)
|
|
248
|
+
keep = sd > 0
|
|
249
|
+
Z = (Xo[:, keep] - Xo[:, keep].mean(axis=0)) / sd[keep]
|
|
250
|
+
zy = (yo - yo.mean()) / yo.std()
|
|
251
|
+
A = np.column_stack([np.ones(len(zy)), Z])
|
|
252
|
+
coef, *_ = np.linalg.lstsq(A, zy, rcond=None)
|
|
253
|
+
pred = A @ coef
|
|
254
|
+
r2 = 1 - float(((zy - pred) ** 2).sum() / (zy**2).sum())
|
|
255
|
+
info["r2"][o] = r2
|
|
256
|
+
for name, c in zip(
|
|
257
|
+
[n for n, k in zip(names, keep, strict=True) if k], coef[1:], strict=True
|
|
258
|
+
):
|
|
259
|
+
rows.append(SensitivityRow(name, o, "src", float(c), detail=f"R2={r2:.3f}"))
|
|
260
|
+
return SensitivityResult(rows, info)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _evaluate_rows(
|
|
264
|
+
model: Callable[..., Any] | Model,
|
|
265
|
+
inputs: Mapping[str, np.ndarray[Any, Any]],
|
|
266
|
+
rows: int,
|
|
267
|
+
*,
|
|
268
|
+
fixed: Mapping[str, Any],
|
|
269
|
+
vectorized: bool,
|
|
270
|
+
seed: int,
|
|
271
|
+
crn_period: int,
|
|
272
|
+
) -> dict[str, np.ndarray[Any, Any]]:
|
|
273
|
+
"""Evaluate ``model`` on every row of ``inputs``. Row ``j`` uses seed index ``j % crn_period``."""
|
|
274
|
+
if vectorized:
|
|
275
|
+
if isinstance(model, Model):
|
|
276
|
+
raise ValueError("vectorized=True is not available for simulation models")
|
|
277
|
+
res = model(**fixed, **inputs)
|
|
278
|
+
arrays = res if isinstance(res, Mapping) else {"value": res}
|
|
279
|
+
return {
|
|
280
|
+
k: np.broadcast_to(np.asarray(v, dtype=float), (rows,)).copy()
|
|
281
|
+
for k, v in arrays.items()
|
|
282
|
+
}
|
|
283
|
+
wants_rng = not isinstance(model, Model) and "rng" in inspect.signature(model).parameters
|
|
284
|
+
out: dict[str, list[float]] = {}
|
|
285
|
+
for j in range(rows):
|
|
286
|
+
params = {
|
|
287
|
+
**fixed,
|
|
288
|
+
**{k: v[j].item() if hasattr(v[j], "item") else v[j] for k, v in inputs.items()},
|
|
289
|
+
}
|
|
290
|
+
rep_seed = derive_seed(seed, "replication", j % crn_period)
|
|
291
|
+
if isinstance(model, Model):
|
|
292
|
+
values: Any = model.simulate(params, seed=rep_seed).metrics
|
|
293
|
+
elif wants_rng:
|
|
294
|
+
values = model(**params, rng=RandomStream(rep_seed))
|
|
295
|
+
else:
|
|
296
|
+
values = model(**params)
|
|
297
|
+
row = dict(values) if isinstance(values, Mapping) else {"value": float(values)}
|
|
298
|
+
for k, v in row.items():
|
|
299
|
+
out.setdefault(k, [math.nan] * j).append(float(v))
|
|
300
|
+
for k in out:
|
|
301
|
+
if len(out[k]) < j + 1:
|
|
302
|
+
out[k].append(math.nan)
|
|
303
|
+
return {k: np.asarray(v, dtype=float) for k, v in out.items()}
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def sobol_indices(
|
|
307
|
+
model: Callable[..., Any] | Model,
|
|
308
|
+
parameters: Mapping[str, DistributionLike | Mapping[str, Any]],
|
|
309
|
+
n: int = 1024,
|
|
310
|
+
*,
|
|
311
|
+
outputs: Sequence[str] | None = None,
|
|
312
|
+
seed: int = 0,
|
|
313
|
+
fixed: Mapping[str, Any] | None = None,
|
|
314
|
+
vectorized: bool = False,
|
|
315
|
+
confidence: float = 0.95,
|
|
316
|
+
n_bootstrap: int = 500,
|
|
317
|
+
) -> SensitivityResult:
|
|
318
|
+
"""First-order and total-effect Sobol indices with bootstrap confidence intervals.
|
|
319
|
+
|
|
320
|
+
Uses the Saltelli (2010) design: two independent sample matrices ``A`` and
|
|
321
|
+
``B`` of ``n`` rows plus, for each of the ``d`` parameters, ``A`` with that
|
|
322
|
+
column taken from ``B``. First-order indices use the Saltelli (2010)
|
|
323
|
+
estimator, total effects the Jansen (1999) estimator. Inputs are assumed
|
|
324
|
+
independent. For simulation models every matrix row ``j`` uses the same
|
|
325
|
+
replication seed (common random numbers), which keeps simulation noise
|
|
326
|
+
from swamping the index estimates; noise still adds some bias, so use
|
|
327
|
+
enough replication length or average several runs per row if outputs are
|
|
328
|
+
very noisy.
|
|
329
|
+
|
|
330
|
+
``value`` is the index estimate (``method`` = ``sobol-first`` or
|
|
331
|
+
``sobol-total``). Estimates can fall slightly outside [0, 1] with small
|
|
332
|
+
``n``; the bootstrap interval shows how precise they are.
|
|
333
|
+
"""
|
|
334
|
+
if n < 2:
|
|
335
|
+
raise ValueError("n must be >= 2")
|
|
336
|
+
names = list(parameters)
|
|
337
|
+
d = len(names)
|
|
338
|
+
if d == 0:
|
|
339
|
+
raise ValueError("need at least one uncertain parameter")
|
|
340
|
+
a = sample_inputs(parameters, n, derive_seed(seed, "sobol-A"))
|
|
341
|
+
b = sample_inputs(parameters, n, derive_seed(seed, "sobol-B"))
|
|
342
|
+
blocks = [a, b] + [{k: (b[k] if k == name else a[k]) for k in names} for name in names]
|
|
343
|
+
design = {k: np.concatenate([blk[k] for blk in blocks]) for k in names}
|
|
344
|
+
total = n * (d + 2)
|
|
345
|
+
y_all = _evaluate_rows(
|
|
346
|
+
model,
|
|
347
|
+
design,
|
|
348
|
+
total,
|
|
349
|
+
fixed=dict(fixed or {}),
|
|
350
|
+
vectorized=vectorized,
|
|
351
|
+
seed=seed,
|
|
352
|
+
crn_period=n,
|
|
353
|
+
)
|
|
354
|
+
rng = np.random.default_rng(derive_seed(seed, "sobol-bootstrap"))
|
|
355
|
+
boot_idx = rng.integers(0, n, size=(n_bootstrap, n))
|
|
356
|
+
alpha = (1 - confidence) / 2
|
|
357
|
+
rows: list[SensitivityRow] = []
|
|
358
|
+
info: dict[str, Any] = {"n": n, "evaluations": total, "variance": {}}
|
|
359
|
+
for o in outputs if outputs is not None else list(y_all):
|
|
360
|
+
if o not in y_all:
|
|
361
|
+
raise KeyError(f"model produced no output {o!r}")
|
|
362
|
+
y = y_all[o].reshape(d + 2, n)
|
|
363
|
+
f_a, f_b, f_ab = y[0], y[1], y[2:] # (n,), (n,), (d, n)
|
|
364
|
+
var = np.var(np.concatenate([f_a, f_b]), ddof=1)
|
|
365
|
+
s1 = np.mean(f_b * (f_ab - f_a), axis=1) / var
|
|
366
|
+
st = 0.5 * np.mean((f_a - f_ab) ** 2, axis=1) / var
|
|
367
|
+
# bootstrap over rows: shapes (B, n) and (d, B, n)
|
|
368
|
+
fa_b, fb_b, fab_b = f_a[boot_idx], f_b[boot_idx], f_ab[:, boot_idx]
|
|
369
|
+
var_b = np.var(np.concatenate([fa_b, fb_b], axis=1), axis=1, ddof=1)
|
|
370
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
371
|
+
b1 = np.mean(fb_b * (fab_b - fa_b), axis=2) / var_b
|
|
372
|
+
bt = 0.5 * np.mean((fa_b - fab_b) ** 2, axis=2) / var_b
|
|
373
|
+
info["variance"][o] = float(var)
|
|
374
|
+
for i, name in enumerate(names):
|
|
375
|
+
lo1, hi1 = np.nanquantile(b1[i], [alpha, 1 - alpha])
|
|
376
|
+
lot, hit = np.nanquantile(bt[i], [alpha, 1 - alpha])
|
|
377
|
+
rows.append(
|
|
378
|
+
SensitivityRow(
|
|
379
|
+
name, o, "sobol-first", float(s1[i]), float(lo1), float(hi1), f"N={n}"
|
|
380
|
+
)
|
|
381
|
+
)
|
|
382
|
+
rows.append(
|
|
383
|
+
SensitivityRow(
|
|
384
|
+
name, o, "sobol-total", float(st[i]), float(lot), float(hit), f"N={n}"
|
|
385
|
+
)
|
|
386
|
+
)
|
|
387
|
+
return SensitivityResult(rows, info)
|