rasad-sim 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rasad/__init__.py +84 -0
- rasad/adapter.py +133 -0
- rasad/adapters/__init__.py +0 -0
- rasad/adapters/omran.py +70 -0
- rasad/analyzer.py +178 -0
- rasad/py.typed +0 -0
- rasad/report.py +136 -0
- rasad/runner.py +60 -0
- rasad/sampler.py +41 -0
- rasad_sim-0.1.0.dist-info/METADATA +333 -0
- rasad_sim-0.1.0.dist-info/RECORD +14 -0
- rasad_sim-0.1.0.dist-info/WHEEL +5 -0
- rasad_sim-0.1.0.dist-info/licenses/LICENSE +21 -0
- rasad_sim-0.1.0.dist-info/top_level.txt +1 -0
rasad/__init__.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Rasad — measuring the variability of simulation results."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from rasad.adapter import split_outputs
|
|
7
|
+
from rasad.analyzer import (
|
|
8
|
+
THRESHOLDS,
|
|
9
|
+
classify,
|
|
10
|
+
divergence,
|
|
11
|
+
summarize,
|
|
12
|
+
validate_thresholds,
|
|
13
|
+
)
|
|
14
|
+
from rasad.report import Report
|
|
15
|
+
from rasad.runner import run_all
|
|
16
|
+
from rasad.sampler import make_plan
|
|
17
|
+
|
|
18
|
+
__all__ = ["measure", "Report"]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def measure(
|
|
22
|
+
fn: Callable[..., dict],
|
|
23
|
+
params: dict[str, Any],
|
|
24
|
+
runs: int = 100,
|
|
25
|
+
base_seed: int = 0,
|
|
26
|
+
thresholds: dict[str, float] | None = None,
|
|
27
|
+
) -> Report:
|
|
28
|
+
"""Measure the variability of a simulation model's output.
|
|
29
|
+
|
|
30
|
+
Parameters
|
|
31
|
+
----------
|
|
32
|
+
fn
|
|
33
|
+
A model matching the simulation interface
|
|
34
|
+
``def run(params: dict, seed: int) -> dict``.
|
|
35
|
+
params
|
|
36
|
+
The simulation parameters, fixed across every run.
|
|
37
|
+
runs
|
|
38
|
+
How many runs to make. Must be at least 2.
|
|
39
|
+
base_seed
|
|
40
|
+
The seed of the first run; run ``i`` gets ``base_seed + i``.
|
|
41
|
+
thresholds
|
|
42
|
+
The classification cutoffs. ``None`` means use the default
|
|
43
|
+
:data:`rasad.analyzer.THRESHOLDS`. These values are a convention
|
|
44
|
+
the caller owns — they describe what counts as "low" /
|
|
45
|
+
"moderate" / "high" variability for this measurement and are not
|
|
46
|
+
derived from the data. A supplied dict is validated before any run.
|
|
47
|
+
|
|
48
|
+
Returns
|
|
49
|
+
-------
|
|
50
|
+
Report
|
|
51
|
+
Per scalar output its summary plus a variability label, per series
|
|
52
|
+
output its divergence curve, the run count, and the thresholds used.
|
|
53
|
+
|
|
54
|
+
Raises
|
|
55
|
+
------
|
|
56
|
+
ValueError
|
|
57
|
+
When ``runs`` is below 2, or when ``thresholds`` fails
|
|
58
|
+
:func:`rasad.analyzer.validate_thresholds`.
|
|
59
|
+
"""
|
|
60
|
+
if thresholds is None:
|
|
61
|
+
thresholds = THRESHOLDS
|
|
62
|
+
thresholds = validate_thresholds(thresholds)
|
|
63
|
+
|
|
64
|
+
plan = make_plan(params, runs, base_seed)
|
|
65
|
+
results = run_all(fn, plan)
|
|
66
|
+
|
|
67
|
+
scalar_values: dict[str, list[float]] = {}
|
|
68
|
+
series_curves: dict[str, list[list[float]]] = {}
|
|
69
|
+
for out in results:
|
|
70
|
+
scalars, series = split_outputs(out)
|
|
71
|
+
for name, value in scalars.items():
|
|
72
|
+
scalar_values.setdefault(name, []).append(value)
|
|
73
|
+
for name, curve in series.items():
|
|
74
|
+
series_curves.setdefault(name, []).append(curve)
|
|
75
|
+
|
|
76
|
+
scalars: dict[str, dict[str, Any]] = {}
|
|
77
|
+
for name, values in scalar_values.items():
|
|
78
|
+
stats = summarize(values)
|
|
79
|
+
stats["variability"] = classify(stats["cv"], thresholds)
|
|
80
|
+
scalars[name] = stats
|
|
81
|
+
|
|
82
|
+
series = {name: divergence(curves) for name, curves in series_curves.items()}
|
|
83
|
+
|
|
84
|
+
return Report(scalars=scalars, series=series, runs=runs, thresholds=thresholds)
|
rasad/adapter.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Check that a user model fits the required interface and sort its output.
|
|
2
|
+
|
|
3
|
+
A model is any plain function matching the required simulation
|
|
4
|
+
interface::
|
|
5
|
+
|
|
6
|
+
def run(params: dict, seed: int) -> dict: ...
|
|
7
|
+
|
|
8
|
+
This module validates that a candidate function actually has those two
|
|
9
|
+
parameters, and splits a returned result dict into scalar results (a
|
|
10
|
+
single final number) and time series (lists of numbers), rejecting
|
|
11
|
+
anything a simulation should never produce.
|
|
12
|
+
|
|
13
|
+
Numpy scalars of numeric dtype and one-dimensional numpy arrays of
|
|
14
|
+
numeric dtype are accepted and converted to plain Python values.
|
|
15
|
+
Booleans are never accepted, whether Python ``bool`` or ``numpy.bool_``.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import inspect
|
|
19
|
+
from collections.abc import Callable
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
import numpy as np
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def validate_model(fn: Callable[..., dict]) -> None:
|
|
26
|
+
"""Check that a function matches the required model interface.
|
|
27
|
+
|
|
28
|
+
Parameters
|
|
29
|
+
----------
|
|
30
|
+
fn
|
|
31
|
+
The candidate model, expected to accept exactly ``params`` and
|
|
32
|
+
``seed`` as its first two parameters.
|
|
33
|
+
|
|
34
|
+
Raises
|
|
35
|
+
------
|
|
36
|
+
TypeError
|
|
37
|
+
When ``fn`` is not callable, or when its first two parameter
|
|
38
|
+
names are not exactly ``("params", "seed")``.
|
|
39
|
+
"""
|
|
40
|
+
if not callable(fn):
|
|
41
|
+
raise TypeError(f"model must be callable, got {type(fn).__name__}")
|
|
42
|
+
|
|
43
|
+
names = [p.name for p in inspect.signature(fn).parameters.values()][:2]
|
|
44
|
+
if names != ["params", "seed"]:
|
|
45
|
+
raise TypeError(
|
|
46
|
+
f"model must accept (params, seed) as its first two arguments, got ({', '.join(names)})"
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _is_number(value: Any) -> bool:
|
|
51
|
+
"""True for a real number — never for a boolean, of either flavour.
|
|
52
|
+
|
|
53
|
+
``isinstance(True, int)`` is True in Python and ``np.bool_`` is a numpy
|
|
54
|
+
scalar, so both must be excluded before any numeric test.
|
|
55
|
+
"""
|
|
56
|
+
if isinstance(value, (bool, np.bool_)):
|
|
57
|
+
return False
|
|
58
|
+
if isinstance(value, (int, float)):
|
|
59
|
+
return True
|
|
60
|
+
return isinstance(value, np.generic) and np.issubdtype(value.dtype, np.number)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def split_outputs(out: dict[str, Any]) -> tuple[dict[str, float], dict[str, list[float]]]:
|
|
64
|
+
"""Split a model result into scalar results and time series.
|
|
65
|
+
|
|
66
|
+
Parameters
|
|
67
|
+
----------
|
|
68
|
+
out
|
|
69
|
+
The dict returned by a model run. Each value must be a number
|
|
70
|
+
(a scalar result) or a list/tuple of numbers (a time series).
|
|
71
|
+
Numpy scalars of numeric dtype and zero-dimensional numpy arrays
|
|
72
|
+
of numeric dtype count as numbers; one-dimensional numpy arrays
|
|
73
|
+
of numeric dtype count as time series.
|
|
74
|
+
|
|
75
|
+
Returns
|
|
76
|
+
-------
|
|
77
|
+
tuple[dict[str, float], dict[str, list[float]]]
|
|
78
|
+
``(scalars, series)``: scalars holds each single-number output
|
|
79
|
+
as a plain ``float``, series holds each sequence output as a
|
|
80
|
+
``list[float]``.
|
|
81
|
+
|
|
82
|
+
Raises
|
|
83
|
+
------
|
|
84
|
+
TypeError
|
|
85
|
+
When a value is a ``bool`` or ``numpy.bool_``, a ``str``,
|
|
86
|
+
``bytes``, or any other type that is neither a single number
|
|
87
|
+
nor a sequence of numbers, including numpy arrays with two or
|
|
88
|
+
more dimensions and arrays of boolean or object dtype.
|
|
89
|
+
"""
|
|
90
|
+
scalars: dict[str, float] = {}
|
|
91
|
+
series: dict[str, list[float]] = {}
|
|
92
|
+
|
|
93
|
+
for key, value in out.items():
|
|
94
|
+
if isinstance(value, (bool, np.bool_)):
|
|
95
|
+
raise TypeError(f"unsupported output {key!r}: boolean values are not allowed")
|
|
96
|
+
if isinstance(value, np.ndarray):
|
|
97
|
+
if value.ndim == 0 and np.issubdtype(value.dtype, np.number):
|
|
98
|
+
scalars[key] = float(value)
|
|
99
|
+
elif value.ndim == 1 and np.issubdtype(value.dtype, np.number):
|
|
100
|
+
series[key] = [float(item) for item in value]
|
|
101
|
+
elif np.issubdtype(value.dtype, np.bool_):
|
|
102
|
+
raise TypeError(
|
|
103
|
+
f"unsupported output {key!r}: boolean arrays are not allowed"
|
|
104
|
+
)
|
|
105
|
+
else:
|
|
106
|
+
raise TypeError(
|
|
107
|
+
f"unsupported output {key!r}: numpy array with dtype "
|
|
108
|
+
f"{value.dtype} and {value.ndim} dimension(s) is not supported"
|
|
109
|
+
)
|
|
110
|
+
elif isinstance(value, (int, float)) or (
|
|
111
|
+
isinstance(value, np.generic) and np.issubdtype(value.dtype, np.number)
|
|
112
|
+
):
|
|
113
|
+
scalars[key] = float(value)
|
|
114
|
+
elif isinstance(value, (list, tuple)):
|
|
115
|
+
# Every element is checked: a list of booleans or strings would
|
|
116
|
+
# otherwise pass silently and produce statistics over numbers
|
|
117
|
+
# that mean nothing.
|
|
118
|
+
bad = next(
|
|
119
|
+
((i, v) for i, v in enumerate(value) if not _is_number(v)), None
|
|
120
|
+
)
|
|
121
|
+
if bad is not None:
|
|
122
|
+
index, item = bad
|
|
123
|
+
raise TypeError(
|
|
124
|
+
f"unsupported output {key!r}: element {index} is "
|
|
125
|
+
f"{type(item).__name__}, not a number"
|
|
126
|
+
)
|
|
127
|
+
series[key] = [float(item) for item in value]
|
|
128
|
+
else:
|
|
129
|
+
raise TypeError(
|
|
130
|
+
f"unsupported output {key!r}: must be a number or a sequence of numbers"
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
return scalars, series
|
|
File without changes
|
rasad/adapters/omran.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Wrap Omran from the outside so Rasad can measure it.
|
|
2
|
+
|
|
3
|
+
This file lives in the Rasad repository. It imports Omran's ``nation`` and
|
|
4
|
+
``world`` modules by putting Omran's ``src`` directory on ``sys.path`` at
|
|
5
|
+
call time, and drives Omran purely through its public interface. Nothing
|
|
6
|
+
inside the Omran project is created, modified, or deleted.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import contextlib
|
|
10
|
+
import io
|
|
11
|
+
import random
|
|
12
|
+
import sys
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def make_omran_run(omran_src: str, years: int) -> Callable[[dict, int], dict]:
|
|
17
|
+
"""Build a Rasad-compatible model function that runs Omran.
|
|
18
|
+
|
|
19
|
+
Parameters
|
|
20
|
+
----------
|
|
21
|
+
omran_src
|
|
22
|
+
Absolute path to Omran's ``src`` directory. It is inserted at the
|
|
23
|
+
front of ``sys.path`` if not already present, so that ``nation``
|
|
24
|
+
and ``world`` can be imported from outside the Omran project.
|
|
25
|
+
years
|
|
26
|
+
How many simulated years each run performs.
|
|
27
|
+
|
|
28
|
+
Returns
|
|
29
|
+
-------
|
|
30
|
+
Callable[[dict, int], dict]
|
|
31
|
+
A closure ``run(params, seed)`` that seeds Python's global RNG,
|
|
32
|
+
builds a fresh set of nations, steps Omran's ``WorldModel`` for
|
|
33
|
+
``years`` years with stdout suppressed, and returns Omran's output
|
|
34
|
+
as scalars plus a population trace.
|
|
35
|
+
"""
|
|
36
|
+
if omran_src not in sys.path:
|
|
37
|
+
sys.path.insert(0, omran_src)
|
|
38
|
+
|
|
39
|
+
from nation import Nation
|
|
40
|
+
from world import WorldModel
|
|
41
|
+
|
|
42
|
+
def run(params: dict, seed: int) -> dict:
|
|
43
|
+
random.seed(seed)
|
|
44
|
+
|
|
45
|
+
nations = [
|
|
46
|
+
Nation(name="Nation_A", population=500, food=2000, growth_rate=0.03),
|
|
47
|
+
Nation(name="Nation_B", population=80, food=2000, growth_rate=0.035),
|
|
48
|
+
Nation(name="Nation_C", population=200, food=2000, growth_rate=0.032),
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
population_trace = []
|
|
52
|
+
with contextlib.redirect_stdout(io.StringIO()):
|
|
53
|
+
world = WorldModel(nations)
|
|
54
|
+
for _ in range(years):
|
|
55
|
+
world.step()
|
|
56
|
+
population_trace.append(
|
|
57
|
+
sum(n.population for n in world.nations if n.is_alive)
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
return {
|
|
61
|
+
"final_total_population": float(
|
|
62
|
+
sum(n.population for n in world.nations if n.is_alive)
|
|
63
|
+
),
|
|
64
|
+
"survivors": float(sum(1 for n in world.nations if n.is_alive)),
|
|
65
|
+
"total_wars": float(sum(n.war_count for n in world.nations)),
|
|
66
|
+
"total_famines": float(sum(n.famine_count for n in world.nations)),
|
|
67
|
+
"population_trace": [float(value) for value in population_trace],
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
return run
|
rasad/analyzer.py
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""Arithmetic on sequences of numbers, nothing more.
|
|
2
|
+
|
|
3
|
+
This module knows nothing about simulations or how the numbers it is
|
|
4
|
+
given were produced. It only describes the spread and location of a
|
|
5
|
+
sample: count, mean, sample standard deviation, coefficient of
|
|
6
|
+
variation, extrema, and percentile interval.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
from collections.abc import Sequence
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
|
|
14
|
+
# Default classification cutoffs. These are a convention chosen by the
|
|
15
|
+
# project's authors, not a derived or theoretical result: a caller may
|
|
16
|
+
# pass their own thresholds to ``classify`` and ``rasad.measure``, in
|
|
17
|
+
# which case these defaults are ignored.
|
|
18
|
+
THRESHOLDS: dict[str, float] = {"low": 0.05, "moderate": 0.20}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def summarize(values: Sequence[float]) -> dict[str, float | int]:
|
|
22
|
+
"""Summarize a sample of numbers.
|
|
23
|
+
|
|
24
|
+
Parameters
|
|
25
|
+
----------
|
|
26
|
+
values
|
|
27
|
+
A sequence of numeric values, e.g. repeated simulation runs.
|
|
28
|
+
|
|
29
|
+
Returns
|
|
30
|
+
-------
|
|
31
|
+
dict
|
|
32
|
+
Keys ``n``, ``mean``, ``std``, ``cv``, ``min``, ``max``, ``p05``, ``p95``.
|
|
33
|
+
``std`` is the sample standard deviation (``ddof=1``); ``cv`` is
|
|
34
|
+
``std / abs(mean)``; it is zero whenever ``std`` is zero, and
|
|
35
|
+
infinite when the mean is zero but the values still vary.
|
|
36
|
+
|
|
37
|
+
Raises
|
|
38
|
+
------
|
|
39
|
+
ValueError
|
|
40
|
+
When fewer than two values are provided.
|
|
41
|
+
"""
|
|
42
|
+
arr = np.asarray(values, dtype=float)
|
|
43
|
+
if arr.size < 2:
|
|
44
|
+
raise ValueError("need at least 2 values to summarize")
|
|
45
|
+
if not np.isfinite(arr).all() :
|
|
46
|
+
raise ValueError("non-finite values")
|
|
47
|
+
|
|
48
|
+
n = int(arr.size)
|
|
49
|
+
mean = float(arr.mean())
|
|
50
|
+
std = float(arr.std(ddof=1))
|
|
51
|
+
# The order of these checks is deliberate: zero spread means a fully
|
|
52
|
+
# determined value whatever the mean. Without this, an output constantly
|
|
53
|
+
# at zero would be classified "high" when it is the most stable one
|
|
54
|
+
# there is.
|
|
55
|
+
if std == 0.0:
|
|
56
|
+
cv = 0.0
|
|
57
|
+
elif mean == 0.0:
|
|
58
|
+
cv = math.inf
|
|
59
|
+
else:
|
|
60
|
+
cv = std / abs(mean)
|
|
61
|
+
p05 = float(np.percentile(arr, 5))
|
|
62
|
+
p95 = float(np.percentile(arr, 95))
|
|
63
|
+
|
|
64
|
+
return {
|
|
65
|
+
"n": n,
|
|
66
|
+
"mean": mean,
|
|
67
|
+
"std": std,
|
|
68
|
+
"cv": cv,
|
|
69
|
+
"min": float(arr.min()),
|
|
70
|
+
"max": float(arr.max()),
|
|
71
|
+
"p05": p05,
|
|
72
|
+
"p95": p95,
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def validate_thresholds(thresholds: dict[str, float]) -> dict[str, float]:
|
|
77
|
+
"""Check caller-supplied classification cutoffs and copy them.
|
|
78
|
+
|
|
79
|
+
The thresholds are a convention, not a measured property of the data,
|
|
80
|
+
so whatever the caller passes is taken as-is once it passes these
|
|
81
|
+
checks.
|
|
82
|
+
|
|
83
|
+
Parameters
|
|
84
|
+
----------
|
|
85
|
+
thresholds
|
|
86
|
+
A dict with exactly the keys ``low`` and ``moderate``. Both
|
|
87
|
+
values must be finite numbers greater than 0, and ``low``
|
|
88
|
+
must be strictly below ``moderate``.
|
|
89
|
+
|
|
90
|
+
Returns
|
|
91
|
+
-------
|
|
92
|
+
dict[str, float]
|
|
93
|
+
A plain-float copy of ``thresholds``, so mutating the caller's
|
|
94
|
+
dict afterwards cannot change the classification.
|
|
95
|
+
|
|
96
|
+
Raises
|
|
97
|
+
------
|
|
98
|
+
ValueError
|
|
99
|
+
When the keys, the positivity/finiteness, or the ordering of the
|
|
100
|
+
values do not satisfy the rules above.
|
|
101
|
+
"""
|
|
102
|
+
keys = set(thresholds)
|
|
103
|
+
if keys != {"low", "moderate"}:
|
|
104
|
+
raise ValueError(
|
|
105
|
+
f"thresholds must have exactly the keys 'low' and 'moderate', got {sorted(keys)}"
|
|
106
|
+
)
|
|
107
|
+
low = thresholds["low"]
|
|
108
|
+
moderate = thresholds["moderate"]
|
|
109
|
+
for name, value in (("low", low), ("moderate", moderate)):
|
|
110
|
+
if not (math.isfinite(value) and value > 0):
|
|
111
|
+
raise ValueError(f"{name} threshold must be a finite positive number, got {value!r}")
|
|
112
|
+
if not low < moderate:
|
|
113
|
+
raise ValueError("low must be below moderate")
|
|
114
|
+
return {"low": float(low), "moderate": float(moderate)}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def classify(cv: float, thresholds: dict[str, float] | None = None) -> str:
|
|
118
|
+
"""Label a coefficient of variation by its variability cutoff.
|
|
119
|
+
|
|
120
|
+
Parameters
|
|
121
|
+
----------
|
|
122
|
+
cv
|
|
123
|
+
The coefficient of variation of a sample.
|
|
124
|
+
thresholds
|
|
125
|
+
The classification cutoffs. ``None`` means use the module-level
|
|
126
|
+
:data:`THRESHOLDS` default; any other dict is validated with
|
|
127
|
+
:func:`validate_thresholds` first.
|
|
128
|
+
|
|
129
|
+
Returns
|
|
130
|
+
-------
|
|
131
|
+
str
|
|
132
|
+
``"low"`` below the low threshold, ``"moderate"`` up to and
|
|
133
|
+
including the moderate threshold, ``"high"`` beyond it.
|
|
134
|
+
"""
|
|
135
|
+
# kept as if/else rather than a ternary: the two branches do different
|
|
136
|
+
# things — one picks a default, the other validates untrusted input.
|
|
137
|
+
if thresholds is None: # noqa: SIM108
|
|
138
|
+
thresholds = THRESHOLDS
|
|
139
|
+
else:
|
|
140
|
+
thresholds = validate_thresholds(thresholds)
|
|
141
|
+
if cv < thresholds["low"]:
|
|
142
|
+
return "low"
|
|
143
|
+
if cv <= thresholds["moderate"]:
|
|
144
|
+
return "moderate"
|
|
145
|
+
return "high"
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def divergence(series: Sequence[Sequence[float]]) -> list[float]:
|
|
149
|
+
"""Measure how far apart a set of time series has drifted by time step.
|
|
150
|
+
|
|
151
|
+
At each time step the runs are treated as a sample and the sample
|
|
152
|
+
standard deviation is taken across runs; a flat zero curve means the
|
|
153
|
+
runs agree, a growing curve means they are drifting apart.
|
|
154
|
+
|
|
155
|
+
Parameters
|
|
156
|
+
----------
|
|
157
|
+
series
|
|
158
|
+
A sequence of equal-length time series, one per simulation run.
|
|
159
|
+
|
|
160
|
+
Returns
|
|
161
|
+
-------
|
|
162
|
+
list[float]
|
|
163
|
+
One sample standard deviation per time step, across runs.
|
|
164
|
+
|
|
165
|
+
Raises
|
|
166
|
+
------
|
|
167
|
+
ValueError
|
|
168
|
+
When fewer than two series are provided, or when the series do
|
|
169
|
+
not all have the same length.
|
|
170
|
+
"""
|
|
171
|
+
rows = [np.asarray(row, dtype=float) for row in series]
|
|
172
|
+
if len(rows) < 2:
|
|
173
|
+
raise ValueError("need at least 2 series to measure divergence")
|
|
174
|
+
if any(len(row) != len(rows[0]) for row in rows):
|
|
175
|
+
raise ValueError("all series must have the same length")
|
|
176
|
+
|
|
177
|
+
stack = np.stack(rows)
|
|
178
|
+
return [float(std) for std in stack.std(axis=0, ddof=1)]
|
rasad/py.typed
ADDED
|
File without changes
|
rasad/report.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""Presentation of a measurement, and nothing else.
|
|
2
|
+
|
|
3
|
+
This is the only module concerned with how a measurement is shown to a
|
|
4
|
+
human. No assumption about the command line may leak below it: a future
|
|
5
|
+
web layer becomes a second view over the same data, built from the same
|
|
6
|
+
:class:`Report`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import plotly.graph_objects as go
|
|
13
|
+
|
|
14
|
+
# Fixed precision so two runs with the same seeds render byte-identically.
|
|
15
|
+
_VALUE_DECIMALS = 2
|
|
16
|
+
_RATIO_DECIMALS = 4
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _fmt(value: float) -> str:
|
|
20
|
+
"""Format a magnitude — grouped thousands, fixed decimals, inf-safe."""
|
|
21
|
+
if math.isinf(value):
|
|
22
|
+
return "inf"
|
|
23
|
+
return f"{value:,.{_VALUE_DECIMALS}f}"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _fmt_ratio(value: float) -> str:
|
|
27
|
+
"""Format a dimensionless ratio, which needs more decimals than a magnitude."""
|
|
28
|
+
if math.isinf(value):
|
|
29
|
+
return "inf"
|
|
30
|
+
return f"{value:.{_RATIO_DECIMALS}f}"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class Report:
|
|
34
|
+
"""The result of measuring one model.
|
|
35
|
+
|
|
36
|
+
Attributes
|
|
37
|
+
----------
|
|
38
|
+
scalars
|
|
39
|
+
Output name to the summary dict from :func:`rasad.analyzer.summarize`
|
|
40
|
+
plus a ``variability`` key holding the string from
|
|
41
|
+
:func:`rasad.analyzer.classify`.
|
|
42
|
+
series
|
|
43
|
+
Output name to the divergence curve from
|
|
44
|
+
:func:`rasad.analyzer.divergence`.
|
|
45
|
+
runs
|
|
46
|
+
How many runs the report was built from.
|
|
47
|
+
thresholds
|
|
48
|
+
The classification cutoffs that were used, as a plain dict of
|
|
49
|
+
floats (see :data:`rasad.analyzer.THRESHOLDS`).
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
def __init__(
|
|
53
|
+
self,
|
|
54
|
+
scalars: dict[str, dict[str, Any]],
|
|
55
|
+
series: dict[str, list[float]],
|
|
56
|
+
runs: int,
|
|
57
|
+
thresholds: dict[str, float],
|
|
58
|
+
) -> None:
|
|
59
|
+
self.scalars = scalars
|
|
60
|
+
self.series = series
|
|
61
|
+
self.runs = runs
|
|
62
|
+
self.thresholds = thresholds
|
|
63
|
+
|
|
64
|
+
def summary(self) -> str:
|
|
65
|
+
"""Render the report as a plain-text table.
|
|
66
|
+
|
|
67
|
+
Returns
|
|
68
|
+
-------
|
|
69
|
+
str
|
|
70
|
+
A header with the run count, one row per scalar output
|
|
71
|
+
(name, mean, standard deviation, coefficient of variation,
|
|
72
|
+
p05-p95 interval, variability) and, when present, one
|
|
73
|
+
line per series showing how the divergence grew.
|
|
74
|
+
"""
|
|
75
|
+
lines = [f"Rasad report of {self.runs} runs"]
|
|
76
|
+
lines.append(
|
|
77
|
+
f"thresholds (a convention, not a rule): low < "
|
|
78
|
+
f"{_fmt_ratio(self.thresholds['low'])} <= moderate <= "
|
|
79
|
+
f"{_fmt_ratio(self.thresholds['moderate'])} < high"
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
if self.scalars:
|
|
83
|
+
lines.append("Scalar outputs:")
|
|
84
|
+
for name, stats in self.scalars.items():
|
|
85
|
+
lines.append(
|
|
86
|
+
f" {name}: mean {_fmt(stats['mean'])} | "
|
|
87
|
+
f"std {_fmt(stats['std'])} | "
|
|
88
|
+
f"cv {_fmt_ratio(stats['cv'])} | "
|
|
89
|
+
f"p05-p95 [{_fmt(stats['p05'])}, {_fmt(stats['p95'])}] | "
|
|
90
|
+
f"variability: {stats['variability']}"
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
if self.series:
|
|
94
|
+
lines.append("Series:")
|
|
95
|
+
for name, curve in self.series.items():
|
|
96
|
+
lines.append(
|
|
97
|
+
f" {name}: divergence from {_fmt(curve[0])} to "
|
|
98
|
+
f"{_fmt(curve[-1])} over {len(curve)} steps"
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
return "\n".join(lines)
|
|
102
|
+
|
|
103
|
+
def plot(self) -> go.Figure:
|
|
104
|
+
"""Render the report as a divergence chart.
|
|
105
|
+
|
|
106
|
+
Returns
|
|
107
|
+
-------
|
|
108
|
+
plotly.graph_objects.Figure
|
|
109
|
+
One line trace per series in :attr:`series`, x being the
|
|
110
|
+
time step and y the standard deviation between runs.
|
|
111
|
+
|
|
112
|
+
Raises
|
|
113
|
+
------
|
|
114
|
+
ValueError
|
|
115
|
+
When there is no time series to plot.
|
|
116
|
+
"""
|
|
117
|
+
if not self.series:
|
|
118
|
+
raise ValueError("no time series to plot")
|
|
119
|
+
|
|
120
|
+
fig = go.Figure()
|
|
121
|
+
for name, curve in self.series.items():
|
|
122
|
+
fig.add_trace(
|
|
123
|
+
go.Scatter(
|
|
124
|
+
x=list(range(len(curve))),
|
|
125
|
+
y=curve,
|
|
126
|
+
mode="lines",
|
|
127
|
+
name=name,
|
|
128
|
+
)
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
fig.update_layout(
|
|
132
|
+
title=f"Rasad report of {self.runs} runs",
|
|
133
|
+
xaxis_title="time step",
|
|
134
|
+
yaxis_title="standard deviation between runs",
|
|
135
|
+
)
|
|
136
|
+
return fig
|
rasad/runner.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Execute a plan of simulation runs and collect the raw outputs.
|
|
2
|
+
|
|
3
|
+
The runner turns a plan from :mod:`rasad.sampler` into results: it
|
|
4
|
+
calls the model once per plan entry, in plan order, and returns the
|
|
5
|
+
output dicts in that same order. Execution is deliberately sequential;
|
|
6
|
+
parallelism is explicitly out of scope for this version.
|
|
7
|
+
|
|
8
|
+
The model is validated up front so a bad model fails immediately
|
|
9
|
+
instead of after wasted work, and every run must return the same set of
|
|
10
|
+
output keys or the batch is rejected — a report built from shifting keys
|
|
11
|
+
could not be summarized.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from collections.abc import Callable
|
|
15
|
+
|
|
16
|
+
from rasad.adapter import validate_model
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def run_all(fn: Callable[..., dict], plan: list[tuple[dict, int]]) -> list[dict]:
|
|
20
|
+
"""Run a model once per plan entry and return the outputs in order.
|
|
21
|
+
|
|
22
|
+
Parameters
|
|
23
|
+
----------
|
|
24
|
+
fn
|
|
25
|
+
A model matching the simulation interface
|
|
26
|
+
``def run(params: dict, seed: int) -> dict``.
|
|
27
|
+
plan
|
|
28
|
+
The list of ``(params, seed)`` pairs produced by
|
|
29
|
+
:func:`rasad.sampler.make_plan`.
|
|
30
|
+
|
|
31
|
+
Returns
|
|
32
|
+
-------
|
|
33
|
+
list[dict]
|
|
34
|
+
One output dict per plan entry, in plan order.
|
|
35
|
+
|
|
36
|
+
Raises
|
|
37
|
+
------
|
|
38
|
+
TypeError
|
|
39
|
+
When ``fn`` is not a valid model, or when a run returns
|
|
40
|
+
something other than a dict.
|
|
41
|
+
ValueError
|
|
42
|
+
When runs do not all return the same set of output keys.
|
|
43
|
+
"""
|
|
44
|
+
validate_model(fn)
|
|
45
|
+
|
|
46
|
+
results: list[dict] = []
|
|
47
|
+
for params, seed in plan:
|
|
48
|
+
out = fn(params, seed)
|
|
49
|
+
if not isinstance(out, dict):
|
|
50
|
+
raise TypeError(
|
|
51
|
+
f"model must return a dict, got {type(out).__name__} from seed {seed}"
|
|
52
|
+
)
|
|
53
|
+
if results and out.keys() != results[0].keys():
|
|
54
|
+
raise ValueError(
|
|
55
|
+
"every run must return the same output keys, "
|
|
56
|
+
f"got {sorted(out)} after {sorted(results[0])}"
|
|
57
|
+
)
|
|
58
|
+
results.append(out)
|
|
59
|
+
|
|
60
|
+
return results
|
rasad/sampler.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Decide what the simulation runs will be.
|
|
2
|
+
|
|
3
|
+
In this version the parameters stay fixed across every run and only the
|
|
4
|
+
random seed varies, so the measurement isolates one question: how much
|
|
5
|
+
of the result is pure randomness? Each plan entry is a fresh deep copy
|
|
6
|
+
of the parameters paired with a seed, so later runs can never disturb
|
|
7
|
+
earlier ones or the caller's original dict.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import copy
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def make_plan(params: dict, runs: int, base_seed: int = 0) -> list[tuple[dict, int]]:
|
|
14
|
+
"""Build the list of (params, seed) pairs for a batch of runs.
|
|
15
|
+
|
|
16
|
+
Parameters
|
|
17
|
+
----------
|
|
18
|
+
params
|
|
19
|
+
The simulation parameters. The same value is used for every
|
|
20
|
+
run, and each run gets its own deep copy so mutations never
|
|
21
|
+
leak between runs.
|
|
22
|
+
runs
|
|
23
|
+
How many runs to plan. Must be at least 2 so the results can
|
|
24
|
+
be summarized.
|
|
25
|
+
base_seed
|
|
26
|
+
The seed of the first run; run ``i`` gets ``base_seed + i``.
|
|
27
|
+
|
|
28
|
+
Returns
|
|
29
|
+
-------
|
|
30
|
+
list[tuple[dict, int]]
|
|
31
|
+
One ``(params, seed)`` pair per run, in run order.
|
|
32
|
+
|
|
33
|
+
Raises
|
|
34
|
+
------
|
|
35
|
+
ValueError
|
|
36
|
+
When ``runs`` is below 2.
|
|
37
|
+
"""
|
|
38
|
+
if runs < 2:
|
|
39
|
+
raise ValueError(f"need at least 2 runs to summarize, got {runs}")
|
|
40
|
+
|
|
41
|
+
return [(copy.deepcopy(params), base_seed + i) for i in range(runs)]
|
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rasad-sim
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Measure how much of a stochastic simulation's result is a property of the model and how much is the random seed.
|
|
5
|
+
Author: Ahmed Aly
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ahmedaly0904-bit64/rasad
|
|
8
|
+
Project-URL: Source, https://github.com/ahmedaly0904-bit64/rasad
|
|
9
|
+
Project-URL: Issues, https://github.com/ahmedaly0904-bit64/rasad/issues
|
|
10
|
+
Keywords: simulation,reproducibility,agent-based-modeling,uncertainty-quantification,monte-carlo,sensitivity-analysis
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Mathematics
|
|
17
|
+
Classifier: Typing :: Typed
|
|
18
|
+
Requires-Python: >=3.12
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: numpy>=1.26
|
|
22
|
+
Requires-Dist: plotly>=5.9
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
25
|
+
Requires-Dist: pytest-cov>=5.0; extra == "dev"
|
|
26
|
+
Provides-Extra: examples
|
|
27
|
+
Requires-Dist: mesa>=3.5; extra == "examples"
|
|
28
|
+
Requires-Dist: networkx>=3.0; extra == "examples"
|
|
29
|
+
Requires-Dist: pandas>=2.0; extra == "examples"
|
|
30
|
+
Requires-Dist: simpy>=4.1; extra == "examples"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# Rasad · رَصَد
|
|
34
|
+
|
|
35
|
+
**Measure how much of your simulation's result is a property of the model, and how much is the random seed.**
|
|
36
|
+
|
|
37
|
+
Stochastic simulations are usually reported without error bars. A paper says *"the civilisation
|
|
38
|
+
collapsed in year 240"* and never answers two questions:
|
|
39
|
+
|
|
40
|
+
1. If the random seed changes, is it still year 240?
|
|
41
|
+
2. Is this a property of the model, or an accident of one run?
|
|
42
|
+
|
|
43
|
+
Rasad answers both by measurement.
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## Why this exists
|
|
48
|
+
|
|
49
|
+
This tool was not designed in the abstract. It was built because another project needed it.
|
|
50
|
+
|
|
51
|
+
**Omran** is a simulation of Ibn Khaldun's theory of *asabiyyah*. Ibn Khaldun (1332–1406) was
|
|
52
|
+
a historian who argued in the *Muqaddimah* that civilisations rise and fall on a single
|
|
53
|
+
force: **asabiyyah**, the cohesion that lets a group act as one. A group with strong
|
|
54
|
+
cohesion overtakes a settled, comfortable one; then comfort erodes its own cohesion over
|
|
55
|
+
generations, and it is overtaken in turn. The cycle repeats. He called the study of this
|
|
56
|
+
*ʿilm al-ʿumran* — the science of human social organisation — which is where the project
|
|
57
|
+
takes its name.
|
|
58
|
+
|
|
59
|
+
Omran models that computationally: nations on a grid, populations that grow and starve,
|
|
60
|
+
ideas that spread between neighbours like an infection, wars along contested borders. Run it
|
|
61
|
+
and you get numbers — a final population, a count of wars, a year the collapse happened.
|
|
62
|
+
|
|
63
|
+
The problem was that nobody knew whether those numbers meant anything.
|
|
64
|
+
|
|
65
|
+
Change the random seed and they change. Was a collapse in year 240 a property of the model,
|
|
66
|
+
or an accident of one run? There was no way to answer without measuring, and no tool that
|
|
67
|
+
measured it — the frameworks that do sensitivity analysis are built for engineering and
|
|
68
|
+
physics models, and the agent-based modelling field is repeatedly criticised for publishing
|
|
69
|
+
results without error bars.
|
|
70
|
+
|
|
71
|
+
So Rasad was written to answer that question, under one hard constraint: **it must not touch
|
|
72
|
+
Omran.** It observes from the outside and modifies nothing, the way an instrument measures a
|
|
73
|
+
specimen without altering it. That constraint shaped the architecture — and it is why Rasad
|
|
74
|
+
works on any simulation, not just the one it was written for.
|
|
75
|
+
|
|
76
|
+
What it found is in [`FINDINGS.md`](FINDINGS.md), and it was not what anyone expected.
|
|
77
|
+
|
|
78
|
+
---
|
|
79
|
+
|
|
80
|
+
## Install
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
pip install rasad-sim
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The distribution is `rasad-sim` because `rasad` collides with an existing name on
|
|
87
|
+
PyPI. The import is unaffected:
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
import rasad
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Python 3.12+ · numpy · plotly
|
|
94
|
+
|
|
95
|
+
## Use
|
|
96
|
+
|
|
97
|
+
Describe your simulation as one function:
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
def run(params: dict, seed: int) -> dict:
|
|
101
|
+
"""One run. A number is a final result; a list of numbers is a time series."""
|
|
102
|
+
...
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Then:
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
import rasad
|
|
109
|
+
|
|
110
|
+
report = rasad.measure(run, params={"growth": 0.03}, runs=100)
|
|
111
|
+
print(report.summary())
|
|
112
|
+
report.plot().write_html("divergence.html")
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
You get, for every output: mean, standard deviation, a 90% interval, and a variability class —
|
|
116
|
+
**low**, **moderate**, or **high** — plus a curve showing how far the runs drift
|
|
117
|
+
apart over time.
|
|
118
|
+
|
|
119
|
+
---
|
|
120
|
+
|
|
121
|
+
## What it found in practice
|
|
122
|
+
|
|
123
|
+
### A simulation that could not reproduce itself
|
|
124
|
+
|
|
125
|
+
Applied to [Omran](https://github.com/ahmedaly0904-bit64/Omran), an agent-based model of
|
|
126
|
+
Ibn Khaldun's theory of *asabiyyah*, **without modifying a line of it**:
|
|
127
|
+
|
|
128
|
+
```
|
|
129
|
+
final_total_population: mean 4,649.39 | cv 0.2841 | p05-p95 [2,704.75, 6,774.30] | high
|
|
130
|
+
survivors: mean 1.14 | cv 0.3059 | p05-p95 [1.00, 2.00] | high
|
|
131
|
+
total_wars: mean 13.44 | cv 0.3986 | p05-p95 [6.00, 22.10] | high
|
|
132
|
+
total_famines: mean 0.00 | cv 0.0000 | p05-p95 [0.00, 0.00] | low
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
No numeric output of the model had low variability. Worse, the measurement exposed something the author
|
|
136
|
+
did not know: **the same seed produced different results in different processes.**
|
|
137
|
+
|
|
138
|
+
Six runs with seed `1`, thirty simulated years:
|
|
139
|
+
|
|
140
|
+
```
|
|
141
|
+
1428 · 1428 · 1512 · 1426 · 1426 · 1512
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Comparing the population curve year by year located the split: the runs are **identical for
|
|
145
|
+
nineteen years**, then diverge at year twenty by **one individual** — which becomes hundreds
|
|
146
|
+
by year one hundred. That is error propagation, measured.
|
|
147
|
+
|
|
148
|
+
This was then confirmed with Rasad out of the path entirely: Omran's own `main.py` carries a
|
|
149
|
+
hard-coded `random.seed(42)`, and running that file directly eight times gave **four distinct
|
|
150
|
+
results**.
|
|
151
|
+
|
|
152
|
+
Full write-up: [`FINDINGS.md`](FINDINGS.md)
|
|
153
|
+
|
|
154
|
+
### Aggregates can be stable while their parts are noise
|
|
155
|
+
|
|
156
|
+
On a [SimPy](https://simpy.readthedocs.io) machine-shop simulation
|
|
157
|
+
([`examples/simpy_machine_shop.py`](examples/simpy_machine_shop.py)):
|
|
158
|
+
|
|
159
|
+
| Output | cv | Verdict |
|
|
160
|
+
|---|---|---|
|
|
161
|
+
| total parts produced | 0.014 | **low** |
|
|
162
|
+
| best machine · worst machine | 0.017 | **low** |
|
|
163
|
+
| **gap between best and worst** | **0.31** | **high** |
|
|
164
|
+
|
|
165
|
+
The shop's total output is stable. The gap between machines is pure noise. Anyone looking at
|
|
166
|
+
one run and saying *"machine 7 is underperforming, investigate it"* is chasing a random seed.
|
|
167
|
+
|
|
168
|
+
**Averages hide fragility.** That alone is a reason to measure what you publish.
|
|
169
|
+
|
|
170
|
+
---
|
|
171
|
+
|
|
172
|
+
## Verified against simulations it was not written for
|
|
173
|
+
|
|
174
|
+
| Framework | Models | Result |
|
|
175
|
+
|---|---|---|
|
|
176
|
+
| [Mesa](https://github.com/projectmesa/mesa) | Schelling, WolfSheep, Boltzmann | works; WolfSheep's sheep population has high variability (cv 2.95 — usually extinct, occasionally not) |
|
|
177
|
+
| [SimPy](https://simpy.readthedocs.io) | machine shop | works; see above |
|
|
178
|
+
| [EoN](https://epidemicsonnetworks.readthedocs.io) | SIR on a network | works; epidemic duration has high variability (7.5 → 14.7) |
|
|
179
|
+
| [Omran](https://github.com/ahmedaly0904-bit64/Omran) | asabiyyah model | works; see above |
|
|
180
|
+
|
|
181
|
+
Examples: [`examples/`](examples/)
|
|
182
|
+
|
|
183
|
+
A control worth stating: the SimPy and Mesa examples reproduce byte-identically across
|
|
184
|
+
separate processes. That establishes that Omran's non-reproducibility is a bug in Omran,
|
|
185
|
+
and that Rasad's own pipeline is deterministic.
|
|
186
|
+
|
|
187
|
+
---
|
|
188
|
+
|
|
189
|
+
## The class is a convention. The numbers are the result.
|
|
190
|
+
|
|
191
|
+
The default cutoffs — 0.05 and 0.20 — **are a choice, not a theory**. An output at cv 0.21
|
|
192
|
+
reads *high*; raise the cutoff to 0.25 and the same data reads *moderate*.
|
|
193
|
+
|
|
194
|
+
So every report prints the thresholds it used and labels them as a convention:
|
|
195
|
+
|
|
196
|
+
```
|
|
197
|
+
thresholds (a convention, not a rule): low < 0.0500 <= moderate <= 0.2000 < high
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
And they belong to the caller:
|
|
201
|
+
|
|
202
|
+
```python
|
|
203
|
+
rasad.measure(run, params={}, runs=100,
|
|
204
|
+
thresholds={"low": 0.01, "moderate": 0.05})
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
**The real result is the interval.** `p05-p95 [7.46, 14.74]` says the duration may double,
|
|
208
|
+
without needing a word on top of it.
|
|
209
|
+
|
|
210
|
+
---
|
|
211
|
+
|
|
212
|
+
## Limitations
|
|
213
|
+
|
|
214
|
+
- **Only the seed varies.** Parameters are held fixed, so *"which parameter drives the
|
|
215
|
+
result?"* is not answered yet. Sensitivity analysis is the next version.
|
|
216
|
+
- **Execution is sequential.** No parallelism.
|
|
217
|
+
- **One value per output name per run.** A model whose keys change between runs is rejected.
|
|
218
|
+
|
|
219
|
+
Accepted outputs: Python numbers, numpy scalars, 1-D numeric numpy arrays, lists and tuples.
|
|
220
|
+
**Booleans are always rejected** — alone or inside a list — because an average of ones and
|
|
221
|
+
zeros means nothing.
|
|
222
|
+
|
|
223
|
+
---
|
|
224
|
+
|
|
225
|
+
## Development
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
python -m venv .venv
|
|
229
|
+
.venv/bin/pip install -e ".[dev]"
|
|
230
|
+
.venv/bin/pytest tests -v
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
73 tests · 97% line coverage · 357 of 452 mutants killed · linted with ruff
|
|
234
|
+
|
|
235
|
+
The coverage figure is the weakest of the three. Mutation testing is what showed why:
|
|
236
|
+
`validate_thresholds` was fully covered and still accepted two equal cutoffs, because
|
|
237
|
+
no test passed the one input that separates `<` from `<=`.
|
|
238
|
+
|
|
239
|
+
The statistics are tested against models whose answers are known analytically: a constant
|
|
240
|
+
must give a standard deviation of exactly zero; a random walk's divergence must grow as the
|
|
241
|
+
square root of time. Those tests caught real bugs — including a collapsed axis that would
|
|
242
|
+
have produced plausible, meaningless numbers.
|
|
243
|
+
|
|
244
|
+
## Notes on the implementation
|
|
245
|
+
|
|
246
|
+
Three analyses of the code, in Arabic:
|
|
247
|
+
|
|
248
|
+
- [فئات المدخلات](docs/input-classes.md) — the five input classes `summarize()` actually
|
|
249
|
+
distinguishes, each with the test that covers it, and why branch order decides which class
|
|
250
|
+
a list falls into.
|
|
251
|
+
- [كم تشغيلة تكفي؟](docs/how-many-runs.md) — the standard error of the mean applied to Omran:
|
|
252
|
+
100 runs pin the mean to 2.8%, and reaching 1% costs 784.
|
|
253
|
+
- [حدود الوحدات](docs/module-boundaries.md) — the deletion test applied to each of Rasad's own
|
|
254
|
+
modules, separating splits forced by something that happened from splits made on a guess.
|
|
255
|
+
|
|
256
|
+
---
|
|
257
|
+
|
|
258
|
+
## بالعربية
|
|
259
|
+
|
|
260
|
+
**رَصَد** أداة تقيس صلابة نتائج المحاكاة: أيُّ المخرجات خاصيةٌ في النموذج، وأيُّها أثرٌ للبذرة العشوائية.
|
|
261
|
+
|
|
262
|
+
تُنشر نتائج المحاكاة العشوائية غالبًا بلا حدود خطأ. يُقال «انهارت الحضارة في السنة ٢٤٠» دون
|
|
263
|
+
الإجابة على سؤالين: هل يظل الرقم ٢٤٠ لو تغيّرت البذرة؟ وهل هذه خاصية في النموذج أم صدفة في
|
|
264
|
+
تشغيلة واحدة؟ يجيب رَصَد عنهما بالقياس لا بالتقدير.
|
|
265
|
+
|
|
266
|
+
يوصّف المستخدم محاكاته بدالةٍ واحدة تستقبل المعاملات والبذرة وترجّع قاموس مخرجات. يشغّلها
|
|
267
|
+
رَصَد مرارًا ببذورٍ مختلفة، ثم يعرض لكل مخرَج متوسطه وانحرافه ومدى تسعين بالمئة وتصنيفًا لتغايره —
|
|
268
|
+
**low** أو **moderate** أو **high** — مع منحنى يبيّن اتساع التباعد بين التشغيلات عبر الزمن.
|
|
269
|
+
|
|
270
|
+
طُبِّق على أربعة مشاريع لم يُكتب لأجلها، فكشف في أحدها — محاكاة لنظرية العصبية عند ابن خلدون —
|
|
271
|
+
أنها **لا تعيد إنتاج نتائجها بالبذرة نفسها**: تشغيلتان متطابقتان تفترقان عند السنة العشرين
|
|
272
|
+
بفارق فردٍ واحد، يصير مئاتٍ بحلول السنة المئة. وهذا انتشار الخطأ في صورته المقيسة.
|
|
273
|
+
|
|
274
|
+
### لماذا كُتب
|
|
275
|
+
|
|
276
|
+
لم تُصمَّم هذه الأداة في الفراغ، بل كُتبت لأن مشروعًا آخر احتاجها.
|
|
277
|
+
|
|
278
|
+
**عُمران** محاكاةٌ لنظرية العصبية عند ابن خلدون (١٣٣٢–١٤٠٦)، الذي رأى في *المقدمة* أن الحضارات
|
|
279
|
+
تنهض وتسقط بقوةٍ واحدة: **العصبية**، أي التماسك الذي يجعل الجماعة تفعل كأنها واحد. جماعةٌ
|
|
280
|
+
عصبيتها قوية تغلب جماعةً مستقرةً مترفة، ثم يُفسد الترف عصبيتها هي عبر الأجيال، فتُغلَب بدورها.
|
|
281
|
+
وسمّى ابن خلدون دراسة ذلك **علم العمران**، ومنه أخذ المشروع اسمه.
|
|
282
|
+
|
|
283
|
+
يحاكي عُمران هذا حاسوبيًّا: دولٌ على شبكة، وسكانٌ ينمون ويجوعون، وأفكارٌ تنتقل بين الجيران
|
|
284
|
+
كالعدوى، وحروبٌ على الحدود المتنازعة. تشغّله فتخرج لك أرقام — سكانٌ في النهاية، وعدد حروب،
|
|
285
|
+
وسنةٌ وقع فيها الانهيار.
|
|
286
|
+
|
|
287
|
+
والمشكلة أن أحدًا لم يكن يعرف: هل لهذه الأرقام معنى؟ غيّر البذرة العشوائية تتغيّر. فهل الانهيار
|
|
288
|
+
في السنة ٢٤٠ خاصيةٌ في النموذج أم صدفةُ تشغيلةٍ واحدة؟ لا سبيل إلى الجواب إلا بالقياس، ولم تكن
|
|
289
|
+
هناك أداةٌ تقيسه.
|
|
290
|
+
|
|
291
|
+
فكُتب رَصَد لهذا السؤال، بقيدٍ واحدٍ صارم: **ألّا يمسّ عُمران**. يراقبه من خارجه ولا يعدّل فيه
|
|
292
|
+
حرفًا، كما يقيس المِجهر عيّنةً دون أن يغيّرها. وهذا القيد هو الذي شكّل بنية الأداة — وهو سبب
|
|
293
|
+
عملها على أي محاكاة، لا على التي كُتبت لأجلها وحدها.
|
|
294
|
+
|
|
295
|
+
### التصنيف
|
|
296
|
+
|
|
297
|
+
وحدود التصنيف الافتراضية اصطلاحٌ لا قاعدة، ولذلك يعلنها كل تقرير ويتركها بيد المستخدم.
|
|
298
|
+
**النتيجة الحقيقية هي المدى**، لا التصنيف الذي يعلوه.
|
|
299
|
+
|
|
300
|
+
---
|
|
301
|
+
|
|
302
|
+
## How this was built — full disclosure
|
|
303
|
+
|
|
304
|
+
Most of the code here was written by AI models under explicit delegation and human review.
|
|
305
|
+
|
|
306
|
+
| Stage | Owner |
|
|
307
|
+
|---|---|
|
|
308
|
+
| Specification and architecture | Ahmed, in dialogue with Claude (Opus 5) |
|
|
309
|
+
| **Test authoring** | Written into each task brief **before** implementation; the implementer was forbidden from altering a character |
|
|
310
|
+
| Implementation | DeepSeek V4 Flash, via `opencode` + AgentRouter — seven tasks |
|
|
311
|
+
| Review gates | Claude (Opus 5): every diff read, tests run independently of the implementer's claim |
|
|
312
|
+
| Decisions and merges | Ahmed — every commit after review |
|
|
313
|
+
|
|
314
|
+
**The ordering is what matters, not the tooling: the tests came first and were the
|
|
315
|
+
specification.** The model was never asked to write code and then write the thing that proves
|
|
316
|
+
it correct. It was given a written contract and held to it.
|
|
317
|
+
|
|
318
|
+
What the gates actually caught: a dead condition in a type check, a deprecated import, a
|
|
319
|
+
missing return annotation, unreadable number formatting. **No logic error got through** —
|
|
320
|
+
credit to the tests, not to the model.
|
|
321
|
+
|
|
322
|
+
And what none of them caught: an output that was constantly zero was classified as *high* variability
|
|
323
|
+
when it was the most stable number in the report. Neither the reference models nor the review
|
|
324
|
+
found it — **the real data did, on the first run against Omran.**
|
|
325
|
+
|
|
326
|
+
That is the boundary. Tests prove the arithmetic is right; only real data reveals the case
|
|
327
|
+
nobody thought to write a test for.
|
|
328
|
+
|
|
329
|
+
---
|
|
330
|
+
|
|
331
|
+
## License
|
|
332
|
+
|
|
333
|
+
MIT
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
rasad/__init__.py,sha256=FlD_koEkFcd-ghAI85SQB3HdIvvoBEKWjZEUPHKnsHs,2687
|
|
2
|
+
rasad/adapter.py,sha256=V1OWkEZRuYaursWF8x9ovfwoWRiadavxXpMTjC_uV-g,5084
|
|
3
|
+
rasad/analyzer.py,sha256=SBEuN8zphRf_-5j0odVfKP1UAp0qvaq23DuGl6H7UNA,5858
|
|
4
|
+
rasad/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
rasad/report.py,sha256=T8-DP5pHkWkWv_Qsb4K5dJ1w1qRSJ-R04KoGFGFMGYI,4275
|
|
6
|
+
rasad/runner.py,sha256=6zaygTJkreNZKtXHjeOWmVUtgYQyvtC_w8h_DY2B5BI,1935
|
|
7
|
+
rasad/sampler.py,sha256=Oq8i741yagWPFLriCPn4uPfaSYHJdgfOw2-8goyhMqA,1291
|
|
8
|
+
rasad/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
rasad/adapters/omran.py,sha256=erEkT6UiL0ewGHTMs9Bt-KZqLgSQS2fkpohZda3C7vs,2526
|
|
10
|
+
rasad_sim-0.1.0.dist-info/licenses/LICENSE,sha256=afpSMmwbfdEODsOT84sfwc4LdduoQD1FQYc8sYa0gLc,1066
|
|
11
|
+
rasad_sim-0.1.0.dist-info/METADATA,sha256=1p6S9C1nMDEHc5vk58SqdSn6NIGS7Q_kgQhA1ya4heo,15332
|
|
12
|
+
rasad_sim-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
13
|
+
rasad_sim-0.1.0.dist-info/top_level.txt,sha256=E7IVloUBu5D9-POCvBj2uRoI9NeotDCcMaTlAjzztek,6
|
|
14
|
+
rasad_sim-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ahmed Aly
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
rasad
|