simloom 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {simloom-0.1.0 → simloom-0.2.0}/.gitignore +6 -0
- simloom-0.2.0/CHANGELOG.md +185 -0
- {simloom-0.1.0 → simloom-0.2.0}/PKG-INFO +108 -38
- {simloom-0.1.0 → simloom-0.2.0}/README.md +103 -34
- {simloom-0.1.0 → simloom-0.2.0}/pyproject.toml +10 -2
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/__init__.py +40 -2
- simloom-0.2.0/src/simloom/_boundary.py +126 -0
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/_buggify.py +27 -0
- simloom-0.2.0/src/simloom/_cli.py +164 -0
- simloom-0.2.0/src/simloom/_consistency.py +206 -0
- simloom-0.2.0/src/simloom/_errors.py +115 -0
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/_explore.py +16 -2
- simloom-0.2.0/src/simloom/_fingerprint.py +94 -0
- simloom-0.2.0/src/simloom/_hashseed.py +49 -0
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/_loop.py +150 -9
- simloom-0.2.0/src/simloom/_monitors.py +219 -0
- simloom-0.2.0/src/simloom/_net.py +1060 -0
- simloom-0.2.0/src/simloom/_patches.py +106 -0
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/_pytest_plugin.py +22 -3
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/_run.py +196 -6
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/_sched.py +21 -1
- simloom-0.2.0/src/simloom/_services.py +175 -0
- simloom-0.2.0/src/simloom/_soak.py +154 -0
- simloom-0.2.0/src/simloom/_systematic.py +155 -0
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/_testing.py +81 -0
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/_version.py +1 -1
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/_world.py +101 -0
- simloom-0.1.0/CHANGELOG.md +0 -57
- simloom-0.1.0/src/simloom/_errors.py +0 -54
- simloom-0.1.0/src/simloom/_net.py +0 -559
- {simloom-0.1.0 → simloom-0.2.0}/LICENSE +0 -0
- {simloom-0.1.0 → simloom-0.2.0}/examples/README.md +0 -0
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/_context.py +0 -0
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/_eventlog.py +0 -0
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/_shrink.py +0 -0
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/_tape.py +0 -0
- {simloom-0.1.0 → simloom-0.2.0}/src/simloom/py.typed +0 -0
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.2.0] - 2026-06-25
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- **Phase L — stand-in services: sim-redis.** `world.run_service(SimRedis(),
|
|
14
|
+
host=, port=)` starts an in-sim RESP server an *unmodified* `redis.asyncio.Redis`
|
|
15
|
+
client speaks to — SET/GET/DEL/EXISTS/INCR(BY)/DECR(BY) and the
|
|
16
|
+
WATCH/MULTI/EXEC optimistic-locking transaction (a watched key changing aborts
|
|
17
|
+
the EXEC). It runs as an ordinary in-sim server, so `world.net` faults apply to
|
|
18
|
+
the wire: a planted connection reset surfaces to the client as a deterministic,
|
|
19
|
+
replayable `ConnectionError`. The roadmap's final phase — simloom now tests your
|
|
20
|
+
logic against a fault-injected dependency with nothing to stand up.
|
|
21
|
+
- **Phase K — consistency checking (finds *wrong answers*, not just crashes).**
|
|
22
|
+
An Elle-style list-append serializability checker: `world.history` records
|
|
23
|
+
append/read transactions, `world.check_serializable()` / `assert_serializable()`
|
|
24
|
+
recover each key's version order from the reads, build the ww/wr/rw dependency
|
|
25
|
+
graph, and report any cycle (the shortest, so a lost update is a minimal
|
|
26
|
+
two-step rw cycle) as a `ConsistencyViolation`. A store with a non-atomic
|
|
27
|
+
read-modify-write is caught under the schedules that lose an update; a locked
|
|
28
|
+
store always passes; the violation shrinks to the minimal cycle.
|
|
29
|
+
- **Phase J (2,3/3) — the `simloom` trace/diff CLI + `observe`.** A causal log
|
|
30
|
+
(`run(..., causal=True)`, `result.log.write_to(path)`) is consumed by:
|
|
31
|
+
`simloom trace LOG --step N` (reconstruct state at any step with its causal
|
|
32
|
+
stack back to a root), `--grep REGEX` / `--changed NAME` (omniscient queries —
|
|
33
|
+
the latter returns exactly the steps where an `observe`-d value changed), and
|
|
34
|
+
`simloom diff A B` (first diverging event between two universes, with a cause
|
|
35
|
+
classification). `simloom.observe(name, value)` is a no-op outside causal mode
|
|
36
|
+
(zero digest impact).
|
|
37
|
+
- **Phase J (1/3) — causal event log v2.** Opt-in `causal=True` annotates every
|
|
38
|
+
`step` with *why it woke*: `woke_by` (the earlier step that scheduled the
|
|
39
|
+
callback now running — the happens-before edge) and `via` (`timer` vs `soon`).
|
|
40
|
+
Off by default, so the hot path and the digest are unchanged. The edges
|
|
41
|
+
satisfy an independent happens-before oracle: causes precede effects, every
|
|
42
|
+
step traces to a root, and an immediate (`soon`) wakeup runs at the same
|
|
43
|
+
virtual instant as its cause while a `timer` wakeup runs at or after it.
|
|
44
|
+
- **Phase I (5/5) — TLS in the simulation.** `create_connection(ssl=)` /
|
|
45
|
+
`create_server(ssl=)` now wrap each protocol in asyncio's own memory-BIO
|
|
46
|
+
`SSLProtocol` over `SimTransport`, so the handshake runs in-process over the
|
|
47
|
+
simulated wire. The run is deterministic and replays byte-for-byte even though
|
|
48
|
+
OpenSSL's RNG is unseeded (the event log records scheduling, not ciphertext).
|
|
49
|
+
`SimTransport` gained `BufferedProtocol` support (`get_buffer`/`buffer_updated`)
|
|
50
|
+
to drive SSL, and `loop.sock_connect` + `create_connection(sock=)` are
|
|
51
|
+
simulated for happy-eyeballs clients. **Gate met:** an unmodified aiohttp HTTPS
|
|
52
|
+
client hits an unmodified aiohttp HTTPS server in-sim over a pinned cert,
|
|
53
|
+
replayable byte-for-byte (status 200, deterministic on 3.12/3.13/3.14).
|
|
54
|
+
- **Phase I (4/5) — per-link / per-node shaping.** `world.net.set_link_latency`
|
|
55
|
+
and `set_link_loss` shape a single directional link (real links are
|
|
56
|
+
asymmetric); `clog_node_out` / `clog_node_in` shape a node's whole egress /
|
|
57
|
+
ingress. Precedence: explicit link > node egress > node ingress > global, and
|
|
58
|
+
the no-override path is byte-identical (the determinism torture is unchanged).
|
|
59
|
+
- **Phase I (3/5) — datagram (UDP) transport.** `loop.create_datagram_endpoint`
|
|
60
|
+
is now simulated: `SimDatagramTransport` delivers in order and exactly once by
|
|
61
|
+
default, but `world.net.set_datagram_loss/duplication/reorder(percent)` inject
|
|
62
|
+
the real UDP faults — packets genuinely dropped, duplicated, or overtaken
|
|
63
|
+
(every fate a tape draw, so it replays). A naive client that assumes in-order
|
|
64
|
+
exactly-once delivery is found buggy under faults; a correct one survives.
|
|
65
|
+
- **Phase I (2/5) — world-on-by-default.** A `World` is now attached for every
|
|
66
|
+
run (network simulated) regardless of whether `main` takes a `world`
|
|
67
|
+
parameter — killing the arity trap where `async def main():` + a network call
|
|
68
|
+
produced a confusing escape. Pass `world=False` to restore escape-on-real-
|
|
69
|
+
network (with an accurate message, not the stale "Phase B" one). `SimServer`
|
|
70
|
+
gained `async with` / `serve_forever` / blocking `wait_closed` for fidelity.
|
|
71
|
+
- **Phase I (1/5) — `connected_pair`.** `world.connected_pair(client_factory,
|
|
72
|
+
server_factory)` drives any two `asyncio.Protocol`s against each other over a
|
|
73
|
+
two-sided simulated connection — no listener, no hand-written stub transport.
|
|
74
|
+
Both `connection_made` callbacks run before it returns, and the pair carries
|
|
75
|
+
`world.net` faults (latency/loss/partition/reset) like a real connection.
|
|
76
|
+
`create_connection` is refactored onto it (behaviour byte-identical).
|
|
77
|
+
- **Systematic deep search — a stateless model checker over the choice tape.**
|
|
78
|
+
`simloom.explore_systematic(main, max_delays=2)` enumerates every distinct
|
|
79
|
+
interleaving within a *delay bound* exactly once (a mixed-radix odometer over
|
|
80
|
+
the tape, reusing FALLBACK replay). Two things random walk and PCT cannot do:
|
|
81
|
+
it finds a bug needing a specific non-default interleaving *deterministically*
|
|
82
|
+
(e.g. a check-then-set race the default schedule never hits), and it *proves
|
|
83
|
+
correctness* — `result.proven_correct` is True when it exhausts the space with
|
|
84
|
+
no failure. Delay bounding (Emmi/Qadeer/Rakamarić) keeps it tractable without
|
|
85
|
+
the happens-before tracking full partial-order reduction needs; found failures
|
|
86
|
+
replay and shrink like any tape. Available as `@simloom.test(systematic=True,
|
|
87
|
+
max_delays=2)` — bounded verification as an ordinary pytest test: the test
|
|
88
|
+
passes as a proof up to the bound, or fails deterministically with a shrunk,
|
|
89
|
+
replayable witness interleaving.
|
|
90
|
+
- **Phase H (3/3) — soak testing.** `simloom.soak(main, count=, shards=, shard=,
|
|
91
|
+
checkpoint=)` runs continuous, shardable, resumable exploration. Sharding is by
|
|
92
|
+
stride, so shards are disjoint and complete by construction; the checkpoint
|
|
93
|
+
records each shard's cursor (atomic write), so a killed soak resumes from its
|
|
94
|
+
next unrun seed — none skipped, none repeated.
|
|
95
|
+
- **Phase H (2/4) — interleaving fingerprints + regression corpus.**
|
|
96
|
+
`interleaving_edges`/`fingerprint` summarise a run by its `(prev_callback,
|
|
97
|
+
callback)` step edges; `InterleavingCorpus` accumulates the global edge set
|
|
98
|
+
and keeps the tapes that first reached new edges. This is the feedback
|
|
99
|
+
substrate for coverage-guided search: sparse novelty (most runs add no edges)
|
|
100
|
+
is the steering signal.
|
|
101
|
+
- **Phase H (1/4) — PCT auto-horizon.** `scheduler="pct:auto"` measures the
|
|
102
|
+
scheduling horizon `k` from a probe run instead of the broken `k=4096`
|
|
103
|
+
default (whose change points land past a small test's end and never fire,
|
|
104
|
+
degrading PCT to a fixed priority schedule). pct:auto finds the
|
|
105
|
+
ordering/starvation bugs the default misses; the resolved descriptor is
|
|
106
|
+
concrete and replayable.
|
|
107
|
+
- **Phase G (1/2) — the determinism boundary.** Opt-in `virtual_time=True`
|
|
108
|
+
redirects `time.time`/`monotonic`/`perf_counter` (+`_ns`) to the virtual
|
|
109
|
+
clock; `seed_randomness=True` seeds the global `random`, `os.urandom`,
|
|
110
|
+
`secrets`/`SystemRandom`, and `uuid.uuid4` from one tape draw. Both default
|
|
111
|
+
ON under `@simloom.test` and are always restored (even on a crashed run).
|
|
112
|
+
An unmodified asyncio library that reads the clock and rolls randomness now
|
|
113
|
+
replays byte-for-byte.
|
|
114
|
+
- **Phase G (2/2) — per-test determinism self-check.** `run(check_determinism=
|
|
115
|
+
True)` / `@simloom.test(check_determinism=True)` / `pytest --simloom-check-
|
|
116
|
+
determinism` run a seed twice and raise `SimloomNondeterminismError` locating
|
|
117
|
+
the first diverging event if the two universes differ — catching
|
|
118
|
+
nondeterminism the tape cannot control (identity-ordered iteration, a stray
|
|
119
|
+
real clock/RNG, threads doing real work).
|
|
120
|
+
- **Phase G (3/3) — boundary registry + PYTHONHASHSEED auto-pin.**
|
|
121
|
+
`simloom.boundary()` is a machine-readable table of every real-world API and
|
|
122
|
+
what simloom does there (detected / simulated / patched / documented); a test
|
|
123
|
+
cross-checks it against the actual escape sites so the honesty contract cannot
|
|
124
|
+
drift. `simloom.pin_hashseed()` (and `pytest --simloom-pin-hashseed`) re-execs
|
|
125
|
+
with `PYTHONHASHSEED=0` if unpinned, so cross-process seed/tape replay is sound.
|
|
126
|
+
- **Phase F — property monitors (oracles).** `world.always` / `world.eventually`
|
|
127
|
+
/ `world.leads_to` (and module-level `simloom.always`/`eventually`/`leads_to`)
|
|
128
|
+
assert safety and liveness properties over the deterministic step sequence;
|
|
129
|
+
`world.assert_converged` checks replica equality. A new livelock oracle
|
|
130
|
+
(`SimLivelockError`) catches busy-but-stuck spins the deadlock oracle cannot
|
|
131
|
+
see. Violations surface as `InvariantViolation` and are found/shrunk/replayed
|
|
132
|
+
like any failure. Passing monitors are zero-perturbation (byte-identical
|
|
133
|
+
digest); liveness deadlines fire at exactly their virtual time and never mask
|
|
134
|
+
a `SimDeadlockError`. This turns simloom from "finds crashes" into "finds
|
|
135
|
+
wrong and stuck behaviour."
|
|
136
|
+
|
|
137
|
+
## [0.1.0] - 2026-06-13
|
|
138
|
+
|
|
139
|
+
First public release. The deterministic core, simulated world, fault matrix,
|
|
140
|
+
explorer, shrinker, and pytest plugin are all present and exercised by a
|
|
141
|
+
10,000-seed determinism torture in CI.
|
|
142
|
+
|
|
143
|
+
### Added
|
|
144
|
+
- Project scaffold: `src/` layout, strict mypy, ruff, pytest + hypothesis, CI.
|
|
145
|
+
- Phase 0 feasibility spikes under `spikes/`.
|
|
146
|
+
- **Phase A core**: the choice tape (`Tape`, labeled bounded draws, strict/fallback
|
|
147
|
+
replay, versioned JSON serialization), the deterministic event loop (`SimLoop`)
|
|
148
|
+
with virtual time, loop-owned task naming, escape detection
|
|
149
|
+
(`EscapedSimulationError`), deadlock reporting (`SimDeadlockError`), controlled
|
|
150
|
+
GC, and deterministic teardown; `run()`/`replay()` entry points returning
|
|
151
|
+
`RunResult` with a sha256 universe digest; event log format v1
|
|
152
|
+
(`docs/event-log.md`); honesty doc (`docs/determinism.md`); 10,000-seed
|
|
153
|
+
determinism-torture CI job.
|
|
154
|
+
- **Phase B world**: simulated network behind the loop primitives (in-memory
|
|
155
|
+
transports with FIFO stream guarantees, `SimServer`, strict `SimDNS`, tape-driven
|
|
156
|
+
latency and loss-as-retransmission-delay), `World`/`Host` with power-cut crash
|
|
157
|
+
semantics and entry-factory restart, `SimDisk` with honest fsync. Gate: unmodified
|
|
158
|
+
aiohttp + httpx exchanging requests over the lossy simulated network, replayable
|
|
159
|
+
byte-for-byte (`tests/test_world_http.py`).
|
|
160
|
+
- **Phase C faults + buggify**: partition/heal (hold-and-release, streams never
|
|
161
|
+
corrupt), asymmetric block, connection-reset injection, partition-aware connects,
|
|
162
|
+
torn/lost/flushed unsynced writes on crash; `simloom.sometimes`/`draw`/`reached`
|
|
163
|
+
with `RunResult.coverage`. Gate: the toy Raft torture suite
|
|
164
|
+
(`tests/test_raft_demo.py`) catches a planted double-vote bug via exploration and
|
|
165
|
+
replays it exactly; the correct implementation survives.
|
|
166
|
+
- **Phase D explorer + shrinker**: `simloom.explore` (random walk or PCT scheduling,
|
|
167
|
+
serial or multiprocess, failure corpus + coverage union), `simloom.shrink`
|
|
168
|
+
(deviations-first greedy minimization with re-recorded candidates and zero-refill
|
|
169
|
+
fallback; human-readable `describe()`), `PCT`/`RandomWalk` strategies with
|
|
170
|
+
replay-matched recording. Gate: the benchmark bug zoo measures find rates — PCT
|
|
171
|
+
finds the starvation-class bug a random walk never does (0 vs 151 of 400 seeds),
|
|
172
|
+
and random dominates shallow races; both results asserted in CI.
|
|
173
|
+
- **Phase E pytest plugin**: `@simloom.test(runs=N)` turns an async test into a
|
|
174
|
+
seed-exploring simulation test — failures are shrunk, written to `.sim/failures/`
|
|
175
|
+
as replayable artifacts, and reported with seed + re-run command + minimal
|
|
176
|
+
schedule. Options: `--simloom-seed`, `--simloom-runs`, `--simloom-tape`,
|
|
177
|
+
`--simloom-no-shrink`; `require_coverage` for per-test sometimes-assertions.
|
|
178
|
+
- **Examples + demo #4**: `examples/toy_raft.py` (runnable) and
|
|
179
|
+
`examples/bpo42130.py` — the historical `asyncio.wait_for` cancellation-swallowing
|
|
180
|
+
race (bpo-42130) reproduced deterministically from a seed; the modern stdlib
|
|
181
|
+
survives the identical torture. Launch-post draft in `docs/launch-post.md`.
|
|
182
|
+
|
|
183
|
+
[Unreleased]: https://github.com/mandipadk/simloom/compare/v0.2.0...HEAD
|
|
184
|
+
[0.2.0]: https://github.com/mandipadk/simloom/compare/v0.1.0...v0.2.0
|
|
185
|
+
[0.1.0]: https://github.com/mandipadk/simloom/releases/tag/v0.1.0
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: simloom
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary: Deterministic simulation testing for asyncio:
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Deterministic simulation testing for asyncio: explore every schedule, inject faults, assert properties, prove correctness, and replay any failure from a seed.
|
|
5
5
|
Project-URL: Homepage, https://github.com/mandipadk/simloom
|
|
6
6
|
Project-URL: Repository, https://github.com/mandipadk/simloom
|
|
7
7
|
Project-URL: Documentation, https://github.com/mandipadk/simloom/tree/main/docs
|
|
@@ -10,8 +10,8 @@ Project-URL: Issues, https://github.com/mandipadk/simloom/issues
|
|
|
10
10
|
Author: Mandip Adhikari
|
|
11
11
|
License-Expression: Apache-2.0
|
|
12
12
|
License-File: LICENSE
|
|
13
|
-
Keywords: asyncio,concurrency,deterministic,fault-injection,simulation,testing
|
|
14
|
-
Classifier: Development Status ::
|
|
13
|
+
Keywords: asyncio,chaos-engineering,concurrency,deterministic,fault-injection,model-checking,property-testing,serializability,simulation,testing
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
15
|
Classifier: Framework :: AsyncIO
|
|
16
16
|
Classifier: Framework :: Pytest
|
|
17
17
|
Classifier: Intended Audience :: Developers
|
|
@@ -30,6 +30,7 @@ Requires-Dist: ruff>=0.9; extra == 'dev'
|
|
|
30
30
|
Provides-Extra: spikes
|
|
31
31
|
Requires-Dist: aiohttp>=3.11; extra == 'spikes'
|
|
32
32
|
Requires-Dist: httpx>=0.28; extra == 'spikes'
|
|
33
|
+
Requires-Dist: redis>=5.0; extra == 'spikes'
|
|
33
34
|
Description-Content-Type: text/markdown
|
|
34
35
|
|
|
35
36
|
<h1 align="center">simloom</h1>
|
|
@@ -56,9 +57,13 @@ state of the art.
|
|
|
56
57
|
**simloom runs your unmodified asyncio program inside a fully simulated world** — a
|
|
57
58
|
seeded scheduler that owns every interleaving, a virtual clock, an in-memory network
|
|
58
59
|
with injectable latency, loss, partitions, and crashes — and explores thousands of
|
|
59
|
-
hostile schedules looking for the one that breaks your invariants.
|
|
60
|
-
it
|
|
61
|
-
|
|
60
|
+
hostile schedules looking for the one that breaks your invariants. It doesn't just find
|
|
61
|
+
crashes: it asserts **safety and liveness** properties, **proves** correctness by
|
|
62
|
+
exhaustive systematic search, and catches **wrong answers** (a store that returns a stale
|
|
63
|
+
or impossible read) with serializability checking. When it finds a problem, it hands you a
|
|
64
|
+
**seed** that replays the failure byte-for-byte, forever, **shrinks** it to the minimal
|
|
65
|
+
schedule that still triggers the bug, and lets you **walk the causal trace** of what woke
|
|
66
|
+
what.
|
|
62
67
|
|
|
63
68
|
```text
|
|
64
69
|
FAILED test_lease_exclusivity — simloom found a failing universe
|
|
@@ -126,6 +131,37 @@ async def test_leader_election(world):
|
|
|
126
131
|
await world.until(lambda: exactly_one_leader(nodes), timeout=120)
|
|
127
132
|
```
|
|
128
133
|
|
|
134
|
+
### Beyond crashes
|
|
135
|
+
|
|
136
|
+
Assert what *should* hold, prove it can't be violated, and catch wrong answers:
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
@simloom.test(runs=5000)
|
|
140
|
+
async def test_lease_safety(world):
|
|
141
|
+
cluster = start_cluster(world)
|
|
142
|
+
# safety: never two leaders. liveness: one eventually emerges.
|
|
143
|
+
world.always("≤1 leader", lambda: sum(n.is_leader for n in cluster) <= 1)
|
|
144
|
+
world.eventually("a leader", lambda: any(n.is_leader for n in cluster), within=120)
|
|
145
|
+
await world.sleep(300)
|
|
146
|
+
|
|
147
|
+
@simloom.test(systematic=True, max_delays=3) # exhaustive, not sampled
|
|
148
|
+
async def test_critical_section():
|
|
149
|
+
... # passes ⇒ a bounded PROOF of correctness
|
|
150
|
+
|
|
151
|
+
@simloom.test(runs=5000)
|
|
152
|
+
async def test_store_is_serializable(world):
|
|
153
|
+
await run_transactions(world) # records ops into world.history
|
|
154
|
+
world.assert_serializable() # finds lost updates / write skew, with the cycle
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
And once it finds something, replay and walk it:
|
|
158
|
+
|
|
159
|
+
```sh
|
|
160
|
+
pytest --simloom-seed=17 -k lease # replay the exact failing universe
|
|
161
|
+
simloom trace failure.jsonl --step 42 # reconstruct state + the happens-before stack
|
|
162
|
+
simloom diff run_a.jsonl run_b.jsonl # first divergence between two universes
|
|
163
|
+
```
|
|
164
|
+
|
|
129
165
|
## Why this didn't exist before
|
|
130
166
|
|
|
131
167
|
Rust has [`loom`](https://github.com/tokio-rs/loom),
|
|
@@ -140,32 +176,62 @@ had **nothing**.
|
|
|
140
176
|
And asyncio is *structurally perfect* for it: every interleaving decision happens at an
|
|
141
177
|
`await`, under a **replaceable event loop**. simloom swaps in a deterministic one — no
|
|
142
178
|
forked interpreter, no hypervisor, no recompilation. Because the ecosystem (aiohttp,
|
|
143
|
-
httpx, the streams API) bottoms out in loop primitives, **real, unmodified
|
|
144
|
-
inside the simulation**. Our CI runs
|
|
145
|
-
|
|
179
|
+
httpx, redis, the streams API) bottoms out in loop primitives, **real, unmodified
|
|
180
|
+
libraries run inside the simulation**. Our CI runs an unmodified **aiohttp HTTPS** server
|
|
181
|
+
against an unmodified aiohttp client over a memory-BIO TLS handshake, and an unmodified
|
|
182
|
+
**`redis.asyncio.Redis`** against an in-sim Redis — all over the simulated network, with
|
|
183
|
+
faults injected, replayable byte-for-byte from a seed.
|
|
146
184
|
|
|
147
185
|
## What you get
|
|
148
186
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
-
|
|
152
|
-
|
|
153
|
-
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
-
|
|
160
|
-
|
|
161
|
-
|
|
187
|
+
**Run it.** Your real asyncio code, unchanged, inside the simulation.
|
|
188
|
+
|
|
189
|
+
- **🎲 Seeded scheduling** — a single *choice tape* (the Hypothesis trick, applied to
|
|
190
|
+
schedules) drives every decision. One seed → one exact universe.
|
|
191
|
+
- **⏱ Virtual time** — an hour of simulated `asyncio.sleep` traffic runs in milliseconds;
|
|
192
|
+
opt-in patching makes `time.time()` and the stdlib RNG tape-driven too, so an unmodified
|
|
193
|
+
library that timestamps or rolls a random timeout replays byte-for-byte.
|
|
194
|
+
- **🌐 A simulated network** — in-memory streams (loss modeled as TCP retransmit delay —
|
|
195
|
+
bytes never corrupt), **UDP datagrams** with real loss/reorder/duplication,
|
|
196
|
+
partitions, asymmetric per-link latency/loss shaping, resets, and **TLS in-sim**
|
|
197
|
+
(memory-BIO SSL, so aiohttp HTTPS just works).
|
|
198
|
+
- **💥 Honest crashes & disk** — `host.crash()` is a power cut: tasks stop with no
|
|
199
|
+
`finally` blocks, unsynced disk writes are lost or *torn*, peers see resets.
|
|
200
|
+
- **🧩 Stand-in services** — `world.run_service(SimRedis(), …)` gives you an in-sim Redis
|
|
201
|
+
(SET/GET, WATCH/MULTI/EXEC) to test your logic against, with wire faults applied.
|
|
202
|
+
|
|
203
|
+
**Break it.** Explore the schedules and faults your laptop never will.
|
|
204
|
+
|
|
205
|
+
- **🔬 Fault injection** — `simloom.sometimes("drop_cache")` is tape-driven inside the sim
|
|
206
|
+
and a constant `False` in production. Annotate rare branches; explore them.
|
|
207
|
+
- **🧭 Pluggable search** — uniform random walk, **`pct:auto`** (auto-tuned
|
|
208
|
+
[Probabilistic Concurrency Testing](https://www.microsoft.com/en-us/research/publication/a-randomized-scheduler-with-probabilistic-guarantees-of-finding-bugs/)
|
|
209
|
+
that finds deep-ordering and starvation bugs random walk never hits), and `simloom.soak`
|
|
210
|
+
(resumable, shardable continuous exploration).
|
|
211
|
+
|
|
212
|
+
**Check it.** Assertions far past "didn't crash."
|
|
213
|
+
|
|
214
|
+
- **🛡 Safety & liveness oracles** — `world.always("one leader", …)`,
|
|
215
|
+
`world.eventually("elects a leader", …, within=120)`, `world.leads_to(…)`, plus a
|
|
216
|
+
livelock detector and replica-convergence checks.
|
|
217
|
+
- **✅ Prove it** — `@simloom.test(systematic=True)` switches from sampling seeds to
|
|
218
|
+
*exhaustive* delay-bounded model checking: it finds a deep interleaving bug
|
|
219
|
+
deterministically, or **passes as a bounded proof of correctness**.
|
|
220
|
+
- **🧾 Find wrong answers** — an Elle-style serializability checker
|
|
221
|
+
(`world.check_serializable`) catches a store that returns a stale or impossible read —
|
|
222
|
+
not just one that crashes — and reports the cyclic read/write dependency.
|
|
223
|
+
|
|
224
|
+
**Reproduce & debug it.**
|
|
225
|
+
|
|
162
226
|
- **🪓 Automatic shrinking** — failures reduce to the minimal schedule deviation, with a
|
|
163
|
-
|
|
164
|
-
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
227
|
+
replayable artifact on disk.
|
|
228
|
+
- **🔁 Determinism guarantees** — a per-test self-check (run twice, locate the first
|
|
229
|
+
diverging event), `PYTHONHASHSEED` auto-pin, and a queryable boundary registry.
|
|
230
|
+
- **🕰 Causal trace** — record with `causal=True` and walk it: `simloom trace LOG --step N`
|
|
231
|
+
reconstructs state and the *happens-before stack* that woke it; `--changed x` is an
|
|
232
|
+
omniscient query; `simloom diff A B` finds the first divergence between two universes.
|
|
233
|
+
- **🚨 Escape detection** — touch a real socket, signal, subprocess, or (un-patched) wall
|
|
234
|
+
clock from inside the sim and you get an `EscapedSimulationError` at the exact call site
|
|
169
235
|
instead of silent nondeterminism.
|
|
170
236
|
|
|
171
237
|
## It finds real bugs
|
|
@@ -187,25 +253,29 @@ version survives, with coverage counters proving the faults actually fired.
|
|
|
187
253
|
|
|
188
254
|
A determinism claim is only as good as its disclosed limits. simloom raises a loud error
|
|
189
255
|
when your code reaches outside the simulation, and [`docs/determinism.md`](docs/determinism.md)
|
|
190
|
-
states **exactly** what is and isn't deterministic
|
|
191
|
-
C-extension I/O (`psycopg2`, `requests`,
|
|
192
|
-
|
|
193
|
-
|
|
256
|
+
states **exactly** what is and isn't deterministic — now machine-readable via
|
|
257
|
+
`simloom.boundary()`. Known boundaries: blocking C-extension I/O (`psycopg2`, `requests`,
|
|
258
|
+
grpc's C core) can't run in-sim; a `from time import …` alias or the C-accelerated
|
|
259
|
+
`datetime.now()` isn't redirected by the clock patch; real subprocesses and external
|
|
260
|
+
servers need stand-ins. None of these fail silently — and the per-test determinism
|
|
261
|
+
self-check locates anything that slips through.
|
|
194
262
|
|
|
195
263
|
## Status
|
|
196
264
|
|
|
197
|
-
**
|
|
198
|
-
matrix,
|
|
199
|
-
|
|
200
|
-
|
|
265
|
+
**Alpha, and built in the open.** The deterministic core, the simulated world and fault
|
|
266
|
+
matrix (streams, UDP, TLS, partitions, crashes, torn disk writes), the explorer, shrinker,
|
|
267
|
+
property oracles, systematic verifier, serializability checker, causal-trace CLI, and the
|
|
268
|
+
pytest plugin all exist — exercised by a 10,000-seed determinism torture on every CI run
|
|
269
|
+
(the harness holds itself to the same hostility it applies to your code) across Python
|
|
270
|
+
3.12/3.13/3.14. The API may still shift before 1.0. If you try it,
|
|
201
271
|
[open an issue](https://github.com/mandipadk/simloom/issues) — early feedback shapes it.
|
|
202
272
|
|
|
203
273
|
## Learn more
|
|
204
274
|
|
|
205
275
|
- [`docs/determinism.md`](docs/determinism.md) — the honest boundary of the simulation
|
|
206
276
|
- [`docs/event-log.md`](docs/event-log.md) — the versioned event-log and tape formats
|
|
277
|
+
- [`CHANGELOG.md`](CHANGELOG.md) — every capability, phase by phase
|
|
207
278
|
- [`examples/`](examples/) — the toy Raft and the bpo-42130 reproduction, both runnable
|
|
208
|
-
- [`docs/plan.md`](docs/plan.md) — the architecture and the road to 1.0
|
|
209
279
|
|
|
210
280
|
## Development
|
|
211
281
|
|
|
@@ -22,9 +22,13 @@ state of the art.
|
|
|
22
22
|
**simloom runs your unmodified asyncio program inside a fully simulated world** — a
|
|
23
23
|
seeded scheduler that owns every interleaving, a virtual clock, an in-memory network
|
|
24
24
|
with injectable latency, loss, partitions, and crashes — and explores thousands of
|
|
25
|
-
hostile schedules looking for the one that breaks your invariants.
|
|
26
|
-
it
|
|
27
|
-
|
|
25
|
+
hostile schedules looking for the one that breaks your invariants. It doesn't just find
|
|
26
|
+
crashes: it asserts **safety and liveness** properties, **proves** correctness by
|
|
27
|
+
exhaustive systematic search, and catches **wrong answers** (a store that returns a stale
|
|
28
|
+
or impossible read) with serializability checking. When it finds a problem, it hands you a
|
|
29
|
+
**seed** that replays the failure byte-for-byte, forever, **shrinks** it to the minimal
|
|
30
|
+
schedule that still triggers the bug, and lets you **walk the causal trace** of what woke
|
|
31
|
+
what.
|
|
28
32
|
|
|
29
33
|
```text
|
|
30
34
|
FAILED test_lease_exclusivity — simloom found a failing universe
|
|
@@ -92,6 +96,37 @@ async def test_leader_election(world):
|
|
|
92
96
|
await world.until(lambda: exactly_one_leader(nodes), timeout=120)
|
|
93
97
|
```
|
|
94
98
|
|
|
99
|
+
### Beyond crashes
|
|
100
|
+
|
|
101
|
+
Assert what *should* hold, prove it can't be violated, and catch wrong answers:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
@simloom.test(runs=5000)
|
|
105
|
+
async def test_lease_safety(world):
|
|
106
|
+
cluster = start_cluster(world)
|
|
107
|
+
# safety: never two leaders. liveness: one eventually emerges.
|
|
108
|
+
world.always("≤1 leader", lambda: sum(n.is_leader for n in cluster) <= 1)
|
|
109
|
+
world.eventually("a leader", lambda: any(n.is_leader for n in cluster), within=120)
|
|
110
|
+
await world.sleep(300)
|
|
111
|
+
|
|
112
|
+
@simloom.test(systematic=True, max_delays=3) # exhaustive, not sampled
|
|
113
|
+
async def test_critical_section():
|
|
114
|
+
... # passes ⇒ a bounded PROOF of correctness
|
|
115
|
+
|
|
116
|
+
@simloom.test(runs=5000)
|
|
117
|
+
async def test_store_is_serializable(world):
|
|
118
|
+
await run_transactions(world) # records ops into world.history
|
|
119
|
+
world.assert_serializable() # finds lost updates / write skew, with the cycle
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
And once it finds something, replay and walk it:
|
|
123
|
+
|
|
124
|
+
```sh
|
|
125
|
+
pytest --simloom-seed=17 -k lease # replay the exact failing universe
|
|
126
|
+
simloom trace failure.jsonl --step 42 # reconstruct state + the happens-before stack
|
|
127
|
+
simloom diff run_a.jsonl run_b.jsonl # first divergence between two universes
|
|
128
|
+
```
|
|
129
|
+
|
|
95
130
|
## Why this didn't exist before
|
|
96
131
|
|
|
97
132
|
Rust has [`loom`](https://github.com/tokio-rs/loom),
|
|
@@ -106,32 +141,62 @@ had **nothing**.
|
|
|
106
141
|
And asyncio is *structurally perfect* for it: every interleaving decision happens at an
|
|
107
142
|
`await`, under a **replaceable event loop**. simloom swaps in a deterministic one — no
|
|
108
143
|
forked interpreter, no hypervisor, no recompilation. Because the ecosystem (aiohttp,
|
|
109
|
-
httpx, the streams API) bottoms out in loop primitives, **real, unmodified
|
|
110
|
-
inside the simulation**. Our CI runs
|
|
111
|
-
|
|
144
|
+
httpx, redis, the streams API) bottoms out in loop primitives, **real, unmodified
|
|
145
|
+
libraries run inside the simulation**. Our CI runs an unmodified **aiohttp HTTPS** server
|
|
146
|
+
against an unmodified aiohttp client over a memory-BIO TLS handshake, and an unmodified
|
|
147
|
+
**`redis.asyncio.Redis`** against an in-sim Redis — all over the simulated network, with
|
|
148
|
+
faults injected, replayable byte-for-byte from a seed.
|
|
112
149
|
|
|
113
150
|
## What you get
|
|
114
151
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
-
|
|
118
|
-
|
|
119
|
-
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
-
|
|
126
|
-
|
|
127
|
-
|
|
152
|
+
**Run it.** Your real asyncio code, unchanged, inside the simulation.
|
|
153
|
+
|
|
154
|
+
- **🎲 Seeded scheduling** — a single *choice tape* (the Hypothesis trick, applied to
|
|
155
|
+
schedules) drives every decision. One seed → one exact universe.
|
|
156
|
+
- **⏱ Virtual time** — an hour of simulated `asyncio.sleep` traffic runs in milliseconds;
|
|
157
|
+
opt-in patching makes `time.time()` and the stdlib RNG tape-driven too, so an unmodified
|
|
158
|
+
library that timestamps or rolls a random timeout replays byte-for-byte.
|
|
159
|
+
- **🌐 A simulated network** — in-memory streams (loss modeled as TCP retransmit delay —
|
|
160
|
+
bytes never corrupt), **UDP datagrams** with real loss/reorder/duplication,
|
|
161
|
+
partitions, asymmetric per-link latency/loss shaping, resets, and **TLS in-sim**
|
|
162
|
+
(memory-BIO SSL, so aiohttp HTTPS just works).
|
|
163
|
+
- **💥 Honest crashes & disk** — `host.crash()` is a power cut: tasks stop with no
|
|
164
|
+
`finally` blocks, unsynced disk writes are lost or *torn*, peers see resets.
|
|
165
|
+
- **🧩 Stand-in services** — `world.run_service(SimRedis(), …)` gives you an in-sim Redis
|
|
166
|
+
(SET/GET, WATCH/MULTI/EXEC) to test your logic against, with wire faults applied.
|
|
167
|
+
|
|
168
|
+
**Break it.** Explore the schedules and faults your laptop never will.
|
|
169
|
+
|
|
170
|
+
- **🔬 Fault injection** — `simloom.sometimes("drop_cache")` is tape-driven inside the sim
|
|
171
|
+
and a constant `False` in production. Annotate rare branches; explore them.
|
|
172
|
+
- **🧭 Pluggable search** — uniform random walk, **`pct:auto`** (auto-tuned
|
|
173
|
+
[Probabilistic Concurrency Testing](https://www.microsoft.com/en-us/research/publication/a-randomized-scheduler-with-probabilistic-guarantees-of-finding-bugs/)
|
|
174
|
+
that finds deep-ordering and starvation bugs random walk never hits), and `simloom.soak`
|
|
175
|
+
(resumable, shardable continuous exploration).
|
|
176
|
+
|
|
177
|
+
**Check it.** Assertions far past "didn't crash."
|
|
178
|
+
|
|
179
|
+
- **🛡 Safety & liveness oracles** — `world.always("one leader", …)`,
|
|
180
|
+
`world.eventually("elects a leader", …, within=120)`, `world.leads_to(…)`, plus a
|
|
181
|
+
livelock detector and replica-convergence checks.
|
|
182
|
+
- **✅ Prove it** — `@simloom.test(systematic=True)` switches from sampling seeds to
|
|
183
|
+
*exhaustive* delay-bounded model checking: it finds a deep interleaving bug
|
|
184
|
+
deterministically, or **passes as a bounded proof of correctness**.
|
|
185
|
+
- **🧾 Find wrong answers** — an Elle-style serializability checker
|
|
186
|
+
(`world.check_serializable`) catches a store that returns a stale or impossible read —
|
|
187
|
+
not just one that crashes — and reports the cyclic read/write dependency.
|
|
188
|
+
|
|
189
|
+
**Reproduce & debug it.**
|
|
190
|
+
|
|
128
191
|
- **🪓 Automatic shrinking** — failures reduce to the minimal schedule deviation, with a
|
|
129
|
-
|
|
130
|
-
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
192
|
+
replayable artifact on disk.
|
|
193
|
+
- **🔁 Determinism guarantees** — a per-test self-check (run twice, locate the first
|
|
194
|
+
diverging event), `PYTHONHASHSEED` auto-pin, and a queryable boundary registry.
|
|
195
|
+
- **🕰 Causal trace** — record with `causal=True` and walk it: `simloom trace LOG --step N`
|
|
196
|
+
reconstructs state and the *happens-before stack* that woke it; `--changed x` is an
|
|
197
|
+
omniscient query; `simloom diff A B` finds the first divergence between two universes.
|
|
198
|
+
- **🚨 Escape detection** — touch a real socket, signal, subprocess, or (un-patched) wall
|
|
199
|
+
clock from inside the sim and you get an `EscapedSimulationError` at the exact call site
|
|
135
200
|
instead of silent nondeterminism.
|
|
136
201
|
|
|
137
202
|
## It finds real bugs
|
|
@@ -153,25 +218,29 @@ version survives, with coverage counters proving the faults actually fired.
|
|
|
153
218
|
|
|
154
219
|
A determinism claim is only as good as its disclosed limits. simloom raises a loud error
|
|
155
220
|
when your code reaches outside the simulation, and [`docs/determinism.md`](docs/determinism.md)
|
|
156
|
-
states **exactly** what is and isn't deterministic
|
|
157
|
-
C-extension I/O (`psycopg2`, `requests`,
|
|
158
|
-
|
|
159
|
-
|
|
221
|
+
states **exactly** what is and isn't deterministic — now machine-readable via
|
|
222
|
+
`simloom.boundary()`. Known boundaries: blocking C-extension I/O (`psycopg2`, `requests`,
|
|
223
|
+
grpc's C core) can't run in-sim; a `from time import …` alias or the C-accelerated
|
|
224
|
+
`datetime.now()` isn't redirected by the clock patch; real subprocesses and external
|
|
225
|
+
servers need stand-ins. None of these fail silently — and the per-test determinism
|
|
226
|
+
self-check locates anything that slips through.
|
|
160
227
|
|
|
161
228
|
## Status
|
|
162
229
|
|
|
163
|
-
**
|
|
164
|
-
matrix,
|
|
165
|
-
|
|
166
|
-
|
|
230
|
+
**Alpha, and built in the open.** The deterministic core, the simulated world and fault
|
|
231
|
+
matrix (streams, UDP, TLS, partitions, crashes, torn disk writes), the explorer, shrinker,
|
|
232
|
+
property oracles, systematic verifier, serializability checker, causal-trace CLI, and the
|
|
233
|
+
pytest plugin all exist — exercised by a 10,000-seed determinism torture on every CI run
|
|
234
|
+
(the harness holds itself to the same hostility it applies to your code) across Python
|
|
235
|
+
3.12/3.13/3.14. The API may still shift before 1.0. If you try it,
|
|
167
236
|
[open an issue](https://github.com/mandipadk/simloom/issues) — early feedback shapes it.
|
|
168
237
|
|
|
169
238
|
## Learn more
|
|
170
239
|
|
|
171
240
|
- [`docs/determinism.md`](docs/determinism.md) — the honest boundary of the simulation
|
|
172
241
|
- [`docs/event-log.md`](docs/event-log.md) — the versioned event-log and tape formats
|
|
242
|
+
- [`CHANGELOG.md`](CHANGELOG.md) — every capability, phase by phase
|
|
173
243
|
- [`examples/`](examples/) — the toy Raft and the bpo-42130 reproduction, both runnable
|
|
174
|
-
- [`docs/plan.md`](docs/plan.md) — the architecture and the road to 1.0
|
|
175
244
|
|
|
176
245
|
## Development
|
|
177
246
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "simloom"
|
|
3
3
|
dynamic = ["version"]
|
|
4
|
-
description = "Deterministic simulation testing for asyncio:
|
|
4
|
+
description = "Deterministic simulation testing for asyncio: explore every schedule, inject faults, assert properties, prove correctness, and replay any failure from a seed."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12"
|
|
7
7
|
license = "Apache-2.0"
|
|
@@ -14,9 +14,13 @@ keywords = [
|
|
|
14
14
|
"testing",
|
|
15
15
|
"fault-injection",
|
|
16
16
|
"concurrency",
|
|
17
|
+
"model-checking",
|
|
18
|
+
"serializability",
|
|
19
|
+
"chaos-engineering",
|
|
20
|
+
"property-testing",
|
|
17
21
|
]
|
|
18
22
|
classifiers = [
|
|
19
|
-
"Development Status ::
|
|
23
|
+
"Development Status :: 3 - Alpha",
|
|
20
24
|
"Framework :: AsyncIO",
|
|
21
25
|
"Framework :: Pytest",
|
|
22
26
|
"Intended Audience :: Developers",
|
|
@@ -46,8 +50,12 @@ dev = [
|
|
|
46
50
|
spikes = [
|
|
47
51
|
"aiohttp>=3.11",
|
|
48
52
|
"httpx>=0.28",
|
|
53
|
+
"redis>=5.0",
|
|
49
54
|
]
|
|
50
55
|
|
|
56
|
+
[project.scripts]
|
|
57
|
+
simloom = "simloom._cli:main"
|
|
58
|
+
|
|
51
59
|
[project.entry-points.pytest11]
|
|
52
60
|
simloom = "simloom._pytest_plugin"
|
|
53
61
|
|