detangle 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- detangle-0.1.0/.github/ISSUE_TEMPLATE/bug_report.md +15 -0
- detangle-0.1.0/.github/ISSUE_TEMPLATE/feature_request.md +9 -0
- detangle-0.1.0/.github/workflows/ci.yml +62 -0
- detangle-0.1.0/.github/workflows/release.yml +21 -0
- detangle-0.1.0/.gitignore +15 -0
- detangle-0.1.0/CHANGELOG.md +31 -0
- detangle-0.1.0/CONTRIBUTING.md +48 -0
- detangle-0.1.0/LICENSE +21 -0
- detangle-0.1.0/PKG-INFO +376 -0
- detangle-0.1.0/README.md +342 -0
- detangle-0.1.0/docs/guide.md +270 -0
- detangle-0.1.0/docs/how-it-works.md +145 -0
- detangle-0.1.0/docs/linearizability.md +108 -0
- detangle-0.1.0/docs/network.md +133 -0
- detangle-0.1.0/examples/bank_race.py +71 -0
- detangle-0.1.0/examples/cache_stampede.py +88 -0
- detangle-0.1.0/examples/connection_pool.py +84 -0
- detangle-0.1.0/examples/dining_philosophers.py +55 -0
- detangle-0.1.0/examples/line_protocol.py +86 -0
- detangle-0.1.0/examples/order_events.py +78 -0
- detangle-0.1.0/examples/replicated_kv.py +118 -0
- detangle-0.1.0/examples/retry_backoff.py +59 -0
- detangle-0.1.0/pyproject.toml +93 -0
- detangle-0.1.0/src/detangle/__init__.py +118 -0
- detangle-0.1.0/src/detangle/__main__.py +157 -0
- detangle-0.1.0/src/detangle/_api.py +144 -0
- detangle-0.1.0/src/detangle/_choices.py +121 -0
- detangle-0.1.0/src/detangle/_config.py +111 -0
- detangle-0.1.0/src/detangle/_context.py +14 -0
- detangle-0.1.0/src/detangle/_database.py +59 -0
- detangle-0.1.0/src/detangle/_deadlock.py +226 -0
- detangle-0.1.0/src/detangle/_frames.py +185 -0
- detangle-0.1.0/src/detangle/_html.py +158 -0
- detangle-0.1.0/src/detangle/_instrument.py +87 -0
- detangle-0.1.0/src/detangle/_loop.py +1044 -0
- detangle-0.1.0/src/detangle/_runner.py +233 -0
- detangle-0.1.0/src/detangle/_settings.py +87 -0
- detangle-0.1.0/src/detangle/_shrink.py +119 -0
- detangle-0.1.0/src/detangle/_timepatch.py +52 -0
- detangle-0.1.0/src/detangle/_trace.py +308 -0
- detangle-0.1.0/src/detangle/_version.py +1 -0
- detangle-0.1.0/src/detangle/errors.py +113 -0
- detangle-0.1.0/src/detangle/explore.py +558 -0
- detangle-0.1.0/src/detangle/lin.py +619 -0
- detangle-0.1.0/src/detangle/net.py +1014 -0
- detangle-0.1.0/src/detangle/py.typed +0 -0
- detangle-0.1.0/src/detangle/pytest_plugin.py +115 -0
- detangle-0.1.0/src/detangle/strategies.py +371 -0
- detangle-0.1.0/tests/conftest.py +24 -0
- detangle-0.1.0/tests/test_examples.py +44 -0
- detangle-0.1.0/tests/test_explore.py +398 -0
- detangle-0.1.0/tests/test_faults.py +253 -0
- detangle-0.1.0/tests/test_integration.py +269 -0
- detangle-0.1.0/tests/test_lin.py +296 -0
- detangle-0.1.0/tests/test_loop.py +382 -0
- detangle-0.1.0/tests/test_net.py +324 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Bug report
|
|
3
|
+
about: detangle crashed, missed a bug, or reported something wrong
|
|
4
|
+
labels: bug
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
**What happened**
|
|
8
|
+
|
|
9
|
+
**Minimal test** (and the replay token, if a report was printed)
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
**Versions**: detangle `x.y.z`, Python `3.x`, OS
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: ["main", "claude/**"]
|
|
6
|
+
pull_request:
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
lint:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
- uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: "3.12"
|
|
20
|
+
- run: python -m pip install -e ".[dev]"
|
|
21
|
+
- run: ruff check src tests examples
|
|
22
|
+
- run: ruff format --check src tests examples
|
|
23
|
+
- run: mypy
|
|
24
|
+
|
|
25
|
+
test:
|
|
26
|
+
runs-on: ${{ matrix.os }}
|
|
27
|
+
strategy:
|
|
28
|
+
fail-fast: false
|
|
29
|
+
matrix:
|
|
30
|
+
os: [ubuntu-latest]
|
|
31
|
+
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
|
32
|
+
include:
|
|
33
|
+
- os: macos-latest
|
|
34
|
+
python-version: "3.13"
|
|
35
|
+
- os: windows-latest
|
|
36
|
+
python-version: "3.13"
|
|
37
|
+
steps:
|
|
38
|
+
- uses: actions/checkout@v4
|
|
39
|
+
- uses: actions/setup-python@v5
|
|
40
|
+
with:
|
|
41
|
+
python-version: ${{ matrix.python-version }}
|
|
42
|
+
allow-prereleases: true
|
|
43
|
+
- run: python -m pip install -e ".[dev]"
|
|
44
|
+
- run: python -m pytest -q
|
|
45
|
+
- name: Run the examples
|
|
46
|
+
shell: bash
|
|
47
|
+
run: for f in examples/*.py; do python "$f" > /dev/null || exit 1; done
|
|
48
|
+
|
|
49
|
+
build:
|
|
50
|
+
runs-on: ubuntu-latest
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/checkout@v4
|
|
53
|
+
- uses: actions/setup-python@v5
|
|
54
|
+
with:
|
|
55
|
+
python-version: "3.12"
|
|
56
|
+
- run: python -m pip install build twine
|
|
57
|
+
- run: python -m build
|
|
58
|
+
- run: python -m twine check dist/*
|
|
59
|
+
- uses: actions/upload-artifact@v4
|
|
60
|
+
with:
|
|
61
|
+
name: dist
|
|
62
|
+
path: dist/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
publish:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
environment: pypi
|
|
11
|
+
permissions:
|
|
12
|
+
id-token: write # PyPI trusted publishing
|
|
13
|
+
contents: read
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: actions/setup-python@v5
|
|
17
|
+
with:
|
|
18
|
+
python-version: "3.12"
|
|
19
|
+
- run: python -m pip install build
|
|
20
|
+
- run: python -m build
|
|
21
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
|
|
5
|
+
[semantic versioning](https://semver.org/).
|
|
6
|
+
|
|
7
|
+
## [0.1.0] - 2026-09-25
|
|
8
|
+
|
|
9
|
+
First public release.
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- `SimLoop`: a deterministic asyncio event loop with virtual time, per-task lanes and
|
|
14
|
+
recorded scheduling decisions. In default order it reproduces asyncio's own scheduling.
|
|
15
|
+
- `detangle.run`, `detangle.explore`, `detangle.replay` and the `@detangle.test` decorator.
|
|
16
|
+
- Strategies: `FIFO`, `RandomWalk`, `PCT`, bounded exhaustive `DFS`, `Replay`, `Portfolio`,
|
|
17
|
+
and the default `auto` portfolio.
|
|
18
|
+
- Hypothesis-style shrinking of failing schedules, and replay tokens (`dt1-...`).
|
|
19
|
+
- Failure detection: exceptions, deadlocks (with lock holders and wait-for cycles), invariant
|
|
20
|
+
violations, unobserved background exceptions, callback errors, step and time limits, task
|
|
21
|
+
leaks.
|
|
22
|
+
- Fault injection: `maybe_timeout`, `inject_cancellation`, timer jitter.
|
|
23
|
+
- Nondeterministic inputs: `choice`, `randint`, `uniform`, `flip`, `shuffle`, plus per-run
|
|
24
|
+
seeding of `random`.
|
|
25
|
+
- Simulated network: TCP-like streams (latency, coalescing, fragmentation, half-close, resets),
|
|
26
|
+
UDP-like datagrams (loss, duplication, reordering), mailboxes, partitions, crashes and
|
|
27
|
+
restarts.
|
|
28
|
+
- Linearizability checker (`detangle.lin`) with Register, KV, Counter, Set, FIFOQueue and
|
|
29
|
+
Mutex models.
|
|
30
|
+
- pytest plugin, CLI (`detangle explore|replay|decode`), interactive HTML reports, JSON
|
|
31
|
+
output, and a failing-example database.
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Contributing to detangle
|
|
2
|
+
|
|
3
|
+
Thanks for helping make async Python more reliable! Bug reports, docs fixes, new examples and
|
|
4
|
+
features are all welcome.
|
|
5
|
+
|
|
6
|
+
## Development setup
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
git clone https://github.com/louistarwars/Detangle detangle && cd detangle
|
|
10
|
+
python -m venv .venv && . .venv/bin/activate
|
|
11
|
+
pip install -e ".[dev]"
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Checks (the same ones CI runs)
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pytest # the test suite, including the examples
|
|
18
|
+
ruff check src tests examples
|
|
19
|
+
ruff format --check src tests examples
|
|
20
|
+
mypy # strict mode
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
detangle supports CPython 3.10 to 3.14 and has no runtime dependencies. Please keep it that
|
|
24
|
+
way. The loop relies on a few asyncio internals (`Handle._run`, `Task._fut_waiter`,
|
|
25
|
+
`Task._log_traceback`...). Any new one needs a test that runs on every supported version.
|
|
26
|
+
|
|
27
|
+
## Guidelines
|
|
28
|
+
|
|
29
|
+
- **Determinism first.** Nothing in detangle may depend on real time, `id()`-based ordering
|
|
30
|
+
or iteration over sets of objects. Use insertion-ordered dicts and lists.
|
|
31
|
+
- **Decisions default to asyncio.** Any new source of nondeterminism must go through
|
|
32
|
+
`SimLoop.choose()` or `SimLoop.flip()`, with `0` / `False` meaning "what a stock event loop
|
|
33
|
+
(or a perfect network) would do". That keeps replay, shrinking and DFS working for free.
|
|
34
|
+
- **Reports are the product.** When you add a failure mode, make sure the report explains it
|
|
35
|
+
in plain words, with a location in *user* code.
|
|
36
|
+
- **Tests.** Every bug fix comes with a regression test. Every feature comes with tests, plus
|
|
37
|
+
documentation in `docs/`.
|
|
38
|
+
|
|
39
|
+
## Reporting a bug in detangle
|
|
40
|
+
|
|
41
|
+
Please include the detangle version, the Python version, a minimal test, and the replay token
|
|
42
|
+
if a report was involved. If you believe detangle reports a schedule that cannot happen with
|
|
43
|
+
a real event loop, say which deviation (`^ ran ahead of ...`) looks impossible and why.
|
|
44
|
+
|
|
45
|
+
## Code of conduct
|
|
46
|
+
|
|
47
|
+
Be kind, assume good faith, and keep discussions technical. Harassment of any kind is not
|
|
48
|
+
tolerated.
|
detangle-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 louistarwars and the detangle contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
detangle-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,376 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: detangle
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Deterministic simulation testing for Python asyncio: find race conditions, deadlocks and cancellation bugs, then replay them exactly.
|
|
5
|
+
Project-URL: Homepage, https://github.com/louistarwars/Detangle
|
|
6
|
+
Project-URL: Documentation, https://github.com/louistarwars/Detangle/tree/main/docs
|
|
7
|
+
Project-URL: Issues, https://github.com/louistarwars/Detangle/issues
|
|
8
|
+
Author: louistarwars and the detangle contributors
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: asyncio,concurrency,deadlock,deterministic-simulation,fuzzing,linearizability,model-checking,pytest,race-condition,testing
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Framework :: AsyncIO
|
|
14
|
+
Classifier: Framework :: Pytest
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
24
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
25
|
+
Classifier: Topic :: Software Development :: Testing
|
|
26
|
+
Classifier: Typing :: Typed
|
|
27
|
+
Requires-Python: >=3.10
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: hypothesis>=6; extra == 'dev'
|
|
30
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
32
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
<div align="center">
|
|
36
|
+
|
|
37
|
+
# detangle
|
|
38
|
+
|
|
39
|
+
**Deterministic simulation testing for Python asyncio.**
|
|
40
|
+
|
|
41
|
+
Find the race conditions, deadlocks and cancellation bugs hiding in your async code.<br>
|
|
42
|
+
Get them back as a *minimal*, *exactly replayable* schedule.
|
|
43
|
+
|
|
44
|
+
[](https://github.com/louistarwars/Detangle/actions/workflows/ci.yml)
|
|
45
|
+

|
|
46
|
+

|
|
47
|
+

|
|
48
|
+
[](LICENSE)
|
|
49
|
+
|
|
50
|
+
</div>
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
Your async test suite runs every test in **one** timeline: the order asyncio happens to pick,
|
|
55
|
+
with a perfect network, a perfect database and no timeouts. Production runs it in millions of
|
|
56
|
+
others. That is where the bugs are.
|
|
57
|
+
|
|
58
|
+
**detangle** runs your `async def` code on a deterministic event loop that it fully controls.
|
|
59
|
+
It replays the same test hundreds of times, and each run explores a different, realistic
|
|
60
|
+
"what if": another task wins the race, a timer fires late, a TCP packet is split in two, the
|
|
61
|
+
network partitions, a timeout hits at the worst possible `await`. When something breaks, it
|
|
62
|
+
**shrinks** the failure to the simplest schedule that still triggers it and hands you a short
|
|
63
|
+
token that replays it **bit for bit**, on any machine.
|
|
64
|
+
|
|
65
|
+
It is the approach that FoundationDB, TigerBeetle and Antithesis made famous, packaged for
|
|
66
|
+
everyday Python: no dependencies, no code changes, a pytest plugin, and virtual time so
|
|
67
|
+
sleeps cost nothing.
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
pip install detangle
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## 30-second tour
|
|
74
|
+
|
|
75
|
+
This test passes with `asyncio.run`, with pytest-asyncio, on your laptop and in CI. Every time.
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
import asyncio
|
|
79
|
+
import detangle
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class OrderService:
|
|
83
|
+
def __init__(self, events):
|
|
84
|
+
self.events = events
|
|
85
|
+
|
|
86
|
+
async def create(self, order_id):
|
|
87
|
+
await asyncio.sleep(0.005) # database write
|
|
88
|
+
self.events.append(("created", order_id))
|
|
89
|
+
|
|
90
|
+
async def ship(self, order_id):
|
|
91
|
+
await asyncio.sleep(0.005) # database write
|
|
92
|
+
self.events.append(("shipped", order_id))
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@detangle.test
|
|
96
|
+
async def test_order_lifecycle():
|
|
97
|
+
events = []
|
|
98
|
+
service = OrderService(events)
|
|
99
|
+
await asyncio.gather(service.create(42), service.ship(42))
|
|
100
|
+
assert events[0] == ("created", 42), f"consumers saw {events}"
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Under detangle it fails on the second run, because in production the "ship" write can finish first:
|
|
104
|
+
|
|
105
|
+
```text
|
|
106
|
+
detangle found a bug in test_orders.test_order_lifecycle after 2 runs [random(seed=40222063875785)]
|
|
107
|
+
|
|
108
|
+
AssertionError: consumers saw [('shipped', 42), ('created', 42)] ...
|
|
109
|
+
at test_orders.py:23 in test_order_lifecycle: assert events[0] == ("created", 42), f"consumers saw {events}"
|
|
110
|
+
|
|
111
|
+
Minimal failing schedule: 1 deviation from asyncio's default behaviour (shrunk from 3 in 5 replays).
|
|
112
|
+
|
|
113
|
+
Interleaving (* = the scheduler deviated from asyncio's FIFO order here):
|
|
114
|
+
1 0 test_order_lifecycle spawn OrderService.create#2 test_orders.py:22 | await asyncio.gather(...)
|
|
115
|
+
2 0 test_order_lifecycle spawn OrderService.ship#3 test_orders.py:22 | await asyncio.gather(...)
|
|
116
|
+
3 0 test_order_lifecycle await test_orders.py:22 | await asyncio.gather(...)
|
|
117
|
+
4 0 OrderService.ship#3 * await sleep test_orders.py:14 | await asyncio.sleep(0.005) # database write
|
|
118
|
+
^ ran ahead of OrderService.create#2
|
|
119
|
+
5 0 OrderService.create#2 await sleep test_orders.py:10 | await asyncio.sleep(0.005) # database write
|
|
120
|
+
6 5ms OrderService.ship#3 return
|
|
121
|
+
7 5ms OrderService.create#2 return
|
|
122
|
+
8 5ms test_order_lifecycle raise AssertionError: consumers saw [('shipped', 42), ('created', 42)]
|
|
123
|
+
|
|
124
|
+
Reproduce this exact run:
|
|
125
|
+
DETANGLE_REPLAY=dt1-cgAB pytest "test_orders.py::test_order_lifecycle"
|
|
126
|
+
detangle.replay(test_order_lifecycle, "dt1-cgAB")
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
The whole exploration (dozens of runs, shrinking, replay) takes a fraction of a second: time is
|
|
130
|
+
virtual, so `asyncio.sleep(3600)` returns instantly while `loop.time()` still advances by an hour.
|
|
131
|
+
|
|
132
|
+
## What it finds
|
|
133
|
+
|
|
134
|
+
| Bug class | How detangle exposes it |
|
|
135
|
+
| --- | --- |
|
|
136
|
+
| **Races and ordering bugs** (check-then-act across `await`, lost updates, events out of order) | Explores which ready task runs next, using PCT, random walks or an exhaustive bounded search. |
|
|
137
|
+
| **Deadlocks** | Detects that nothing can make progress, then reports each blocked task, the lock it waits for, **who holds it**, and the wait-for cycle. |
|
|
138
|
+
| **Cancellation-safety bugs** (leaked connections or locks, half-applied updates) | `detangle.maybe_timeout()` lets the explorer choose the exact `await` a timeout hits. |
|
|
139
|
+
| **Hangs and lost wake-ups** | "`worker#3` waits for Event `ready` to be set (nothing left can set it)". |
|
|
140
|
+
| **Livelocks and retry storms** | Step and virtual-time budgets (`max_steps`, `max_time`). |
|
|
141
|
+
| **Silent background crashes** | Exceptions in tasks that nobody awaited, and exceptions in callbacks, fail the test. |
|
|
142
|
+
| **Protocol and framing bugs** | TCP writes coalesced and split into several reads; datagrams lost, duplicated or reordered. |
|
|
143
|
+
| **Distributed-systems bugs** | Partitions, crashes and restarts, message loss, plus a linearizability checker (Jepsen-style) for your histories. |
|
|
144
|
+
| **Broken invariants** | `detangle.invariant(check)` runs *after every scheduling step*, not just at the end. |
|
|
145
|
+
|
|
146
|
+
## Features
|
|
147
|
+
|
|
148
|
+
- **Drop-in.** `SimLoop` is a real `asyncio.AbstractEventLoop`. Tasks, `gather`, `wait_for`,
|
|
149
|
+
`timeout`, `TaskGroup`, locks, queues, events, async generators, `to_thread`, contextvars and
|
|
150
|
+
eager tasks all work. In default order it reproduces asyncio's scheduling **exactly**, and the
|
|
151
|
+
test suite checks this against the real event loop.
|
|
152
|
+
- **Virtual time.** `detangle.run(main)` is `asyncio.run` with an instant clock. Test a
|
|
153
|
+
10-minute exponential backoff in under a millisecond.
|
|
154
|
+
- **Smart exploration.** Strategies include `PCT` (probabilistic concurrency testing, with
|
|
155
|
+
guaranteed detection probability for bugs of bounded depth), `RandomWalk`, `DFS` (exhaustive,
|
|
156
|
+
delay-bounded: *"no bug with ≤ k deviations exists"*), `FIFO` and `Replay`. The default
|
|
157
|
+
`auto` portfolio mixes them.
|
|
158
|
+
- **Shrinking.** Hypothesis-style minimisation of the failing schedule: you debug *one* context
|
|
159
|
+
switch, not forty.
|
|
160
|
+
- **Perfect replay.** Every source of nondeterminism is a recorded decision. A failure is a
|
|
161
|
+
token like `dt1-cgAB` that reproduces it exactly, including network latencies, injected
|
|
162
|
+
faults, and `random` values drawn by your code.
|
|
163
|
+
- **Simulated network.** Unmodified `asyncio.open_connection` / `start_server` /
|
|
164
|
+
`create_datagram_endpoint` code runs over an in-memory network with latency,
|
|
165
|
+
fragmentation, loss, duplication, partitions, host crashes and restarts. There is also a
|
|
166
|
+
`Mailbox` API for prototyping protocols such as Raft or Paxos.
|
|
167
|
+
- **Linearizability checking.** A Wing & Gong / Lowe search with P-compositionality, like
|
|
168
|
+
Knossos and Porcupine, with ready-made models (register, KV, queue, set, counter, mutex) and
|
|
169
|
+
support for operations of unknown outcome.
|
|
170
|
+
- **Great failure reports.** A readable interleaving trace, deadlock wait-for graphs,
|
|
171
|
+
linearizability counterexamples with a timeline, and an interactive HTML report.
|
|
172
|
+
- **Regression database.** Failing schedules are saved to `.detangle/` and replayed first on
|
|
173
|
+
the next run, so a bug found once keeps failing until it is really fixed.
|
|
174
|
+
- **Tooling.** A pytest plugin (marker, decorator, `--detangle-*` options), a CLI
|
|
175
|
+
(`detangle explore|replay|decode`), JSON output, strict typing, and zero dependencies.
|
|
176
|
+
|
|
177
|
+
## Usage
|
|
178
|
+
|
|
179
|
+
### With pytest
|
|
180
|
+
|
|
181
|
+
```python
|
|
182
|
+
import detangle, pytest
|
|
183
|
+
|
|
184
|
+
@detangle.test # 200 schedules by default
|
|
185
|
+
async def test_transfer(): ...
|
|
186
|
+
|
|
187
|
+
@detangle.test(runs=2000, strategy="pct:3", timer_jitter=0.01)
|
|
188
|
+
async def test_harder(): ...
|
|
189
|
+
|
|
190
|
+
@pytest.mark.detangle(runs=500) # no import needed: the plugin handles async tests
|
|
191
|
+
async def test_marked(tmp_path): ... # fixtures work
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
pytest --detangle-runs=5000 # explore more (e.g. nightly)
|
|
196
|
+
pytest --detangle-seed=1234 # reproducible exploration
|
|
197
|
+
pytest --detangle-replay=dt1-cgAB tests/test_orders.py::test_order_lifecycle
|
|
198
|
+
pytest --detangle-report-dir=reports # interactive HTML report per bug
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
### Without pytest
|
|
202
|
+
|
|
203
|
+
```python
|
|
204
|
+
stats = detangle.explore(my_async_fn, runs=1000) # raises detangle.BugFound
|
|
205
|
+
result = detangle.run(my_async_fn) # like asyncio.run, virtual time
|
|
206
|
+
result = detangle.run(my_async_fn, seed=7) # one random schedule, reproducible
|
|
207
|
+
detangle.replay(my_async_fn, "dt1-cgAB") # re-run a failure with a full trace
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
detangle explore tests/test_orders.py:test_order_lifecycle --runs 1000
|
|
212
|
+
detangle replay dt1-cgAB tests/test_orders.py:test_order_lifecycle --html report.html
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
### Prove the absence of bugs (within a bound)
|
|
216
|
+
|
|
217
|
+
```python
|
|
218
|
+
stats = detangle.explore(fixed_version, strategy=detangle.DFS(max_delays=3), runs=100_000)
|
|
219
|
+
assert stats.exhausted # every schedule with at most 3 deviations was checked
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
### Timeouts at the worst moment
|
|
223
|
+
|
|
224
|
+
```python
|
|
225
|
+
async def client():
|
|
226
|
+
try:
|
|
227
|
+
await detangle.maybe_timeout(pool.query("SELECT 1")) # may be cancelled at any await
|
|
228
|
+
except TimeoutError:
|
|
229
|
+
pass
|
|
230
|
+
|
|
231
|
+
await asyncio.gather(*(client() for _ in range(4)))
|
|
232
|
+
assert pool.in_use == 0, "connection leaked on timeout"
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
### Invariants after every step
|
|
236
|
+
|
|
237
|
+
```python
|
|
238
|
+
detangle.invariant(lambda: accounts.total() == 1_000, "money is conserved")
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
### A network that fights back
|
|
242
|
+
|
|
243
|
+
```python
|
|
244
|
+
@detangle.test(net={"latency": (0.001, 0.05), "fragment": 0.3, "drop": 0.01})
|
|
245
|
+
async def test_replication():
|
|
246
|
+
net = detangle.network()
|
|
247
|
+
primary = detangle.spawn(run_server(), host="db-1") # tasks run on simulated hosts
|
|
248
|
+
replica = detangle.spawn(run_replica(), host="db-2")
|
|
249
|
+
reader, writer = await asyncio.open_connection("db-1", 5432) # unmodified asyncio code
|
|
250
|
+
...
|
|
251
|
+
net.partition(["db-1"], ["db-2"]) # split brain
|
|
252
|
+
net.crash("db-1"); net.restart("db-1") # kill -9 and reboot
|
|
253
|
+
net.heal()
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
### Linearizability (Jepsen in a unit test)
|
|
257
|
+
|
|
258
|
+
```python
|
|
259
|
+
history = detangle.History()
|
|
260
|
+
|
|
261
|
+
async def client(name):
|
|
262
|
+
await history.call(name, "put", ("x", 1), kv.put("x", 1))
|
|
263
|
+
await history.call(name, "get", "x", kv.get("x"))
|
|
264
|
+
|
|
265
|
+
await asyncio.gather(*(client(f"c{i}") for i in range(3)))
|
|
266
|
+
history.assert_linearizable(detangle.lin.KV())
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
```text
|
|
270
|
+
NotLinearizable: history is not linearizable with respect to the kv model
|
|
271
|
+
longest linearizable prefix (4 of 9 operations):
|
|
272
|
+
1. 'c2': get('x') -> None
|
|
273
|
+
2. 'c2': put('x', 'c2-1') -> None
|
|
274
|
+
...
|
|
275
|
+
no valid ordering can explain: 'c2': get('x') -> None (invoked at event 7, returned at event 8)
|
|
276
|
+
timeline (real-time order, left to right; > = never returned):
|
|
277
|
+
'c0' [-----------1-----------] [---6----] [---8----]
|
|
278
|
+
'c1' [------------2-------------] [---7----] [--9--]
|
|
279
|
+
'c2' [3-] [4-] [5-]
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
### Nondeterministic test inputs
|
|
283
|
+
|
|
284
|
+
```python
|
|
285
|
+
size = detangle.randint(1, 10) # explored by the strategy, shrunk towards 1
|
|
286
|
+
mode = detangle.choice(["fast", "safe"])
|
|
287
|
+
delay = detangle.uniform(0, 0.1)
|
|
288
|
+
detangle.shuffle(requests)
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
## How it works
|
|
292
|
+
|
|
293
|
+
```mermaid
|
|
294
|
+
flowchart LR
|
|
295
|
+
T["async def test"] --> L["SimLoop<br/>(virtual time)"]
|
|
296
|
+
S["Strategy<br/>PCT · DFS · random"] -- "decisions" --> L
|
|
297
|
+
L -- "ready tasks, timers,<br/>packets, faults" --> S
|
|
298
|
+
L --> R{"failure?"}
|
|
299
|
+
R -- "no" --> N["next run"]
|
|
300
|
+
N --> L
|
|
301
|
+
R -- "yes" --> K["shrink the<br/>decision list"]
|
|
302
|
+
K --> P["minimal schedule<br/>+ replay token + trace"]
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
asyncio runs *handles*, and every step of every task is one. `SimLoop` puts ready handles into
|
|
306
|
+
**lanes** (one per task, one for plain callbacks, one per executor job). Whenever more than one
|
|
307
|
+
lane can run, it asks a **strategy**. Answer `0` means "what asyncio would do", so a run of all
|
|
308
|
+
zeros *is* stock asyncio. Timer jitter, network latency, packet loss, fragmentation, injected
|
|
309
|
+
cancellations and your own `detangle.choice()` calls are decisions too.
|
|
310
|
+
|
|
311
|
+
A run is therefore fully described by its list of integers. Replaying the list reproduces the
|
|
312
|
+
run. Shrinking the list, while keeping the same failure, simplifies the bug. Enumerating lists
|
|
313
|
+
with a bounded sum explores all schedules with at most *k* deviations. The details are in
|
|
314
|
+
[docs/how-it-works.md](docs/how-it-works.md).
|
|
315
|
+
|
|
316
|
+
## How it compares
|
|
317
|
+
|
|
318
|
+
| | detangle | plain asyncio / pytest-asyncio | trio `MockClock` | Hypothesis | Loom / Shuttle (Rust) | Coyote (.NET) | Jepsen |
|
|
319
|
+
|---|:-:|:-:|:-:|:-:|:-:|:-:|:-:|
|
|
320
|
+
| Target | Python asyncio | Python | Python trio | Python | Rust | C# | any (black box) |
|
|
321
|
+
| Controls task interleavings | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ (real concurrency) |
|
|
322
|
+
| Virtual time | ✅ | ❌ | ✅ | ❌ | ❌ | partly | ❌ |
|
|
323
|
+
| Exact replay of a failure | ✅ token | ❌ | ❌ | ✅ inputs | ✅ | ✅ | ❌ |
|
|
324
|
+
| Shrinks failing schedules | ✅ | ❌ | ❌ | inputs only | ❌ | ❌ | ❌ |
|
|
325
|
+
| Bounded exhaustive search | ✅ DFS | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ |
|
|
326
|
+
| Simulated network and faults | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (real network) |
|
|
327
|
+
| Linearizability checker | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
|
328
|
+
| Runs as a unit test, in milliseconds | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
|
|
329
|
+
|
|
330
|
+
## Honest limitations
|
|
331
|
+
|
|
332
|
+
- **Simulated I/O only.** Real sockets, subprocesses and signal handlers raise
|
|
333
|
+
`detangle.RealIOError`. Use asyncio streams or protocols (served by the simulated network),
|
|
334
|
+
or fake the dependency. Clients built on asyncio streams work unchanged; libraries that open
|
|
335
|
+
raw sockets do not.
|
|
336
|
+
- **Reordering is an over-approximation by design.** With `reorder=True` (the default),
|
|
337
|
+
detangle assumes every `await` may take longer than usual, which is what real I/O does. For
|
|
338
|
+
purely in-memory code, asyncio guarantees FIFO wake-ups, so a reported schedule might be one
|
|
339
|
+
your production code can never hit. The report tells you exactly which deviation was needed.
|
|
340
|
+
`reorder=False` keeps strict asyncio order and explores only time, network and faults.
|
|
341
|
+
- **Threads are simulated.** `to_thread` and `run_in_executor` jobs run on the loop thread, in
|
|
342
|
+
their own lane. Code that spins its own threads is outside the model.
|
|
343
|
+
- **Determinism comes from your code too.** Replays are exact if the code under test gets
|
|
344
|
+
randomness from `random` (seeded per run) or `detangle.*`, and time from the loop
|
|
345
|
+
(`patch_time=True` also covers `time.monotonic()` and friends). Iterating over sets of
|
|
346
|
+
objects hashed by `id()` can differ between processes; detangle warns when a replay diverges.
|
|
347
|
+
- **Speed.** About 100k scheduling steps per second in pure Python, so a few hundred runs of a
|
|
348
|
+
typical test take well under a second.
|
|
349
|
+
|
|
350
|
+
## Documentation
|
|
351
|
+
|
|
352
|
+
- [Guide](docs/guide.md): every API, with examples
|
|
353
|
+
- [How it works](docs/how-it-works.md): lanes, decisions, strategies, shrinking, soundness
|
|
354
|
+
- [Simulated network](docs/network.md): TCP, UDP, mailboxes, partitions, crashes
|
|
355
|
+
- [Linearizability](docs/linearizability.md): recording histories and writing models
|
|
356
|
+
- [Examples](examples/): runnable scenarios, each with a bug and its fix
|
|
357
|
+
([bank race](examples/bank_race.py), [order events](examples/order_events.py),
|
|
358
|
+
[dining philosophers](examples/dining_philosophers.py),
|
|
359
|
+
[connection pool](examples/connection_pool.py), [cache stampede](examples/cache_stampede.py),
|
|
360
|
+
[line protocol](examples/line_protocol.py), [replicated KV](examples/replicated_kv.py),
|
|
361
|
+
[retry backoff](examples/retry_backoff.py))
|
|
362
|
+
|
|
363
|
+
## Roadmap
|
|
364
|
+
|
|
365
|
+
- Dynamic partial-order reduction (DPOR) to skip equivalent interleavings
|
|
366
|
+
- Native trio / AnyIO backends
|
|
367
|
+
- A simulated disk with `fsync` semantics (torn writes, lost unsynced data)
|
|
368
|
+
- Coverage-guided exploration (steer the strategy towards new interleavings)
|
|
369
|
+
- A GitHub Action that uploads HTML reports as artifacts
|
|
370
|
+
- Adapters for popular clients (Redis, PostgreSQL, HTTP) on top of the simulated network
|
|
371
|
+
|
|
372
|
+
Contributions are very welcome: see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
373
|
+
|
|
374
|
+
## License
|
|
375
|
+
|
|
376
|
+
[MIT](LICENSE)
|