flakehound 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flakehound-0.1.0/.gitignore +38 -0
- flakehound-0.1.0/CHANGELOG.md +48 -0
- flakehound-0.1.0/LICENSE +21 -0
- flakehound-0.1.0/PKG-INFO +210 -0
- flakehound-0.1.0/README.md +179 -0
- flakehound-0.1.0/SECURITY.md +26 -0
- flakehound-0.1.0/demo/README.md +17 -0
- flakehound-0.1.0/demo/tests/test_cart.py +11 -0
- flakehound-0.1.0/demo/tests/test_currency.py +12 -0
- flakehound-0.1.0/demo/tests/test_dates.py +18 -0
- flakehound-0.1.0/demo/tests/test_discount.py +5 -0
- flakehound-0.1.0/demo/tests/test_inventory.py +11 -0
- flakehound-0.1.0/demo/tests/test_math.py +9 -0
- flakehound-0.1.0/demo/tests/test_tags.py +6 -0
- flakehound-0.1.0/demo/tests/test_upload.py +10 -0
- flakehound-0.1.0/demo/tests/test_wallet.py +21 -0
- flakehound-0.1.0/pyproject.toml +77 -0
- flakehound-0.1.0/src/flakehound/__init__.py +3 -0
- flakehound-0.1.0/src/flakehound/__main__.py +3 -0
- flakehound-0.1.0/src/flakehound/adapters/__init__.py +0 -0
- flakehound-0.1.0/src/flakehound/adapters/base.py +41 -0
- flakehound-0.1.0/src/flakehound/adapters/pytest/__init__.py +1 -0
- flakehound-0.1.0/src/flakehound/adapters/pytest/clock.py +135 -0
- flakehound-0.1.0/src/flakehound/adapters/pytest/paths.py +53 -0
- flakehound-0.1.0/src/flakehound/adapters/pytest/plugin.py +276 -0
- flakehound-0.1.0/src/flakehound/adapters/pytest/runner.py +208 -0
- flakehound-0.1.0/src/flakehound/adapters/pytest/snapshot.py +205 -0
- flakehound-0.1.0/src/flakehound/adapters/pytest/timing.py +141 -0
- flakehound-0.1.0/src/flakehound/cli.py +250 -0
- flakehound-0.1.0/src/flakehound/cli_quarantine.py +157 -0
- flakehound-0.1.0/src/flakehound/core/__init__.py +0 -0
- flakehound-0.1.0/src/flakehound/core/executor.py +87 -0
- flakehound-0.1.0/src/flakehound/core/models.py +156 -0
- flakehound-0.1.0/src/flakehound/core/procs.py +127 -0
- flakehound-0.1.0/src/flakehound/core/scheduler.py +57 -0
- flakehound-0.1.0/src/flakehound/core/stats.py +22 -0
- flakehound-0.1.0/src/flakehound/detectors/__init__.py +0 -0
- flakehound-0.1.0/src/flakehound/detectors/probes.py +83 -0
- flakehound-0.1.0/src/flakehound/detectors/shake.py +171 -0
- flakehound-0.1.0/src/flakehound/diagnosis/__init__.py +1 -0
- flakehound-0.1.0/src/flakehound/diagnosis/common.py +43 -0
- flakehound-0.1.0/src/flakehound/diagnosis/ddmin.py +62 -0
- flakehound-0.1.0/src/flakehound/diagnosis/env.py +295 -0
- flakehound-0.1.0/src/flakehound/diagnosis/fixes.py +61 -0
- flakehound-0.1.0/src/flakehound/diagnosis/order.py +256 -0
- flakehound-0.1.0/src/flakehound/diagnosis/timing.py +263 -0
- flakehound-0.1.0/src/flakehound/py.typed +0 -0
- flakehound-0.1.0/src/flakehound/quarantine.py +112 -0
- flakehound-0.1.0/src/flakehound/quarantine_plugin.py +49 -0
- flakehound-0.1.0/src/flakehound/report/__init__.py +0 -0
- flakehound-0.1.0/src/flakehound/report/findings.py +181 -0
- flakehound-0.1.0/src/flakehound/report/json_report.py +16 -0
- flakehound-0.1.0/src/flakehound/report/markdown.py +139 -0
- flakehound-0.1.0/src/flakehound/report/redact.py +69 -0
- flakehound-0.1.0/src/flakehound/report/terminal.py +277 -0
- flakehound-0.1.0/src/flakehound/session.py +283 -0
- flakehound-0.1.0/tests/check_demo_report.py +106 -0
- flakehound-0.1.0/tests/conftest.py +34 -0
- flakehound-0.1.0/tests/test_ddmin.py +34 -0
- flakehound-0.1.0/tests/test_env_diagnosis.py +140 -0
- flakehound-0.1.0/tests/test_integration.py +105 -0
- flakehound-0.1.0/tests/test_order_diagnosis.py +140 -0
- flakehound-0.1.0/tests/test_probes_and_clock.py +111 -0
- flakehound-0.1.0/tests/test_procs.py +41 -0
- flakehound-0.1.0/tests/test_quarantine.py +155 -0
- flakehound-0.1.0/tests/test_redact.py +48 -0
- flakehound-0.1.0/tests/test_reports_and_ci.py +107 -0
- flakehound-0.1.0/tests/test_shake.py +50 -0
- flakehound-0.1.0/tests/test_snapshot.py +112 -0
- flakehound-0.1.0/tests/test_stats.py +20 -0
- flakehound-0.1.0/tests/test_timing.py +230 -0
- flakehound-0.1.0/tests/test_workflows.py +45 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Private working context (never pushed)
|
|
2
|
+
PROJECT_CONTEXT.md
|
|
3
|
+
|
|
4
|
+
# Python
|
|
5
|
+
__pycache__/
|
|
6
|
+
*.py[cod]
|
|
7
|
+
*.egg-info/
|
|
8
|
+
.eggs/
|
|
9
|
+
build/
|
|
10
|
+
dist/
|
|
11
|
+
.venv/
|
|
12
|
+
venv/
|
|
13
|
+
|
|
14
|
+
# Tool caches
|
|
15
|
+
.pytest_cache/
|
|
16
|
+
.mypy_cache/
|
|
17
|
+
.ruff_cache/
|
|
18
|
+
.coverage
|
|
19
|
+
htmlcov/
|
|
20
|
+
|
|
21
|
+
# flakehound run artifacts
|
|
22
|
+
.flakehound/runs/
|
|
23
|
+
|
|
24
|
+
# Secrets / local env (never commit)
|
|
25
|
+
.env
|
|
26
|
+
.env.*
|
|
27
|
+
|
|
28
|
+
# OS / editors
|
|
29
|
+
.DS_Store
|
|
30
|
+
Thumbs.db
|
|
31
|
+
.idea/
|
|
32
|
+
.vscode/
|
|
33
|
+
|
|
34
|
+
# Demo output
|
|
35
|
+
demo/report.json
|
|
36
|
+
|
|
37
|
+
# flakehound output files
|
|
38
|
+
demo/report.md
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to flakehound are listed here.
|
|
4
|
+
The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the
|
|
5
|
+
project uses [Semantic Versioning](https://semver.org/) (pre-1.0: minor versions may break things).
|
|
6
|
+
|
|
7
|
+
## [0.1.0] - 2026-11-10
|
|
8
|
+
|
|
9
|
+
First release. Python + pytest.
|
|
10
|
+
|
|
11
|
+
### Detection
|
|
12
|
+
- `flakehound run`: runs the suite many times in parallel with shuffled order and varied
|
|
13
|
+
`random` seed and `PYTHONHASHSEED`; failure rates with 95% confidence intervals.
|
|
14
|
+
- Tests that fail every run are rechecked alone before being called broken.
|
|
15
|
+
- Probe runs find **time bombs** (UTC+14, UTC−12, just before midnight, last day of the
|
|
16
|
+
month, New Year's Eve, Feb 29) and **racy tests** (main-thread sleeps 20× shorter;
|
|
17
|
+
thread-switch stress).
|
|
18
|
+
|
|
19
|
+
### Diagnosis (every verdict backed by real runs)
|
|
20
|
+
- Order dependency: isolation with the failing runs' exact seeds, polluter or
|
|
21
|
+
state-setter search (correlation fast path + parallel delta debugging), and the
|
|
22
|
+
**leaked state** named (module globals, object/class attributes, env vars, cwd,
|
|
23
|
+
`sys.path`, project files, logging).
|
|
24
|
+
- Randomness vs set/dict ordering (hash seed), told apart by swapping one seed at a time.
|
|
25
|
+
- Timezone and date/clock time bombs.
|
|
26
|
+
- Sleep used as a wait, pinned to the exact `sleep()` line; race conditions.
|
|
27
|
+
- Rule-based fix suggestions; repro commands that are run for real and marked *verified*.
|
|
28
|
+
|
|
29
|
+
### Output and CI
|
|
30
|
+
- Terminal report, JSON report (schema v3), Markdown report (`--markdown`), and the
|
|
31
|
+
GitHub Actions job summary.
|
|
32
|
+
- `--fail-on` to choose what fails a build.
|
|
33
|
+
- Quarantine: `.flakehound/quarantine.toml` + pytest plugin (non-strict xfail or skip)
|
|
34
|
+
+ `flakehound quarantine list|add|remove` with diff and confirmation.
|
|
35
|
+
- Works with projects that use pytest-xdist (`-n auto` in addopts).
|
|
36
|
+
|
|
37
|
+
### Security and cost
|
|
38
|
+
- Fully offline, no telemetry, secrets redacted from every report.
|
|
39
|
+
- No shell commands; every child process tree is cleaned up on timeout or Ctrl+C.
|
|
40
|
+
- Runtime dependencies: pytest, rich, and tomli on Python 3.10 only.
|
|
41
|
+
|
|
42
|
+
### Known limitations
|
|
43
|
+
- `asyncio.sleep` is not shortened by the timing probes.
|
|
44
|
+
- State hidden in fixtures, databases or external services is not visible to the leak
|
|
45
|
+
detector (the report says so instead of guessing).
|
|
46
|
+
- Thread stress needs the race to happen around a Python function call.
|
|
47
|
+
|
|
48
|
+
[0.1.0]: https://github.com/urumb/flakehound/releases/tag/v0.1.0
|
flakehound-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Noel Jacob Antony
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: flakehound
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Find flaky tests and prove why they're flaky. Free, local, private.
|
|
5
|
+
Project-URL: Homepage, https://github.com/urumb/flakehound
|
|
6
|
+
Project-URL: Issues, https://github.com/urumb/flakehound/issues
|
|
7
|
+
Project-URL: Security, https://github.com/urumb/flakehound/security/policy
|
|
8
|
+
Project-URL: Changelog, https://github.com/urumb/flakehound/blob/main/CHANGELOG.md
|
|
9
|
+
Author: Noel Jacob Antony
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: ci,flaky,flaky-tests,pytest,test-order,testing
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Framework :: Pytest
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
25
|
+
Classifier: Topic :: Software Development :: Testing
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Requires-Dist: pytest>=7.0
|
|
28
|
+
Requires-Dist: rich>=13.0
|
|
29
|
+
Requires-Dist: tomli>=1.1; python_version < '3.11'
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# flakehound
|
|
33
|
+
|
|
34
|
+
[](https://pypi.org/project/flakehound/)
|
|
35
|
+
[](https://pypi.org/project/flakehound/)
|
|
36
|
+
[](https://github.com/urumb/flakehound/actions/workflows/ci.yml)
|
|
37
|
+
[](https://github.com/urumb/flakehound/actions/workflows/codeql.yml)
|
|
38
|
+
[](https://github.com/urumb/flakehound/blob/main/LICENSE)
|
|
39
|
+
|
|
40
|
+
**Find flaky tests and prove *why* they're flaky.**
|
|
41
|
+
|
|
42
|
+
Flaky tests pass sometimes and fail sometimes with no code change. Most tools just retry
|
|
43
|
+
them, which hides the problem. flakehound runs your pytest suite under deliberately hostile
|
|
44
|
+
conditions, finds the tests that flip, and tells you the cause, with evidence from real runs
|
|
45
|
+
and a command that reproduces it. It also finds problems that haven't bitten you *yet*.
|
|
46
|
+
|
|
47
|
+

|
|
48
|
+
|
|
49
|
+
- 🆓 **Free forever.** No accounts, no servers, no paid services.
|
|
50
|
+
- 🔒 **Local and private.** Offline, no telemetry, secrets redacted from every report.
|
|
51
|
+
- 🧪 **Evidence, not guesses.** Every verdict comes from runs that prove it, and every repro
|
|
52
|
+
command is run for real before it's marked *verified*.
|
|
53
|
+
|
|
54
|
+
## What it finds
|
|
55
|
+
|
|
56
|
+
| Problem | Example | What flakehound tells you |
|
|
57
|
+
|---|---|---|
|
|
58
|
+
| **Order dependency** | a test leaves an item in a shared cart | the polluter test and the leaked state: `shop.cart: [] → ['apple']` |
|
|
59
|
+
| **Needs another test** | a test only passes after a setup-like test ran | the test it secretly depends on |
|
|
60
|
+
| **Randomness** | `assert random.random() < 0.8` | the failing seed, and a repro that replays it |
|
|
61
|
+
| **Set/dict ordering** | comparing `",".join(a_set)` to a string | that `PYTHONHASHSEED` is the cause |
|
|
62
|
+
| **Time bombs** | breaks on the last day of the month, on Feb 29, in another timezone | the exact date/timezone, *before* it happens |
|
|
63
|
+
| **Sleep used as a wait** | `sleep(0.06)` then assert a thread finished | the exact `sleep()` line that will fail on a slow CI machine |
|
|
64
|
+
| **Race conditions** | unlocked read-modify-write from threads | that it fails when threads interleave |
|
|
65
|
+
|
|
66
|
+
## Quickstart
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
pip install flakehound
|
|
70
|
+
cd your-project
|
|
71
|
+
flakehound run
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
That's it: no config. Try it on the included [demo project](https://github.com/urumb/flakehound/tree/main/demo) to see every kind of
|
|
75
|
+
problem. The same results as a Markdown report: [docs/example-report.md](https://github.com/urumb/flakehound/blob/main/docs/example-report.md).
|
|
76
|
+
|
|
77
|
+
> ⚠️ flakehound **runs your test suite** (many times). Only run it on code you trust.
|
|
78
|
+
|
|
79
|
+
### Options
|
|
80
|
+
|
|
81
|
+
| Option | Meaning |
|
|
82
|
+
|---|---|
|
|
83
|
+
| `-n, --runs N` | How many times to run the suite (default 10) |
|
|
84
|
+
| `-j, --workers N` | Parallel runs (default: CPU cores − 1, max 8) |
|
|
85
|
+
| `--seed N` | Replay an earlier session exactly |
|
|
86
|
+
| `--json PATH` | Also write a machine-readable report |
|
|
87
|
+
| `--markdown PATH` | Also write a Markdown report (for PR comments) |
|
|
88
|
+
| `--fail-on KINDS` | What makes exit code 1: `flaky`, `time-bomb`, `racy`, `broken`, `suspected`, `none` (default `flaky,time-bomb,racy`) |
|
|
89
|
+
| `--no-diagnose` | Only detect flaky tests, don't explain them |
|
|
90
|
+
| `--max-diagnose N` | Diagnose at most N flaky tests (default 20) |
|
|
91
|
+
| `--no-probes` | Skip all probe runs (timezones, risky dates, timing) |
|
|
92
|
+
| `--no-timing` | Skip only the timing probes |
|
|
93
|
+
| `--no-step-summary` | Don't add the report to the GitHub Actions job summary |
|
|
94
|
+
| `--timeout S` | Kill a single run after S seconds (default 600) |
|
|
95
|
+
| `-- ARGS` | Anything after `--` goes straight to pytest |
|
|
96
|
+
|
|
97
|
+
Exit codes: `0` no problems, `1` problems found (by default only flaky tests and *confirmed*
|
|
98
|
+
time bombs or racy tests fail a build), `2` usage or collection error.
|
|
99
|
+
|
|
100
|
+
## In CI
|
|
101
|
+
|
|
102
|
+
Hunting runs your suite many times, so run it on a schedule rather than on every push:
|
|
103
|
+
|
|
104
|
+
```yaml
|
|
105
|
+
# .github/workflows/flakehound.yml
|
|
106
|
+
on:
|
|
107
|
+
schedule: [{ cron: "0 3 * * *" }] # nightly
|
|
108
|
+
workflow_dispatch:
|
|
109
|
+
permissions:
|
|
110
|
+
contents: read
|
|
111
|
+
jobs:
|
|
112
|
+
hunt:
|
|
113
|
+
runs-on: ubuntu-latest
|
|
114
|
+
steps:
|
|
115
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
116
|
+
with: { persist-credentials: false }
|
|
117
|
+
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
|
|
118
|
+
with: { python-version: "3.12" }
|
|
119
|
+
- run: pip install -e . flakehound
|
|
120
|
+
- run: flakehound run --json flakehound.json
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
In GitHub Actions the Markdown report appears on the run's summary page automatically.
|
|
124
|
+
|
|
125
|
+
## Quarantine
|
|
126
|
+
|
|
127
|
+
Found a flaky test you can't fix today? Quarantine it so it stops failing the build, without
|
|
128
|
+
hiding it:
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
flakehound quarantine add tests/test_cart.py::test_cart_starts_empty --reason "polluted by test_add_item"
|
|
132
|
+
flakehound quarantine add --from-report flakehound.json # everything flakehound found
|
|
133
|
+
flakehound quarantine list
|
|
134
|
+
flakehound quarantine remove tests/test_cart.py::test_cart_starts_empty
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
- The list is one file, `.flakehound/quarantine.toml`: commit it, review it, revert it.
|
|
138
|
+
flakehound never edits your test files.
|
|
139
|
+
- Every change shows a diff and asks first (`--yes` skips the prompt).
|
|
140
|
+
- pytest still runs quarantined tests as non-strict `xfail` (or `skip`), so you see `XPASS`
|
|
141
|
+
once one is fixed. Disable for one run with `pytest -p no:flakehound-quarantine`. The
|
|
142
|
+
plugin does nothing when the file doesn't exist.
|
|
143
|
+
- `flakehound run` ignores the quarantine, so quarantined tests keep being hunted.
|
|
144
|
+
|
|
145
|
+
## How it works
|
|
146
|
+
|
|
147
|
+
flakehound changes **one factor at a time**, like a science experiment:
|
|
148
|
+
|
|
149
|
+
1. **Shake.** Run the whole suite many times in parallel, each with a different test order,
|
|
150
|
+
`random` seed and `PYTHONHASHSEED`, all recorded. Tests that both pass and fail are
|
|
151
|
+
flaky; tests that always fail are rechecked, then reported as broken.
|
|
152
|
+
2. **Isolate.** Run each flaky test alone with the *exact seeds* of the runs where it
|
|
153
|
+
failed. Only the order changed, so if it now passes, order is proven to be the cause.
|
|
154
|
+
3. **Find the culprit.** Try the tests whose position best correlates with the failure,
|
|
155
|
+
then shrink the preceding tests with delta debugging, in parallel batches.
|
|
156
|
+
4. **Name the leaked state.** Snapshot global state before and after the culprit: module
|
|
157
|
+
variables, object and class attributes, environment variables (secret values are never
|
|
158
|
+
shown), working directory, `sys.path`, project files and logging config.
|
|
159
|
+
5. **Seeds.** For flaky tests that aren't about order, swap *one* seed at a time between a
|
|
160
|
+
failing and a passing run: the one that flips the result is the cause. If identical
|
|
161
|
+
seeds still give different results, the test is nondeterministic.
|
|
162
|
+
6. **Time bomb probes.** Runs with the baseline's exact order and seeds, but in UTC+14,
|
|
163
|
+
UTC−12, just before midnight, on the last day of the month, on New Year's Eve and on
|
|
164
|
+
Feb 29. The clock is *shifted*, not frozen, so code that waits on time still works.
|
|
165
|
+
7. **Timing probes.** One run where every `sleep()` on the test's main thread is 20×
|
|
166
|
+
shorter (background work keeps its real speed, like a slow machine), and one where
|
|
167
|
+
threads switch far more often. Then one `sleep()` call site at a time, to find the line.
|
|
168
|
+
8. **Confirm and verify.** Findings are re-run, and each printed repro command is run for
|
|
169
|
+
real; it's marked *verified* only if it fails exactly as printed.
|
|
170
|
+
|
|
171
|
+
Repro commands that replay a seed, clock or timing stress use flakehound's pytest plugin
|
|
172
|
+
(`-p flakehound.adapters.pytest.plugin`), so they need flakehound installed. That plugin
|
|
173
|
+
never loads into your normal pytest runs on its own.
|
|
174
|
+
|
|
175
|
+
## How it compares
|
|
176
|
+
|
|
177
|
+
| Tool | Retries | Detects | Explains why | Free & local |
|
|
178
|
+
|---|---|---|---|---|
|
|
179
|
+
| pytest-rerunfailures, flaky | ✅ | ❌ | ❌ | ✅ |
|
|
180
|
+
| pytest-randomly | ❌ | order/seed issues show up | ❌ | ✅ |
|
|
181
|
+
| detect-test-pollution | ❌ | ❌ | the polluter test (you drive it) | ✅ |
|
|
182
|
+
| Trunk, BuildPulse, Datadog | ✅ | from CI history | little | ❌ paid / uploads data |
|
|
183
|
+
| **flakehound** | ❌ (on purpose) | ✅ + time bombs + racy tests | polluter, leaked state, seed, date, timezone, `sleep()` line, race | ✅ |
|
|
184
|
+
|
|
185
|
+
## Limitations
|
|
186
|
+
|
|
187
|
+
- Python + pytest only, for now.
|
|
188
|
+
- `asyncio.sleep` is not shortened by the timing probes.
|
|
189
|
+
- State hidden in fixtures, databases or external services can't be seen by the leak
|
|
190
|
+
detector; the report says so instead of guessing.
|
|
191
|
+
- Thread stress exposes races that happen around Python function calls.
|
|
192
|
+
- Hunting takes many runs of your suite: use `-n`, `-j` and `--max-diagnose` on large suites.
|
|
193
|
+
|
|
194
|
+
## Roadmap
|
|
195
|
+
|
|
196
|
+
- GitHub Actions history mode: find flaky tests from past CI runs without running anything
|
|
197
|
+
- A ready-made GitHub Action
|
|
198
|
+
- Adapters for Jest/Vitest, Go test and JUnit
|
|
199
|
+
- Optional AI explanations (local models via Ollama, or your own key; never required)
|
|
200
|
+
|
|
201
|
+
See [CHANGELOG.md](https://github.com/urumb/flakehound/blob/main/CHANGELOG.md) for what's in each release.
|
|
202
|
+
|
|
203
|
+
## Contributing and security
|
|
204
|
+
|
|
205
|
+
Contributions are welcome: see [CONTRIBUTING.md](https://github.com/urumb/flakehound/blob/main/CONTRIBUTING.md). To report a
|
|
206
|
+
vulnerability privately, see [SECURITY.md](https://github.com/urumb/flakehound/blob/main/SECURITY.md).
|
|
207
|
+
|
|
208
|
+
## License
|
|
209
|
+
|
|
210
|
+
MIT
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# flakehound
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/flakehound/)
|
|
4
|
+
[](https://pypi.org/project/flakehound/)
|
|
5
|
+
[](https://github.com/urumb/flakehound/actions/workflows/ci.yml)
|
|
6
|
+
[](https://github.com/urumb/flakehound/actions/workflows/codeql.yml)
|
|
7
|
+
[](https://github.com/urumb/flakehound/blob/main/LICENSE)
|
|
8
|
+
|
|
9
|
+
**Find flaky tests and prove *why* they're flaky.**
|
|
10
|
+
|
|
11
|
+
Flaky tests pass sometimes and fail sometimes with no code change. Most tools just retry
|
|
12
|
+
them, which hides the problem. flakehound runs your pytest suite under deliberately hostile
|
|
13
|
+
conditions, finds the tests that flip, and tells you the cause, with evidence from real runs
|
|
14
|
+
and a command that reproduces it. It also finds problems that haven't bitten you *yet*.
|
|
15
|
+
|
|
16
|
+

|
|
17
|
+
|
|
18
|
+
- 🆓 **Free forever.** No accounts, no servers, no paid services.
|
|
19
|
+
- 🔒 **Local and private.** Offline, no telemetry, secrets redacted from every report.
|
|
20
|
+
- 🧪 **Evidence, not guesses.** Every verdict comes from runs that prove it, and every repro
|
|
21
|
+
command is run for real before it's marked *verified*.
|
|
22
|
+
|
|
23
|
+
## What it finds
|
|
24
|
+
|
|
25
|
+
| Problem | Example | What flakehound tells you |
|
|
26
|
+
|---|---|---|
|
|
27
|
+
| **Order dependency** | a test leaves an item in a shared cart | the polluter test and the leaked state: `shop.cart: [] → ['apple']` |
|
|
28
|
+
| **Needs another test** | a test only passes after a setup-like test ran | the test it secretly depends on |
|
|
29
|
+
| **Randomness** | `assert random.random() < 0.8` | the failing seed, and a repro that replays it |
|
|
30
|
+
| **Set/dict ordering** | comparing `",".join(a_set)` to a string | that `PYTHONHASHSEED` is the cause |
|
|
31
|
+
| **Time bombs** | breaks on the last day of the month, on Feb 29, in another timezone | the exact date/timezone, *before* it happens |
|
|
32
|
+
| **Sleep used as a wait** | `sleep(0.06)` then assert a thread finished | the exact `sleep()` line that will fail on a slow CI machine |
|
|
33
|
+
| **Race conditions** | unlocked read-modify-write from threads | that it fails when threads interleave |
|
|
34
|
+
|
|
35
|
+
## Quickstart
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install flakehound
|
|
39
|
+
cd your-project
|
|
40
|
+
flakehound run
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
That's it: no config. Try it on the included [demo project](https://github.com/urumb/flakehound/tree/main/demo) to see every kind of
|
|
44
|
+
problem. The same results as a Markdown report: [docs/example-report.md](https://github.com/urumb/flakehound/blob/main/docs/example-report.md).
|
|
45
|
+
|
|
46
|
+
> ⚠️ flakehound **runs your test suite** (many times). Only run it on code you trust.
|
|
47
|
+
|
|
48
|
+
### Options
|
|
49
|
+
|
|
50
|
+
| Option | Meaning |
|
|
51
|
+
|---|---|
|
|
52
|
+
| `-n, --runs N` | How many times to run the suite (default 10) |
|
|
53
|
+
| `-j, --workers N` | Parallel runs (default: CPU cores − 1, max 8) |
|
|
54
|
+
| `--seed N` | Replay an earlier session exactly |
|
|
55
|
+
| `--json PATH` | Also write a machine-readable report |
|
|
56
|
+
| `--markdown PATH` | Also write a Markdown report (for PR comments) |
|
|
57
|
+
| `--fail-on KINDS` | What makes exit code 1: `flaky`, `time-bomb`, `racy`, `broken`, `suspected`, `none` (default `flaky,time-bomb,racy`) |
|
|
58
|
+
| `--no-diagnose` | Only detect flaky tests, don't explain them |
|
|
59
|
+
| `--max-diagnose N` | Diagnose at most N flaky tests (default 20) |
|
|
60
|
+
| `--no-probes` | Skip all probe runs (timezones, risky dates, timing) |
|
|
61
|
+
| `--no-timing` | Skip only the timing probes |
|
|
62
|
+
| `--no-step-summary` | Don't add the report to the GitHub Actions job summary |
|
|
63
|
+
| `--timeout S` | Kill a single run after S seconds (default 600) |
|
|
64
|
+
| `-- ARGS` | Anything after `--` goes straight to pytest |
|
|
65
|
+
|
|
66
|
+
Exit codes: `0` no problems, `1` problems found (by default only flaky tests and *confirmed*
|
|
67
|
+
time bombs or racy tests fail a build), `2` usage or collection error.
|
|
68
|
+
|
|
69
|
+
## In CI
|
|
70
|
+
|
|
71
|
+
Hunting runs your suite many times, so run it on a schedule rather than on every push:
|
|
72
|
+
|
|
73
|
+
```yaml
|
|
74
|
+
# .github/workflows/flakehound.yml
|
|
75
|
+
on:
|
|
76
|
+
schedule: [{ cron: "0 3 * * *" }] # nightly
|
|
77
|
+
workflow_dispatch:
|
|
78
|
+
permissions:
|
|
79
|
+
contents: read
|
|
80
|
+
jobs:
|
|
81
|
+
hunt:
|
|
82
|
+
runs-on: ubuntu-latest
|
|
83
|
+
steps:
|
|
84
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
85
|
+
with: { persist-credentials: false }
|
|
86
|
+
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
|
|
87
|
+
with: { python-version: "3.12" }
|
|
88
|
+
- run: pip install -e . flakehound
|
|
89
|
+
- run: flakehound run --json flakehound.json
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
In GitHub Actions the Markdown report appears on the run's summary page automatically.
|
|
93
|
+
|
|
94
|
+
## Quarantine
|
|
95
|
+
|
|
96
|
+
Found a flaky test you can't fix today? Quarantine it so it stops failing the build, without
|
|
97
|
+
hiding it:
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
flakehound quarantine add tests/test_cart.py::test_cart_starts_empty --reason "polluted by test_add_item"
|
|
101
|
+
flakehound quarantine add --from-report flakehound.json # everything flakehound found
|
|
102
|
+
flakehound quarantine list
|
|
103
|
+
flakehound quarantine remove tests/test_cart.py::test_cart_starts_empty
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
- The list is one file, `.flakehound/quarantine.toml`: commit it, review it, revert it.
|
|
107
|
+
flakehound never edits your test files.
|
|
108
|
+
- Every change shows a diff and asks first (`--yes` skips the prompt).
|
|
109
|
+
- pytest still runs quarantined tests as non-strict `xfail` (or `skip`), so you see `XPASS`
|
|
110
|
+
once one is fixed. Disable for one run with `pytest -p no:flakehound-quarantine`. The
|
|
111
|
+
plugin does nothing when the file doesn't exist.
|
|
112
|
+
- `flakehound run` ignores the quarantine, so quarantined tests keep being hunted.
|
|
113
|
+
|
|
114
|
+
## How it works
|
|
115
|
+
|
|
116
|
+
flakehound changes **one factor at a time**, like a science experiment:
|
|
117
|
+
|
|
118
|
+
1. **Shake.** Run the whole suite many times in parallel, each with a different test order,
|
|
119
|
+
`random` seed and `PYTHONHASHSEED`, all recorded. Tests that both pass and fail are
|
|
120
|
+
flaky; tests that always fail are rechecked, then reported as broken.
|
|
121
|
+
2. **Isolate.** Run each flaky test alone with the *exact seeds* of the runs where it
|
|
122
|
+
failed. Only the order changed, so if it now passes, order is proven to be the cause.
|
|
123
|
+
3. **Find the culprit.** Try the tests whose position best correlates with the failure,
|
|
124
|
+
then shrink the preceding tests with delta debugging, in parallel batches.
|
|
125
|
+
4. **Name the leaked state.** Snapshot global state before and after the culprit: module
|
|
126
|
+
variables, object and class attributes, environment variables (secret values are never
|
|
127
|
+
shown), working directory, `sys.path`, project files and logging config.
|
|
128
|
+
5. **Seeds.** For flaky tests that aren't about order, swap *one* seed at a time between a
|
|
129
|
+
failing and a passing run: the one that flips the result is the cause. If identical
|
|
130
|
+
seeds still give different results, the test is nondeterministic.
|
|
131
|
+
6. **Time bomb probes.** Runs with the baseline's exact order and seeds, but in UTC+14,
|
|
132
|
+
UTC−12, just before midnight, on the last day of the month, on New Year's Eve and on
|
|
133
|
+
Feb 29. The clock is *shifted*, not frozen, so code that waits on time still works.
|
|
134
|
+
7. **Timing probes.** One run where every `sleep()` on the test's main thread is 20×
|
|
135
|
+
shorter (background work keeps its real speed, like a slow machine), and one where
|
|
136
|
+
threads switch far more often. Then one `sleep()` call site at a time, to find the line.
|
|
137
|
+
8. **Confirm and verify.** Findings are re-run, and each printed repro command is run for
|
|
138
|
+
real; it's marked *verified* only if it fails exactly as printed.
|
|
139
|
+
|
|
140
|
+
Repro commands that replay a seed, clock or timing stress use flakehound's pytest plugin
|
|
141
|
+
(`-p flakehound.adapters.pytest.plugin`), so they need flakehound installed. That plugin
|
|
142
|
+
never loads into your normal pytest runs on its own.
|
|
143
|
+
|
|
144
|
+
## How it compares
|
|
145
|
+
|
|
146
|
+
| Tool | Retries | Detects | Explains why | Free & local |
|
|
147
|
+
|---|---|---|---|---|
|
|
148
|
+
| pytest-rerunfailures, flaky | ✅ | ❌ | ❌ | ✅ |
|
|
149
|
+
| pytest-randomly | ❌ | order/seed issues show up | ❌ | ✅ |
|
|
150
|
+
| detect-test-pollution | ❌ | ❌ | the polluter test (you drive it) | ✅ |
|
|
151
|
+
| Trunk, BuildPulse, Datadog | ✅ | from CI history | little | ❌ paid / uploads data |
|
|
152
|
+
| **flakehound** | ❌ (on purpose) | ✅ + time bombs + racy tests | polluter, leaked state, seed, date, timezone, `sleep()` line, race | ✅ |
|
|
153
|
+
|
|
154
|
+
## Limitations
|
|
155
|
+
|
|
156
|
+
- Python + pytest only, for now.
|
|
157
|
+
- `asyncio.sleep` is not shortened by the timing probes.
|
|
158
|
+
- State hidden in fixtures, databases or external services can't be seen by the leak
|
|
159
|
+
detector; the report says so instead of guessing.
|
|
160
|
+
- Thread stress exposes races that happen around Python function calls.
|
|
161
|
+
- Hunting takes many runs of your suite: use `-n`, `-j` and `--max-diagnose` on large suites.
|
|
162
|
+
|
|
163
|
+
## Roadmap
|
|
164
|
+
|
|
165
|
+
- GitHub Actions history mode: find flaky tests from past CI runs without running anything
|
|
166
|
+
- A ready-made GitHub Action
|
|
167
|
+
- Adapters for Jest/Vitest, Go test and JUnit
|
|
168
|
+
- Optional AI explanations (local models via Ollama, or your own key; never required)
|
|
169
|
+
|
|
170
|
+
See [CHANGELOG.md](https://github.com/urumb/flakehound/blob/main/CHANGELOG.md) for what's in each release.
|
|
171
|
+
|
|
172
|
+
## Contributing and security
|
|
173
|
+
|
|
174
|
+
Contributions are welcome: see [CONTRIBUTING.md](https://github.com/urumb/flakehound/blob/main/CONTRIBUTING.md). To report a
|
|
175
|
+
vulnerability privately, see [SECURITY.md](https://github.com/urumb/flakehound/blob/main/SECURITY.md).
|
|
176
|
+
|
|
177
|
+
## License
|
|
178
|
+
|
|
179
|
+
MIT
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Security Policy
|
|
2
|
+
|
|
3
|
+
## Reporting a vulnerability
|
|
4
|
+
|
|
5
|
+
Please **do not open a public issue** for security problems.
|
|
6
|
+
|
|
7
|
+
Report privately through GitHub's
|
|
8
|
+
[private vulnerability reporting](https://github.com/urumb/flakehound/security/advisories/new).
|
|
9
|
+
You should get a response within 7 days. Please include steps to reproduce and the affected version.
|
|
10
|
+
|
|
11
|
+
## Supported versions
|
|
12
|
+
|
|
13
|
+
Only the latest release receives security fixes while the project is pre-1.0.
|
|
14
|
+
|
|
15
|
+
## Security model
|
|
16
|
+
|
|
17
|
+
- **flakehound runs your test suite, which is arbitrary code.** Only run it on code you trust.
|
|
18
|
+
Never point it at untrusted pull requests or repositories without a sandbox.
|
|
19
|
+
- **Offline by default.** flakehound makes no network calls and collects no telemetry.
|
|
20
|
+
Features that need the network (such as future GitHub history mode or cloud AI) are opt-in.
|
|
21
|
+
- **No secrets in output.** Reports never include environment variables; values that look like
|
|
22
|
+
tokens, keys or passwords are redacted.
|
|
23
|
+
- **Safe subprocesses.** Tests run via argument lists, never `shell=True`. Child processes and
|
|
24
|
+
temporary files are cleaned up, even on Ctrl+C.
|
|
25
|
+
- **Supply chain.** Minimal dependencies, releases published with PyPI Trusted Publishing
|
|
26
|
+
(no long-lived tokens), GitHub Actions pinned to commit SHAs with least-privilege permissions.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# flakehound demo
|
|
2
|
+
|
|
3
|
+
A tiny shop whose tests are flaky on purpose. Each file is one classic trap:
|
|
4
|
+
|
|
5
|
+
| Test file | Trap |
|
|
6
|
+
|---|---|
|
|
7
|
+
| `tests/test_cart.py` | Order dependency: `test_add_item` leaves an item in the shared cart |
|
|
8
|
+
| `tests/test_currency.py` | Order dependency: `test_euro_store` leaks the `SHOP_CURRENCY` env var |
|
|
9
|
+
| `tests/test_inventory.py` | Needs another test: `test_apples_in_stock` only passes after `test_restock_apples` |
|
|
10
|
+
| `tests/test_discount.py` | Unseeded randomness |
|
|
11
|
+
| `tests/test_tags.py` | Set iteration order depends on `PYTHONHASHSEED` |
|
|
12
|
+
| `tests/test_upload.py` | Timing: `sleep()` used to wait for a background thread |
|
|
13
|
+
| `tests/test_wallet.py` | Race condition: unlocked read-modify-write from several threads |
|
|
14
|
+
| `tests/test_dates.py` | Time bombs: mixes local time and UTC, breaks on the last day of the month and on Feb 29 |
|
|
15
|
+
| `tests/test_math.py` | Stable tests, for contrast |
|
|
16
|
+
|
|
17
|
+
Run `flakehound run` in this folder.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import shop
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def test_cart_starts_empty():
|
|
5
|
+
# Passes in the default order, fails whenever test_add_item ran first.
|
|
6
|
+
assert shop.cart == []
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_add_item():
|
|
10
|
+
shop.add_item("apple") # never cleaned up: leaks into later tests
|
|
11
|
+
assert "apple" in shop.cart
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from datetime import date, datetime, timedelta, timezone
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def test_order_date_matches_utc():
|
|
5
|
+
# Bug: mixes local time and UTC. Fails wherever the local date differs from
|
|
6
|
+
# the UTC date (e.g. a CI machine in another timezone, or late evenings).
|
|
7
|
+
assert datetime.now().date() == datetime.now(timezone.utc).date()
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_coupon_valid_until_tomorrow():
|
|
11
|
+
# Bug: assumes tomorrow is in the same month. False on the last day of every month.
|
|
12
|
+
assert (date.today() + timedelta(days=1)).month == date.today().month
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_anniversary_next_year():
|
|
16
|
+
today = date.today()
|
|
17
|
+
# Bug: Feb 29 plus one year does not exist -> ValueError on leap days.
|
|
18
|
+
assert today.replace(year=today.year + 1) > today
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import shop
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def test_restock_apples():
|
|
5
|
+
shop.Inventory.restock("apple", 5)
|
|
6
|
+
assert shop.Inventory.stock["apple"] >= 5
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_apples_in_stock():
|
|
10
|
+
# Bug: silently relies on test_restock_apples having run first.
|
|
11
|
+
assert shop.Inventory.stock.get("apple", 0) > 0
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import threading
|
|
2
|
+
|
|
3
|
+
import shop
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_concurrent_top_ups():
|
|
7
|
+
wallet = shop.Wallet()
|
|
8
|
+
start = threading.Barrier(4) # all four "users" pay at the same moment
|
|
9
|
+
|
|
10
|
+
def worker():
|
|
11
|
+
start.wait()
|
|
12
|
+
for _ in range(2000):
|
|
13
|
+
wallet.top_up(1)
|
|
14
|
+
|
|
15
|
+
threads = [threading.Thread(target=worker) for _ in range(4)]
|
|
16
|
+
for t in threads:
|
|
17
|
+
t.start()
|
|
18
|
+
for t in threads:
|
|
19
|
+
t.join()
|
|
20
|
+
# Bug: top_up is not thread-safe. Passes almost always, until threads interleave.
|
|
21
|
+
assert wallet.balance == 8000
|