flakehound 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. flakehound-0.1.0/.gitignore +38 -0
  2. flakehound-0.1.0/CHANGELOG.md +48 -0
  3. flakehound-0.1.0/LICENSE +21 -0
  4. flakehound-0.1.0/PKG-INFO +210 -0
  5. flakehound-0.1.0/README.md +179 -0
  6. flakehound-0.1.0/SECURITY.md +26 -0
  7. flakehound-0.1.0/demo/README.md +17 -0
  8. flakehound-0.1.0/demo/tests/test_cart.py +11 -0
  9. flakehound-0.1.0/demo/tests/test_currency.py +12 -0
  10. flakehound-0.1.0/demo/tests/test_dates.py +18 -0
  11. flakehound-0.1.0/demo/tests/test_discount.py +5 -0
  12. flakehound-0.1.0/demo/tests/test_inventory.py +11 -0
  13. flakehound-0.1.0/demo/tests/test_math.py +9 -0
  14. flakehound-0.1.0/demo/tests/test_tags.py +6 -0
  15. flakehound-0.1.0/demo/tests/test_upload.py +10 -0
  16. flakehound-0.1.0/demo/tests/test_wallet.py +21 -0
  17. flakehound-0.1.0/pyproject.toml +77 -0
  18. flakehound-0.1.0/src/flakehound/__init__.py +3 -0
  19. flakehound-0.1.0/src/flakehound/__main__.py +3 -0
  20. flakehound-0.1.0/src/flakehound/adapters/__init__.py +0 -0
  21. flakehound-0.1.0/src/flakehound/adapters/base.py +41 -0
  22. flakehound-0.1.0/src/flakehound/adapters/pytest/__init__.py +1 -0
  23. flakehound-0.1.0/src/flakehound/adapters/pytest/clock.py +135 -0
  24. flakehound-0.1.0/src/flakehound/adapters/pytest/paths.py +53 -0
  25. flakehound-0.1.0/src/flakehound/adapters/pytest/plugin.py +276 -0
  26. flakehound-0.1.0/src/flakehound/adapters/pytest/runner.py +208 -0
  27. flakehound-0.1.0/src/flakehound/adapters/pytest/snapshot.py +205 -0
  28. flakehound-0.1.0/src/flakehound/adapters/pytest/timing.py +141 -0
  29. flakehound-0.1.0/src/flakehound/cli.py +250 -0
  30. flakehound-0.1.0/src/flakehound/cli_quarantine.py +157 -0
  31. flakehound-0.1.0/src/flakehound/core/__init__.py +0 -0
  32. flakehound-0.1.0/src/flakehound/core/executor.py +87 -0
  33. flakehound-0.1.0/src/flakehound/core/models.py +156 -0
  34. flakehound-0.1.0/src/flakehound/core/procs.py +127 -0
  35. flakehound-0.1.0/src/flakehound/core/scheduler.py +57 -0
  36. flakehound-0.1.0/src/flakehound/core/stats.py +22 -0
  37. flakehound-0.1.0/src/flakehound/detectors/__init__.py +0 -0
  38. flakehound-0.1.0/src/flakehound/detectors/probes.py +83 -0
  39. flakehound-0.1.0/src/flakehound/detectors/shake.py +171 -0
  40. flakehound-0.1.0/src/flakehound/diagnosis/__init__.py +1 -0
  41. flakehound-0.1.0/src/flakehound/diagnosis/common.py +43 -0
  42. flakehound-0.1.0/src/flakehound/diagnosis/ddmin.py +62 -0
  43. flakehound-0.1.0/src/flakehound/diagnosis/env.py +295 -0
  44. flakehound-0.1.0/src/flakehound/diagnosis/fixes.py +61 -0
  45. flakehound-0.1.0/src/flakehound/diagnosis/order.py +256 -0
  46. flakehound-0.1.0/src/flakehound/diagnosis/timing.py +263 -0
  47. flakehound-0.1.0/src/flakehound/py.typed +0 -0
  48. flakehound-0.1.0/src/flakehound/quarantine.py +112 -0
  49. flakehound-0.1.0/src/flakehound/quarantine_plugin.py +49 -0
  50. flakehound-0.1.0/src/flakehound/report/__init__.py +0 -0
  51. flakehound-0.1.0/src/flakehound/report/findings.py +181 -0
  52. flakehound-0.1.0/src/flakehound/report/json_report.py +16 -0
  53. flakehound-0.1.0/src/flakehound/report/markdown.py +139 -0
  54. flakehound-0.1.0/src/flakehound/report/redact.py +69 -0
  55. flakehound-0.1.0/src/flakehound/report/terminal.py +277 -0
  56. flakehound-0.1.0/src/flakehound/session.py +283 -0
  57. flakehound-0.1.0/tests/check_demo_report.py +106 -0
  58. flakehound-0.1.0/tests/conftest.py +34 -0
  59. flakehound-0.1.0/tests/test_ddmin.py +34 -0
  60. flakehound-0.1.0/tests/test_env_diagnosis.py +140 -0
  61. flakehound-0.1.0/tests/test_integration.py +105 -0
  62. flakehound-0.1.0/tests/test_order_diagnosis.py +140 -0
  63. flakehound-0.1.0/tests/test_probes_and_clock.py +111 -0
  64. flakehound-0.1.0/tests/test_procs.py +41 -0
  65. flakehound-0.1.0/tests/test_quarantine.py +155 -0
  66. flakehound-0.1.0/tests/test_redact.py +48 -0
  67. flakehound-0.1.0/tests/test_reports_and_ci.py +107 -0
  68. flakehound-0.1.0/tests/test_shake.py +50 -0
  69. flakehound-0.1.0/tests/test_snapshot.py +112 -0
  70. flakehound-0.1.0/tests/test_stats.py +20 -0
  71. flakehound-0.1.0/tests/test_timing.py +230 -0
  72. flakehound-0.1.0/tests/test_workflows.py +45 -0
@@ -0,0 +1,38 @@
1
+ # Private working context (never pushed)
2
+ PROJECT_CONTEXT.md
3
+
4
+ # Python
5
+ __pycache__/
6
+ *.py[cod]
7
+ *.egg-info/
8
+ .eggs/
9
+ build/
10
+ dist/
11
+ .venv/
12
+ venv/
13
+
14
+ # Tool caches
15
+ .pytest_cache/
16
+ .mypy_cache/
17
+ .ruff_cache/
18
+ .coverage
19
+ htmlcov/
20
+
21
+ # flakehound run artifacts
22
+ .flakehound/runs/
23
+
24
+ # Secrets / local env (never commit)
25
+ .env
26
+ .env.*
27
+
28
+ # OS / editors
29
+ .DS_Store
30
+ Thumbs.db
31
+ .idea/
32
+ .vscode/
33
+
34
+ # Demo output
35
+ demo/report.json
36
+
37
+ # flakehound output files
38
+ demo/report.md
@@ -0,0 +1,48 @@
1
+ # Changelog
2
+
3
+ All notable changes to flakehound are listed here.
4
+ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the
5
+ project uses [Semantic Versioning](https://semver.org/) (pre-1.0: minor versions may break things).
6
+
7
+ ## [0.1.0] - 2026-11-10
8
+
9
+ First release. Python + pytest.
10
+
11
+ ### Detection
12
+ - `flakehound run`: runs the suite many times in parallel with shuffled order and varied
13
+ `random` seed and `PYTHONHASHSEED`; failure rates with 95% confidence intervals.
14
+ - Tests that fail every run are rechecked alone before being called broken.
15
+ - Probe runs find **time bombs** (UTC+14, UTC−12, just before midnight, last day of the
16
+ month, New Year's Eve, Feb 29) and **racy tests** (main-thread sleeps 20× shorter;
17
+ thread-switch stress).
18
+
19
+ ### Diagnosis (every verdict backed by real runs)
20
+ - Order dependency: isolation with the failing runs' exact seeds, polluter or
21
+ state-setter search (correlation fast path + parallel delta debugging), and the
22
+ **leaked state** named (module globals, object/class attributes, env vars, cwd,
23
+ `sys.path`, project files, logging).
24
+ - Randomness vs set/dict ordering (hash seed), told apart by swapping one seed at a time.
25
+ - Timezone and date/clock time bombs.
26
+ - Sleep used as a wait, pinned to the exact `sleep()` line; race conditions.
27
+ - Rule-based fix suggestions; repro commands that are run for real and marked *verified*.
28
+
29
+ ### Output and CI
30
+ - Terminal report, JSON report (schema v3), Markdown report (`--markdown`), and the
31
+ GitHub Actions job summary.
32
+ - `--fail-on` to choose what fails a build.
33
+ - Quarantine: `.flakehound/quarantine.toml` + pytest plugin (non-strict xfail or skip)
34
+ + `flakehound quarantine list|add|remove` with diff and confirmation.
35
+ - Works with projects that use pytest-xdist (`-n auto` in addopts).
36
+
37
+ ### Security and cost
38
+ - Fully offline, no telemetry, secrets redacted from every report.
39
+ - No shell commands; every child process tree is cleaned up on timeout or Ctrl+C.
40
+ - Runtime dependencies: pytest, rich, and tomli on Python 3.10 only.
41
+
42
+ ### Known limitations
43
+ - `asyncio.sleep` is not shortened by the timing probes.
44
+ - State hidden in fixtures, databases or external services is not visible to the leak
45
+ detector (the report says so instead of guessing).
46
+ - Thread stress needs the race to happen around a Python function call.
47
+
48
+ [0.1.0]: https://github.com/urumb/flakehound/releases/tag/v0.1.0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Noel Jacob Antony
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,210 @@
1
+ Metadata-Version: 2.5
2
+ Name: flakehound
3
+ Version: 0.1.0
4
+ Summary: Find flaky tests and prove why they're flaky. Free, local, private.
5
+ Project-URL: Homepage, https://github.com/urumb/flakehound
6
+ Project-URL: Issues, https://github.com/urumb/flakehound/issues
7
+ Project-URL: Security, https://github.com/urumb/flakehound/security/policy
8
+ Project-URL: Changelog, https://github.com/urumb/flakehound/blob/main/CHANGELOG.md
9
+ Author: Noel Jacob Antony
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ Keywords: ci,flaky,flaky-tests,pytest,test-order,testing
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: Environment :: Console
15
+ Classifier: Framework :: Pytest
16
+ Classifier: Intended Audience :: Developers
17
+ Classifier: Operating System :: OS Independent
18
+ Classifier: Programming Language :: Python :: 3
19
+ Classifier: Programming Language :: Python :: 3 :: Only
20
+ Classifier: Programming Language :: Python :: 3.10
21
+ Classifier: Programming Language :: Python :: 3.11
22
+ Classifier: Programming Language :: Python :: 3.12
23
+ Classifier: Programming Language :: Python :: 3.13
24
+ Classifier: Programming Language :: Python :: 3.14
25
+ Classifier: Topic :: Software Development :: Testing
26
+ Requires-Python: >=3.10
27
+ Requires-Dist: pytest>=7.0
28
+ Requires-Dist: rich>=13.0
29
+ Requires-Dist: tomli>=1.1; python_version < '3.11'
30
+ Description-Content-Type: text/markdown
31
+
32
+ # flakehound
33
+
34
+ [![PyPI](https://img.shields.io/pypi/v/flakehound.svg)](https://pypi.org/project/flakehound/)
35
+ [![Python](https://img.shields.io/pypi/pyversions/flakehound.svg)](https://pypi.org/project/flakehound/)
36
+ [![CI](https://github.com/urumb/flakehound/actions/workflows/ci.yml/badge.svg)](https://github.com/urumb/flakehound/actions/workflows/ci.yml)
37
+ [![CodeQL](https://github.com/urumb/flakehound/actions/workflows/codeql.yml/badge.svg)](https://github.com/urumb/flakehound/actions/workflows/codeql.yml)
38
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/urumb/flakehound/blob/main/LICENSE)
39
+
40
+ **Find flaky tests and prove *why* they're flaky.**
41
+
42
+ Flaky tests pass sometimes and fail sometimes with no code change. Most tools just retry
43
+ them, which hides the problem. flakehound runs your pytest suite under deliberately hostile
44
+ conditions, finds the tests that flip, and tells you the cause, with evidence from real runs
45
+ and a command that reproduces it. It also finds problems that haven't bitten you *yet*.
46
+
47
+ ![flakehound output on the demo project](https://raw.githubusercontent.com/urumb/flakehound/main/docs/demo.svg)
48
+
49
+ - 🆓 **Free forever.** No accounts, no servers, no paid services.
50
+ - 🔒 **Local and private.** Offline, no telemetry, secrets redacted from every report.
51
+ - 🧪 **Evidence, not guesses.** Every verdict comes from runs that prove it, and every repro
52
+ command is run for real before it's marked *verified*.
53
+
54
+ ## What it finds
55
+
56
+ | Problem | Example | What flakehound tells you |
57
+ |---|---|---|
58
+ | **Order dependency** | a test leaves an item in a shared cart | the polluter test and the leaked state: `shop.cart: [] → ['apple']` |
59
+ | **Needs another test** | a test only passes after a setup-like test ran | the test it secretly depends on |
60
+ | **Randomness** | `assert random.random() < 0.8` | the failing seed, and a repro that replays it |
61
+ | **Set/dict ordering** | comparing `",".join(a_set)` to a string | that `PYTHONHASHSEED` is the cause |
62
+ | **Time bombs** | breaks on the last day of the month, on Feb 29, in another timezone | the exact date/timezone, *before* it happens |
63
+ | **Sleep used as a wait** | `sleep(0.06)` then assert a thread finished | the exact `sleep()` line that will fail on a slow CI machine |
64
+ | **Race conditions** | unlocked read-modify-write from threads | that it fails when threads interleave |
65
+
66
+ ## Quickstart
67
+
68
+ ```bash
69
+ pip install flakehound
70
+ cd your-project
71
+ flakehound run
72
+ ```
73
+
74
+ That's it: no config. Try it on the included [demo project](https://github.com/urumb/flakehound/tree/main/demo) to see every kind of
75
+ problem. The same results as a Markdown report: [docs/example-report.md](https://github.com/urumb/flakehound/blob/main/docs/example-report.md).
76
+
77
+ > ⚠️ flakehound **runs your test suite** (many times). Only run it on code you trust.
78
+
79
+ ### Options
80
+
81
+ | Option | Meaning |
82
+ |---|---|
83
+ | `-n, --runs N` | How many times to run the suite (default 10) |
84
+ | `-j, --workers N` | Parallel runs (default: CPU cores − 1, max 8) |
85
+ | `--seed N` | Replay an earlier session exactly |
86
+ | `--json PATH` | Also write a machine-readable report |
87
+ | `--markdown PATH` | Also write a Markdown report (for PR comments) |
88
+ | `--fail-on KINDS` | What makes exit code 1: `flaky`, `time-bomb`, `racy`, `broken`, `suspected`, `none` (default `flaky,time-bomb,racy`) |
89
+ | `--no-diagnose` | Only detect flaky tests, don't explain them |
90
+ | `--max-diagnose N` | Diagnose at most N flaky tests (default 20) |
91
+ | `--no-probes` | Skip all probe runs (timezones, risky dates, timing) |
92
+ | `--no-timing` | Skip only the timing probes |
93
+ | `--no-step-summary` | Don't add the report to the GitHub Actions job summary |
94
+ | `--timeout S` | Kill a single run after S seconds (default 600) |
95
+ | `-- ARGS` | Anything after `--` goes straight to pytest |
96
+
97
+ Exit codes: `0` no problems, `1` problems found (by default only flaky tests and *confirmed*
98
+ time bombs or racy tests fail a build), `2` usage or collection error.
99
+
100
+ ## In CI
101
+
102
+ Hunting runs your suite many times, so run it on a schedule rather than on every push:
103
+
104
+ ```yaml
105
+ # .github/workflows/flakehound.yml
106
+ on:
107
+ schedule: [{ cron: "0 3 * * *" }] # nightly
108
+ workflow_dispatch:
109
+ permissions:
110
+ contents: read
111
+ jobs:
112
+ hunt:
113
+ runs-on: ubuntu-latest
114
+ steps:
115
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
116
+ with: { persist-credentials: false }
117
+ - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
118
+ with: { python-version: "3.12" }
119
+ - run: pip install -e . flakehound
120
+ - run: flakehound run --json flakehound.json
121
+ ```
122
+
123
+ In GitHub Actions the Markdown report appears on the run's summary page automatically.
124
+
125
+ ## Quarantine
126
+
127
+ Found a flaky test you can't fix today? Quarantine it so it stops failing the build, without
128
+ hiding it:
129
+
130
+ ```bash
131
+ flakehound quarantine add tests/test_cart.py::test_cart_starts_empty --reason "polluted by test_add_item"
132
+ flakehound quarantine add --from-report flakehound.json # everything flakehound found
133
+ flakehound quarantine list
134
+ flakehound quarantine remove tests/test_cart.py::test_cart_starts_empty
135
+ ```
136
+
137
+ - The list is one file, `.flakehound/quarantine.toml`: commit it, review it, revert it.
138
+ flakehound never edits your test files.
139
+ - Every change shows a diff and asks first (`--yes` skips the prompt).
140
+ - pytest still runs quarantined tests as non-strict `xfail` (or `skip`), so you see `XPASS`
141
+ once one is fixed. Disable for one run with `pytest -p no:flakehound-quarantine`. The
142
+ plugin does nothing when the file doesn't exist.
143
+ - `flakehound run` ignores the quarantine, so quarantined tests keep being hunted.
144
+
145
+ ## How it works
146
+
147
+ flakehound changes **one factor at a time**, like a science experiment:
148
+
149
+ 1. **Shake.** Run the whole suite many times in parallel, each with a different test order,
150
+ `random` seed and `PYTHONHASHSEED`, all recorded. Tests that both pass and fail are
151
+ flaky; tests that always fail are rechecked, then reported as broken.
152
+ 2. **Isolate.** Run each flaky test alone with the *exact seeds* of the runs where it
153
+ failed. Only the order changed, so if it now passes, order is proven to be the cause.
154
+ 3. **Find the culprit.** Try the tests whose position best correlates with the failure,
155
+ then shrink the preceding tests with delta debugging, in parallel batches.
156
+ 4. **Name the leaked state.** Snapshot global state before and after the culprit: module
157
+ variables, object and class attributes, environment variables (secret values are never
158
+ shown), working directory, `sys.path`, project files and logging config.
159
+ 5. **Seeds.** For flaky tests that aren't about order, swap *one* seed at a time between a
160
+ failing and a passing run: the one that flips the result is the cause. If identical
161
+ seeds still give different results, the test is nondeterministic.
162
+ 6. **Time bomb probes.** Runs with the baseline's exact order and seeds, but in UTC+14,
163
+ UTC−12, just before midnight, on the last day of the month, on New Year's Eve and on
164
+ Feb 29. The clock is *shifted*, not frozen, so code that waits on time still works.
165
+ 7. **Timing probes.** One run where every `sleep()` on the test's main thread is 20×
166
+ shorter (background work keeps its real speed, like a slow machine), and one where
167
+ threads switch far more often. Then one `sleep()` call site at a time, to find the line.
168
+ 8. **Confirm and verify.** Findings are re-run, and each printed repro command is run for
169
+ real; it's marked *verified* only if it fails exactly as printed.
170
+
171
+ Repro commands that replay a seed, clock or timing stress use flakehound's pytest plugin
172
+ (`-p flakehound.adapters.pytest.plugin`), so they need flakehound installed. That plugin
173
+ never loads into your normal pytest runs on its own.
174
+
175
+ ## How it compares
176
+
177
+ | Tool | Retries | Detects | Explains why | Free & local |
178
+ |---|---|---|---|---|
179
+ | pytest-rerunfailures, flaky | ✅ | ❌ | ❌ | ✅ |
180
+ | pytest-randomly | ❌ | order/seed issues show up | ❌ | ✅ |
181
+ | detect-test-pollution | ❌ | ❌ | the polluter test (you drive it) | ✅ |
182
+ | Trunk, BuildPulse, Datadog | ✅ | from CI history | little | ❌ paid / uploads data |
183
+ | **flakehound** | ❌ (on purpose) | ✅ + time bombs + racy tests | polluter, leaked state, seed, date, timezone, `sleep()` line, race | ✅ |
184
+
185
+ ## Limitations
186
+
187
+ - Python + pytest only, for now.
188
+ - `asyncio.sleep` is not shortened by the timing probes.
189
+ - State hidden in fixtures, databases or external services can't be seen by the leak
190
+ detector; the report says so instead of guessing.
191
+ - Thread stress exposes races that happen around Python function calls.
192
+ - Hunting takes many runs of your suite: use `-n`, `-j` and `--max-diagnose` on large suites.
193
+
194
+ ## Roadmap
195
+
196
+ - GitHub Actions history mode: find flaky tests from past CI runs without running anything
197
+ - A ready-made GitHub Action
198
+ - Adapters for Jest/Vitest, Go test and JUnit
199
+ - Optional AI explanations (local models via Ollama, or your own key; never required)
200
+
201
+ See [CHANGELOG.md](https://github.com/urumb/flakehound/blob/main/CHANGELOG.md) for what's in each release.
202
+
203
+ ## Contributing and security
204
+
205
+ Contributions are welcome: see [CONTRIBUTING.md](https://github.com/urumb/flakehound/blob/main/CONTRIBUTING.md). To report a
206
+ vulnerability privately, see [SECURITY.md](https://github.com/urumb/flakehound/blob/main/SECURITY.md).
207
+
208
+ ## License
209
+
210
+ MIT
@@ -0,0 +1,179 @@
1
+ # flakehound
2
+
3
+ [![PyPI](https://img.shields.io/pypi/v/flakehound.svg)](https://pypi.org/project/flakehound/)
4
+ [![Python](https://img.shields.io/pypi/pyversions/flakehound.svg)](https://pypi.org/project/flakehound/)
5
+ [![CI](https://github.com/urumb/flakehound/actions/workflows/ci.yml/badge.svg)](https://github.com/urumb/flakehound/actions/workflows/ci.yml)
6
+ [![CodeQL](https://github.com/urumb/flakehound/actions/workflows/codeql.yml/badge.svg)](https://github.com/urumb/flakehound/actions/workflows/codeql.yml)
7
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/urumb/flakehound/blob/main/LICENSE)
8
+
9
+ **Find flaky tests and prove *why* they're flaky.**
10
+
11
+ Flaky tests pass sometimes and fail sometimes with no code change. Most tools just retry
12
+ them, which hides the problem. flakehound runs your pytest suite under deliberately hostile
13
+ conditions, finds the tests that flip, and tells you the cause, with evidence from real runs
14
+ and a command that reproduces it. It also finds problems that haven't bitten you *yet*.
15
+
16
+ ![flakehound output on the demo project](https://raw.githubusercontent.com/urumb/flakehound/main/docs/demo.svg)
17
+
18
+ - 🆓 **Free forever.** No accounts, no servers, no paid services.
19
+ - 🔒 **Local and private.** Offline, no telemetry, secrets redacted from every report.
20
+ - 🧪 **Evidence, not guesses.** Every verdict comes from runs that prove it, and every repro
21
+ command is run for real before it's marked *verified*.
22
+
23
+ ## What it finds
24
+
25
+ | Problem | Example | What flakehound tells you |
26
+ |---|---|---|
27
+ | **Order dependency** | a test leaves an item in a shared cart | the polluter test and the leaked state: `shop.cart: [] → ['apple']` |
28
+ | **Needs another test** | a test only passes after a setup-like test ran | the test it secretly depends on |
29
+ | **Randomness** | `assert random.random() < 0.8` | the failing seed, and a repro that replays it |
30
+ | **Set/dict ordering** | comparing `",".join(a_set)` to a string | that `PYTHONHASHSEED` is the cause |
31
+ | **Time bombs** | breaks on the last day of the month, on Feb 29, in another timezone | the exact date/timezone, *before* it happens |
32
+ | **Sleep used as a wait** | `sleep(0.06)` then assert a thread finished | the exact `sleep()` line that will fail on a slow CI machine |
33
+ | **Race conditions** | unlocked read-modify-write from threads | that it fails when threads interleave |
34
+
35
+ ## Quickstart
36
+
37
+ ```bash
38
+ pip install flakehound
39
+ cd your-project
40
+ flakehound run
41
+ ```
42
+
43
+ That's it: no config. Try it on the included [demo project](https://github.com/urumb/flakehound/tree/main/demo) to see every kind of
44
+ problem. The same results as a Markdown report: [docs/example-report.md](https://github.com/urumb/flakehound/blob/main/docs/example-report.md).
45
+
46
+ > ⚠️ flakehound **runs your test suite** (many times). Only run it on code you trust.
47
+
48
+ ### Options
49
+
50
+ | Option | Meaning |
51
+ |---|---|
52
+ | `-n, --runs N` | How many times to run the suite (default 10) |
53
+ | `-j, --workers N` | Parallel runs (default: CPU cores − 1, max 8) |
54
+ | `--seed N` | Replay an earlier session exactly |
55
+ | `--json PATH` | Also write a machine-readable report |
56
+ | `--markdown PATH` | Also write a Markdown report (for PR comments) |
57
+ | `--fail-on KINDS` | What makes exit code 1: `flaky`, `time-bomb`, `racy`, `broken`, `suspected`, `none` (default `flaky,time-bomb,racy`) |
58
+ | `--no-diagnose` | Only detect flaky tests, don't explain them |
59
+ | `--max-diagnose N` | Diagnose at most N flaky tests (default 20) |
60
+ | `--no-probes` | Skip all probe runs (timezones, risky dates, timing) |
61
+ | `--no-timing` | Skip only the timing probes |
62
+ | `--no-step-summary` | Don't add the report to the GitHub Actions job summary |
63
+ | `--timeout S` | Kill a single run after S seconds (default 600) |
64
+ | `-- ARGS` | Anything after `--` goes straight to pytest |
65
+
66
+ Exit codes: `0` no problems, `1` problems found (by default only flaky tests and *confirmed*
67
+ time bombs or racy tests fail a build), `2` usage or collection error.
68
+
69
+ ## In CI
70
+
71
+ Hunting runs your suite many times, so run it on a schedule rather than on every push:
72
+
73
+ ```yaml
74
+ # .github/workflows/flakehound.yml
75
+ on:
76
+ schedule: [{ cron: "0 3 * * *" }] # nightly
77
+ workflow_dispatch:
78
+ permissions:
79
+ contents: read
80
+ jobs:
81
+ hunt:
82
+ runs-on: ubuntu-latest
83
+ steps:
84
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
85
+ with: { persist-credentials: false }
86
+ - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
87
+ with: { python-version: "3.12" }
88
+ - run: pip install -e . flakehound
89
+ - run: flakehound run --json flakehound.json
90
+ ```
91
+
92
+ In GitHub Actions the Markdown report appears on the run's summary page automatically.
93
+
94
+ ## Quarantine
95
+
96
+ Found a flaky test you can't fix today? Quarantine it so it stops failing the build, without
97
+ hiding it:
98
+
99
+ ```bash
100
+ flakehound quarantine add tests/test_cart.py::test_cart_starts_empty --reason "polluted by test_add_item"
101
+ flakehound quarantine add --from-report flakehound.json # everything flakehound found
102
+ flakehound quarantine list
103
+ flakehound quarantine remove tests/test_cart.py::test_cart_starts_empty
104
+ ```
105
+
106
+ - The list is one file, `.flakehound/quarantine.toml`: commit it, review it, revert it.
107
+ flakehound never edits your test files.
108
+ - Every change shows a diff and asks first (`--yes` skips the prompt).
109
+ - pytest still runs quarantined tests as non-strict `xfail` (or `skip`), so you see `XPASS`
110
+ once one is fixed. Disable for one run with `pytest -p no:flakehound-quarantine`. The
111
+ plugin does nothing when the file doesn't exist.
112
+ - `flakehound run` ignores the quarantine, so quarantined tests keep being hunted.
113
+
114
+ ## How it works
115
+
116
+ flakehound changes **one factor at a time**, like a science experiment:
117
+
118
+ 1. **Shake.** Run the whole suite many times in parallel, each with a different test order,
119
+ `random` seed and `PYTHONHASHSEED`, all recorded. Tests that both pass and fail are
120
+ flaky; tests that always fail are rechecked, then reported as broken.
121
+ 2. **Isolate.** Run each flaky test alone with the *exact seeds* of the runs where it
122
+ failed. Only the order changed, so if it now passes, order is proven to be the cause.
123
+ 3. **Find the culprit.** Try the tests whose position best correlates with the failure,
124
+ then shrink the preceding tests with delta debugging, in parallel batches.
125
+ 4. **Name the leaked state.** Snapshot global state before and after the culprit: module
126
+ variables, object and class attributes, environment variables (secret values are never
127
+ shown), working directory, `sys.path`, project files and logging config.
128
+ 5. **Seeds.** For flaky tests that aren't about order, swap *one* seed at a time between a
129
+ failing and a passing run: the one that flips the result is the cause. If identical
130
+ seeds still give different results, the test is nondeterministic.
131
+ 6. **Time bomb probes.** Runs with the baseline's exact order and seeds, but in UTC+14,
132
+ UTC−12, just before midnight, on the last day of the month, on New Year's Eve and on
133
+ Feb 29. The clock is *shifted*, not frozen, so code that waits on time still works.
134
+ 7. **Timing probes.** One run where every `sleep()` on the test's main thread is 20×
135
+ shorter (background work keeps its real speed, like a slow machine), and one where
136
+ threads switch far more often. Then one `sleep()` call site at a time, to find the line.
137
+ 8. **Confirm and verify.** Findings are re-run, and each printed repro command is run for
138
+ real; it's marked *verified* only if it fails exactly as printed.
139
+
140
+ Repro commands that replay a seed, clock or timing stress use flakehound's pytest plugin
141
+ (`-p flakehound.adapters.pytest.plugin`), so they need flakehound installed. That plugin
142
+ never loads into your normal pytest runs on its own.
143
+
144
+ ## How it compares
145
+
146
+ | Tool | Retries | Detects | Explains why | Free & local |
147
+ |---|---|---|---|---|
148
+ | pytest-rerunfailures, flaky | ✅ | ❌ | ❌ | ✅ |
149
+ | pytest-randomly | ❌ | order/seed issues show up | ❌ | ✅ |
150
+ | detect-test-pollution | ❌ | ❌ | the polluter test (you drive it) | ✅ |
151
+ | Trunk, BuildPulse, Datadog | ✅ | from CI history | little | ❌ paid / uploads data |
152
+ | **flakehound** | ❌ (on purpose) | ✅ + time bombs + racy tests | polluter, leaked state, seed, date, timezone, `sleep()` line, race | ✅ |
153
+
154
+ ## Limitations
155
+
156
+ - Python + pytest only, for now.
157
+ - `asyncio.sleep` is not shortened by the timing probes.
158
+ - State hidden in fixtures, databases or external services can't be seen by the leak
159
+ detector; the report says so instead of guessing.
160
+ - Thread stress exposes races that happen around Python function calls.
161
+ - Hunting takes many runs of your suite: use `-n`, `-j` and `--max-diagnose` on large suites.
162
+
163
+ ## Roadmap
164
+
165
+ - GitHub Actions history mode: find flaky tests from past CI runs without running anything
166
+ - A ready-made GitHub Action
167
+ - Adapters for Jest/Vitest, Go test and JUnit
168
+ - Optional AI explanations (local models via Ollama, or your own key; never required)
169
+
170
+ See [CHANGELOG.md](https://github.com/urumb/flakehound/blob/main/CHANGELOG.md) for what's in each release.
171
+
172
+ ## Contributing and security
173
+
174
+ Contributions are welcome: see [CONTRIBUTING.md](https://github.com/urumb/flakehound/blob/main/CONTRIBUTING.md). To report a
175
+ vulnerability privately, see [SECURITY.md](https://github.com/urumb/flakehound/blob/main/SECURITY.md).
176
+
177
+ ## License
178
+
179
+ MIT
@@ -0,0 +1,26 @@
1
+ # Security Policy
2
+
3
+ ## Reporting a vulnerability
4
+
5
+ Please **do not open a public issue** for security problems.
6
+
7
+ Report privately through GitHub's
8
+ [private vulnerability reporting](https://github.com/urumb/flakehound/security/advisories/new).
9
+ You should get a response within 7 days. Please include steps to reproduce and the affected version.
10
+
11
+ ## Supported versions
12
+
13
+ Only the latest release receives security fixes while the project is pre-1.0.
14
+
15
+ ## Security model
16
+
17
+ - **flakehound runs your test suite, which is arbitrary code.** Only run it on code you trust.
18
+ Never point it at untrusted pull requests or repositories without a sandbox.
19
+ - **Offline by default.** flakehound makes no network calls and collects no telemetry.
20
+ Features that need the network (such as future GitHub history mode or cloud AI) are opt-in.
21
+ - **No secrets in output.** Reports never include environment variables; values that look like
22
+ tokens, keys or passwords are redacted.
23
+ - **Safe subprocesses.** Tests run via argument lists, never `shell=True`. Child processes and
24
+ temporary files are cleaned up, even on Ctrl+C.
25
+ - **Supply chain.** Minimal dependencies, releases published with PyPI Trusted Publishing
26
+ (no long-lived tokens), GitHub Actions pinned to commit SHAs with least-privilege permissions.
@@ -0,0 +1,17 @@
1
+ # flakehound demo
2
+
3
+ A tiny shop whose tests are flaky on purpose. Each file is one classic trap:
4
+
5
+ | Test file | Trap |
6
+ |---|---|
7
+ | `tests/test_cart.py` | Order dependency: `test_add_item` leaves an item in the shared cart |
8
+ | `tests/test_currency.py` | Order dependency: `test_euro_store` leaks the `SHOP_CURRENCY` env var |
9
+ | `tests/test_inventory.py` | Needs another test: `test_apples_in_stock` only passes after `test_restock_apples` |
10
+ | `tests/test_discount.py` | Unseeded randomness |
11
+ | `tests/test_tags.py` | Set iteration order depends on `PYTHONHASHSEED` |
12
+ | `tests/test_upload.py` | Timing: `sleep()` used to wait for a background thread |
13
+ | `tests/test_wallet.py` | Race condition: unlocked read-modify-write from several threads |
14
+ | `tests/test_dates.py` | Time bombs: mixes local time and UTC, breaks on the last day of the month and on Feb 29 |
15
+ | `tests/test_math.py` | Stable tests, for contrast |
16
+
17
+ Run `flakehound run` in this folder.
@@ -0,0 +1,11 @@
1
+ import shop
2
+
3
+
4
+ def test_cart_starts_empty():
5
+ # Passes in the default order, fails whenever test_add_item ran first.
6
+ assert shop.cart == []
7
+
8
+
9
+ def test_add_item():
10
+ shop.add_item("apple") # never cleaned up: leaks into later tests
11
+ assert "apple" in shop.cart
@@ -0,0 +1,12 @@
1
+ import os
2
+
3
+ import shop
4
+
5
+
6
+ def test_prices_default_to_usd():
7
+ assert shop.currency() == "USD"
8
+
9
+
10
+ def test_euro_store():
11
+ os.environ["SHOP_CURRENCY"] = "EUR" # bug: never restored (use monkeypatch.setenv)
12
+ assert shop.currency() == "EUR"
@@ -0,0 +1,18 @@
1
+ from datetime import date, datetime, timedelta, timezone
2
+
3
+
4
+ def test_order_date_matches_utc():
5
+ # Bug: mixes local time and UTC. Fails wherever the local date differs from
6
+ # the UTC date (e.g. a CI machine in another timezone, or late evenings).
7
+ assert datetime.now().date() == datetime.now(timezone.utc).date()
8
+
9
+
10
+ def test_coupon_valid_until_tomorrow():
11
+ # Bug: assumes tomorrow is in the same month. False on the last day of every month.
12
+ assert (date.today() + timedelta(days=1)).month == date.today().month
13
+
14
+
15
+ def test_anniversary_next_year():
16
+ today = date.today()
17
+ # Bug: Feb 29 plus one year does not exist -> ValueError on leap days.
18
+ assert today.replace(year=today.year + 1) > today
@@ -0,0 +1,5 @@
1
+ import shop
2
+
3
+
4
+ def test_lucky_discount_is_small():
5
+ assert shop.lucky_discount() < 0.8 # fails about 20% of the time
@@ -0,0 +1,11 @@
1
+ import shop
2
+
3
+
4
+ def test_restock_apples():
5
+ shop.Inventory.restock("apple", 5)
6
+ assert shop.Inventory.stock["apple"] >= 5
7
+
8
+
9
+ def test_apples_in_stock():
10
+ # Bug: silently relies on test_restock_apples having run first.
11
+ assert shop.Inventory.stock.get("apple", 0) > 0
@@ -0,0 +1,9 @@
1
+ import shop
2
+
3
+
4
+ def test_single_tag_renders():
5
+ assert shop.render_tags({"python"}) == "python"
6
+
7
+
8
+ def test_addition():
9
+ assert 1 + 1 == 2
@@ -0,0 +1,6 @@
1
+ import shop
2
+
3
+
4
+ def test_tags_render_alphabetically():
5
+ # Bug: sets have no defined order. Passes only for some hash seeds.
6
+ assert shop.render_tags({"flaky", "pytest", "python"}) == "flaky,pytest,python"
@@ -0,0 +1,10 @@
1
+ import time
2
+
3
+ import shop
4
+
5
+
6
+ def test_upload_finishes():
7
+ up = shop.Uploader()
8
+ up.start()
9
+ time.sleep(0.06) # "should be long enough" (it isn't on a busy machine)
10
+ assert up.done
@@ -0,0 +1,21 @@
1
+ import threading
2
+
3
+ import shop
4
+
5
+
6
+ def test_concurrent_top_ups():
7
+ wallet = shop.Wallet()
8
+ start = threading.Barrier(4) # all four "users" pay at the same moment
9
+
10
+ def worker():
11
+ start.wait()
12
+ for _ in range(2000):
13
+ wallet.top_up(1)
14
+
15
+ threads = [threading.Thread(target=worker) for _ in range(4)]
16
+ for t in threads:
17
+ t.start()
18
+ for t in threads:
19
+ t.join()
20
+ # Bug: top_up is not thread-safe. Passes almost always, until threads interleave.
21
+ assert wallet.balance == 8000