pytest-flakefighters 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. pytest_flakefighters-0.2.0/.github/workflows/ci-mega-linter.yml +36 -0
  2. pytest_flakefighters-0.2.0/.github/workflows/ci-tests-drafts.yaml +37 -0
  3. pytest_flakefighters-0.2.0/.github/workflows/ci-tests.yaml +42 -0
  4. pytest_flakefighters-0.2.0/.github/workflows/publish-pypi.yaml +32 -0
  5. pytest_flakefighters-0.2.0/.gitignore +220 -0
  6. pytest_flakefighters-0.2.0/.mega-linter.yaml +14 -0
  7. pytest_flakefighters-0.2.0/.pre-commit-config.yaml +21 -0
  8. pytest_flakefighters-0.2.0/.readthedocs.yaml +27 -0
  9. pytest_flakefighters-0.2.0/LICENSE +22 -0
  10. pytest_flakefighters-0.2.0/PKG-INFO +112 -0
  11. pytest_flakefighters-0.2.0/README.md +68 -0
  12. pytest_flakefighters-0.2.0/codecov.yml +20 -0
  13. pytest_flakefighters-0.2.0/docs/Makefile +20 -0
  14. pytest_flakefighters-0.2.0/docs/make.bat +263 -0
  15. pytest_flakefighters-0.2.0/docs/source/_static/css/custom.css +49 -0
  16. pytest_flakefighters-0.2.0/docs/source/_static/images/favicon.png +0 -0
  17. pytest_flakefighters-0.2.0/docs/source/_static/images/logo.png +0 -0
  18. pytest_flakefighters-0.2.0/docs/source/acknowlegements.rst +10 -0
  19. pytest_flakefighters-0.2.0/docs/source/conf.py +312 -0
  20. pytest_flakefighters-0.2.0/docs/source/configuration.rst +26 -0
  21. pytest_flakefighters-0.2.0/docs/source/dev/actions_and_webhooks.rst +29 -0
  22. pytest_flakefighters-0.2.0/docs/source/dev/documentation.rst +58 -0
  23. pytest_flakefighters-0.2.0/docs/source/dev/version_release.rst +31 -0
  24. pytest_flakefighters-0.2.0/docs/source/glossary.rst +7 -0
  25. pytest_flakefighters-0.2.0/docs/source/index.rst +114 -0
  26. pytest_flakefighters-0.2.0/docs/source/installation.rst +65 -0
  27. pytest_flakefighters-0.2.0/docs/source/requirements.txt +9 -0
  28. pytest_flakefighters-0.2.0/pyproject.toml +116 -0
  29. pytest_flakefighters-0.2.0/setup.cfg +4 -0
  30. pytest_flakefighters-0.2.0/src/_version.py +34 -0
  31. pytest_flakefighters-0.2.0/src/pytest_flakefighters/__init__.py +0 -0
  32. pytest_flakefighters-0.2.0/src/pytest_flakefighters/database_management.py +233 -0
  33. pytest_flakefighters-0.2.0/src/pytest_flakefighters/flakefighters/__init__.py +0 -0
  34. pytest_flakefighters-0.2.0/src/pytest_flakefighters/flakefighters/abstract_flakefighter.py +50 -0
  35. pytest_flakefighters-0.2.0/src/pytest_flakefighters/flakefighters/coverage_independence.py +100 -0
  36. pytest_flakefighters-0.2.0/src/pytest_flakefighters/flakefighters/deflaker.py +141 -0
  37. pytest_flakefighters-0.2.0/src/pytest_flakefighters/flakefighters/traceback_matching.py +176 -0
  38. pytest_flakefighters-0.2.0/src/pytest_flakefighters/function_coverage.py +71 -0
  39. pytest_flakefighters-0.2.0/src/pytest_flakefighters/main.py +167 -0
  40. pytest_flakefighters-0.2.0/src/pytest_flakefighters/plugin.py +227 -0
  41. pytest_flakefighters-0.2.0/src/pytest_flakefighters/rerun_strategies.py +90 -0
  42. pytest_flakefighters-0.2.0/src/pytest_flakefighters.egg-info/PKG-INFO +112 -0
  43. pytest_flakefighters-0.2.0/src/pytest_flakefighters.egg-info/SOURCES.txt +60 -0
  44. pytest_flakefighters-0.2.0/src/pytest_flakefighters.egg-info/dependency_links.txt +1 -0
  45. pytest_flakefighters-0.2.0/src/pytest_flakefighters.egg-info/entry_points.txt +8 -0
  46. pytest_flakefighters-0.2.0/src/pytest_flakefighters.egg-info/requires.txt +27 -0
  47. pytest_flakefighters-0.2.0/src/pytest_flakefighters.egg-info/top_level.txt +2 -0
  48. pytest_flakefighters-0.2.0/tests/__init__.py +0 -0
  49. pytest_flakefighters-0.2.0/tests/conftest.py +67 -0
  50. pytest_flakefighters-0.2.0/tests/flakefighters/test_coverage_independence.py +126 -0
  51. pytest_flakefighters-0.2.0/tests/flakefighters/test_deflaker.py +241 -0
  52. pytest_flakefighters-0.2.0/tests/flakefighters/test_traceback_matching.py +140 -0
  53. pytest_flakefighters-0.2.0/tests/resources/deflaker_broken.py +16 -0
  54. pytest_flakefighters-0.2.0/tests/resources/deflaker_example.py +16 -0
  55. pytest_flakefighters-0.2.0/tests/resources/flaky_reruns.py +22 -0
  56. pytest_flakefighters-0.2.0/tests/resources/pass_fail_flaky.py +15 -0
  57. pytest_flakefighters-0.2.0/tests/resources/test.txt +0 -0
  58. pytest_flakefighters-0.2.0/tests/resources/triangle.py +31 -0
  59. pytest_flakefighters-0.2.0/tests/test_database_management.py +131 -0
  60. pytest_flakefighters-0.2.0/tests/test_end_2_end.py +96 -0
  61. pytest_flakefighters-0.2.0/tests/test_function_coverage.py +80 -0
  62. pytest_flakefighters-0.2.0/tests/test_rerun_strategies.py +138 -0
@@ -0,0 +1,36 @@
1
+ name: Linting and formatting
2
+
3
+ on:
4
+ pull_request:
5
+ branches:
6
+ - main
7
+
8
+ concurrency:
9
+ group: ${{ github.ref }}-${{ github.workflow }}
10
+ cancel-in-progress: true
11
+
12
+ jobs:
13
+ build:
14
+ name: MegaLinter
15
+ runs-on: ubuntu-latest
16
+ permissions:
17
+ contents: write
18
+ pull-requests: write
19
+ steps:
20
+ - name: Checkout Code
21
+ uses: actions/checkout@v2
22
+
23
+ - name: MegaLinter
24
+ id: ml
25
+ uses: oxsecurity/megalinter/flavors/python@latest
26
+ env:
27
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # For detailed check status
28
+
29
+ - name: Archive production artifacts
30
+ if: ${{ success() }} || ${{ failure() }}
31
+ uses: actions/upload-artifact@v4
32
+ with:
33
+ name: MegaLinter reports
34
+ path: |
35
+ megalinter-reports
36
+ mega-linter.log
@@ -0,0 +1,37 @@
1
+ name: CI Tests Draft
2
+ # This duplicate ci workflow is required so the badge in the README.md is not effected by draft PRs
3
+ on:
4
+ pull_request:
5
+ branches:
6
+ - main
7
+
8
+ jobs:
9
+ build:
10
+ if: github.event.pull_request.draft == true
11
+ name: Ex1 (${{ matrix.python-version }}, ${{ matrix.os }})
12
+ runs-on: ${{ matrix.os }}
13
+ strategy:
14
+ matrix:
15
+ os: ["ubuntu-latest", "windows-latest", "macos-latest"]
16
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+ - name: Set up Python
20
+ uses: actions/setup-python@v5
21
+ with:
22
+ python-version: ${{ matrix.python-version }}
23
+ - name: Install dependencies
24
+ run: |
25
+ python --version
26
+ python -m pip install --upgrade pip
27
+ pip install -e .
28
+ pip install -e .[test]
29
+ pip install pytest pytest-cov
30
+ - name: Test with pytest
31
+ run: |
32
+ pytest -p no:flakefighters --cov=src --cov-report=xml
33
+ - name: "Upload coverage to Codecov"
34
+ uses: codecov/codecov-action@v5
35
+ with:
36
+ fail_ci_if_error: true
37
+ token: ${{ secrets.CODECOV_TOKEN }}
@@ -0,0 +1,42 @@
1
+ name: CI Tests
2
+
3
+ on:
4
+ pull_request:
5
+ branches:
6
+ - main
7
+ types:
8
+ - opened
9
+ - synchronize
10
+ - reopened
11
+ - ready_for_review
12
+
13
+ jobs:
14
+ build:
15
+ if: github.event.pull_request.draft == false # Filter out draft PRs
16
+ name: Ex1 (${{ matrix.python-version }}, ${{ matrix.os }})
17
+ runs-on: ${{ matrix.os }}
18
+ strategy:
19
+ matrix:
20
+ os: ["ubuntu-latest", "windows-latest", "macos-latest"]
21
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
22
+ steps:
23
+ - uses: actions/checkout@v4
24
+ - name: Set up Python
25
+ uses: actions/setup-python@v5
26
+ with:
27
+ python-version: ${{ matrix.python-version }}
28
+ - name: Install dependencies
29
+ run: |
30
+ python --version
31
+ python -m pip install --upgrade pip
32
+ pip install -e .
33
+ pip install -e .[test]
34
+ pip install pytest pytest-cov
35
+ - name: Test with pytest
36
+ run: |
37
+ pytest -p no:flakefighters --cov=src --cov-report=xml
38
+ - name: "Upload coverage to Codecov"
39
+ uses: codecov/codecov-action@v5
40
+ with:
41
+ fail_ci_if_error: true
42
+ token: ${{ secrets.CODECOV_TOKEN }}
@@ -0,0 +1,32 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - v*
7
+
8
+ jobs:
9
+ build-release:
10
+ name: Build and publish PyPI
11
+ runs-on: ubuntu-latest
12
+ steps:
13
+ - uses: actions/checkout@v4
14
+ with:
15
+ fetch-depth: 0
16
+ - name: Set up Python
17
+ uses: actions/setup-python@v5
18
+ with:
19
+ python-version: '3.10'
20
+
21
+ - name: Install build tools
22
+ run: |
23
+ pip install --upgrade pip setuptools wheel build setuptools_scm
24
+
25
+ - name: Build Package
26
+ run: |
27
+ python -m build --no-isolation
28
+
29
+ - name: Publish package to PyPI
30
+ uses: pypa/gh-action-pypi-publish@release/v1
31
+ with:
32
+ password: ${{ secrets.PYPI_API_TOKEN }}
@@ -0,0 +1,220 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[codz]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py.cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+ docs/build/
74
+ docs/source/autoapi
75
+ .ipynb_checkpoints/
76
+
77
+ # PyBuilder
78
+ .pybuilder/
79
+ target/
80
+
81
+ # Jupyter Notebook
82
+ .ipynb_checkpoints
83
+
84
+ # IPython
85
+ profile_default/
86
+ ipython_config.py
87
+
88
+ # pyenv
89
+ # For a library or package, you might want to ignore these files since the code is
90
+ # intended to run in multiple environments; otherwise, check them in:
91
+ # .python-version
92
+
93
+ # pipenv
94
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
95
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
96
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
97
+ # install all needed dependencies.
98
+ #Pipfile.lock
99
+
100
+ # UV
101
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
102
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
103
+ # commonly ignored for libraries.
104
+ #uv.lock
105
+
106
+ # poetry
107
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
108
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
109
+ # commonly ignored for libraries.
110
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
111
+ #poetry.lock
112
+ #poetry.toml
113
+
114
+ # pdm
115
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
116
+ # pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
117
+ # https://pdm-project.org/en/latest/usage/project/#working-with-version-control
118
+ #pdm.lock
119
+ #pdm.toml
120
+ .pdm-python
121
+ .pdm-build/
122
+
123
+ # pixi
124
+ # Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
125
+ #pixi.lock
126
+ # Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
127
+ # in the .venv directory. It is recommended not to include this directory in version control.
128
+ .pixi
129
+
130
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
131
+ __pypackages__/
132
+
133
+ # Celery stuff
134
+ celerybeat-schedule
135
+ celerybeat.pid
136
+
137
+ # SageMath parsed files
138
+ *.sage.py
139
+
140
+ # Environments
141
+ .env
142
+ .envrc
143
+ .venv
144
+ env/
145
+ venv/
146
+ ENV/
147
+ env.bak/
148
+ venv.bak/
149
+
150
+ # Spyder project settings
151
+ .spyderproject
152
+ .spyproject
153
+
154
+ # Rope project settings
155
+ .ropeproject
156
+
157
+ # mkdocs documentation
158
+ /site
159
+
160
+ # mypy
161
+ .mypy_cache/
162
+ .dmypy.json
163
+ dmypy.json
164
+
165
+ # Pyre type checker
166
+ .pyre/
167
+
168
+ # pytype static type analyzer
169
+ .pytype/
170
+
171
+ # Cython debug symbols
172
+ cython_debug/
173
+
174
+ # PyCharm
175
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
176
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
177
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
178
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
179
+ #.idea/
180
+
181
+ # Abstra
182
+ # Abstra is an AI-powered process automation framework.
183
+ # Ignore directories containing user credentials, local state, and settings.
184
+ # Learn more at https://abstra.io/docs
185
+ .abstra/
186
+
187
+ # Visual Studio Code
188
+ # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
189
+ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
190
+ # and can be added to the global gitignore or merged into this file. However, if you prefer,
191
+ # you could uncomment the following to ignore the entire vscode folder
192
+ # .vscode/
193
+
194
+ # Ruff stuff:
195
+ .ruff_cache/
196
+
197
+ # PyPI configuration file
198
+ .pypirc
199
+
200
+ # Cursor
201
+ # Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
202
+ # exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
203
+ # refer to https://docs.cursor.com/context/ignore-files
204
+ .cursorignore
205
+ .cursorindexingignore
206
+
207
+ # Marimo
208
+ marimo/_static/
209
+ marimo/_lsp/
210
+ __marimo__/
211
+
212
+ megalinter-reports/
213
+
214
+ *.db
215
+
216
+ # Hide autogenerated sourcecode docs
217
+ docs/source/autoapi
218
+
219
+ # Hide autogenerated version file
220
+ _version.py
@@ -0,0 +1,14 @@
1
+ # Configuration file for MegaLinter
2
+ # See all available variables at https://oxsecurity.github.io/megalinter/configuration/ and in linters documentation
3
+
4
+ ENABLE_LINTERS:
5
+ - PYTHON_PYLINT
6
+ - PYTHON_BLACK
7
+
8
+ GITHUB_STATUS_REPORTER: true
9
+
10
+ FILTER_REGEX_INCLUDE: (src/|tests/)
11
+ FILTER_REGEX_EXCLUDE: (tests/resources/)
12
+
13
+ APPLY_FIXES: all
14
+ PYTHON_PYLINT_CONFIG_FILE: pyproject.toml
@@ -0,0 +1,21 @@
1
+ # See https://pre-commit.com for more information
2
+ # See https://pre-commit.com/hooks.html for more hooks
3
+ repos:
4
+ - repo: https://github.com/PyCQA/isort
5
+ rev: 6.0.1 # Use the latest version
6
+ hooks:
7
+ - id: isort
8
+ - repo: https://github.com/ambv/black
9
+ rev: 25.1.0
10
+ hooks:
11
+ - id: black
12
+ language_version: python3.11
13
+ - repo: local
14
+ hooks:
15
+ - id: pylint
16
+ name: pylint
17
+ entry: pylint
18
+ language: system
19
+ types: [python]
20
+ args: ['--max-line-length=120', '--max-positional-arguments=12']
21
+ files: ^src/|^tests/
@@ -0,0 +1,27 @@
1
+ # .readthedocs.yaml
2
+ # Read the Docs configuration file
3
+ # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
4
+
5
+ # Required
6
+ version: 2
7
+
8
+ # Set the version of Python
9
+ build:
10
+ os: ubuntu-20.04
11
+ tools:
12
+ python: "3.10"
13
+
14
+ # Build documentation in the docs/ directory with Sphinx
15
+ sphinx:
16
+ configuration: docs/source/conf.py
17
+
18
+ # If using Sphinx, optionally build your docs in additional formats such as PDF
19
+ # formats:
20
+ # - pdf
21
+
22
+ # Optionally declare the Python requirements required to build your docs
23
+ python:
24
+ install:
25
+ - requirements: docs/source/requirements.txt
26
+ - method: pip
27
+ path: . # This should tell RTD to install the CTF from root to render the Estimators page correctly.
@@ -0,0 +1,22 @@
1
+
2
+ The MIT License (MIT)
3
+
4
+ Copyright (c) 2025 TestFLARE Team
5
+
6
+ Permission is hereby granted, free of charge, to any person obtaining a copy
7
+ of this software and associated documentation files (the "Software"), to deal
8
+ in the Software without restriction, including without limitation the rights
9
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
+ copies of the Software, and to permit persons to whom the Software is
11
+ furnished to do so, subject to the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be included in
14
+ all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22
+ THE SOFTWARE.
@@ -0,0 +1,112 @@
1
+ Metadata-Version: 2.4
2
+ Name: pytest-flakefighters
3
+ Version: 0.2.0
4
+ Summary: Pytest plugin implementing flaky test failure detection and classification.
5
+ Author: TestFLARE Team
6
+ Project-URL: Homepage, https://test-flare.github.io/
7
+ Project-URL: Documentation, https://pytest-flakefighters.readthedocs.io
8
+ Project-URL: Repository, https://github.com/test-flare/pytest-flakefighters
9
+ Project-URL: Issues, https://github.com/test-flare/pytest-flakefighters/issues
10
+ Classifier: Programming Language :: Python :: 3.10
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Programming Language :: Python :: 3.13
14
+ Classifier: Programming Language :: Python :: 3.14
15
+ Requires-Python: <3.14,>=3.10
16
+ Description-Content-Type: text/markdown
17
+ License-File: LICENSE
18
+ Requires-Dist: pytest>=6.2.0
19
+ Requires-Dist: coverage>=7.10.6
20
+ Requires-Dist: GitPython>=3.1.45
21
+ Requires-Dist: unidiff>=0.7.5
22
+ Requires-Dist: sqlalchemy>=2.0.43
23
+ Requires-Dist: dotenv>=0.9.9
24
+ Requires-Dist: pandas>=2.3
25
+ Requires-Dist: scipy<=1.15
26
+ Requires-Dist: pyyaml>=6
27
+ Requires-Dist: scikit-learn>=1.7
28
+ Requires-Dist: nltk>=3.9
29
+ Provides-Extra: dev
30
+ Requires-Dist: black; extra == "dev"
31
+ Requires-Dist: pytest-cov; extra == "dev"
32
+ Requires-Dist: pylint; extra == "dev"
33
+ Requires-Dist: pre_commit; extra == "dev"
34
+ Requires-Dist: sphinx-autoapi; extra == "dev"
35
+ Requires-Dist: sphinx_rtd_theme; extra == "dev"
36
+ Requires-Dist: tox>=4.31.0; extra == "dev"
37
+ Requires-Dist: myst_parser; extra == "dev"
38
+ Requires-Dist: nbsphinx; extra == "dev"
39
+ Requires-Dist: pytest-html; extra == "dev"
40
+ Requires-Dist: pytest-json-report>=1.5; extra == "dev"
41
+ Provides-Extra: pg
42
+ Requires-Dist: psycopg2>2.9; extra == "pg"
43
+ Dynamic: license-file
44
+
45
+ # Pytest FlakeFighters
46
+ ### Pytest plugin implementing flaky test failure detection and classification.
47
+
48
+ [![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active)
49
+ [![PyPI version](https://img.shields.io/pypi/v/pytest-flakefighters.svg)](https://pypi.org/project/pytest-flakefighters)
50
+ [![Python versions](https://img.shields.io/pypi/pyversions/pytest-flakefighters.svg)](https://pypi.org/project/pytest-flakefighters)
51
+ ![Test status](https://github.com/test-flare/pytest-flakefighters/actions/workflows/ci-tests.yaml/badge.svg)
52
+ [![codecov](https://codecov.io/gh/test-flare/pytest-flakefighters/branch/main/graph/badge.svg?token=04ijFVrb4a)](https://codecov.io/gh/test-flare/pytest-flakefighters)
53
+ [![Documentation Status](https://readthedocs.org/projects/causal-testing-framework/badge/?version=latest)](https://causal-testing-framework.readthedocs.io/en/latest/?badge=latest)
54
+ ![GitHub License](https://img.shields.io/github/license/test-flare/pytest-flakefighters)
55
+
56
+
57
+
58
+ ## Features
59
+
60
+ - Implements the [DeFlaker algorithm](https://deflaker.com/) for pytest
61
+
62
+
63
+ ## Installation
64
+
65
+ You can install \"pytest-flakefighters\" by cloning this repo and running `pip install .` from the root directory.
66
+ If you intend to develop the plugin, run `pip install -e .[dev]` instead.
67
+
68
+ We eventually intend to distribute our tool on PyPI.
69
+
70
+ ## Usage
71
+
72
+ FlakeFighter is intended to run on git repositories that have test suites runnable with `pytest`.
73
+ Once you have installed FlakeFighter, you can run it from the root directory of your repo simply by running `pytest` in your usual way.
74
+ FlakeFighter has the following arguments.
75
+
76
+ ```
77
+ --target-commit=TARGET_COMMIT
78
+ The target (newer) commit hash. Defaults to HEAD (the most recent commit).
79
+ --source-commit=SOURCE_COMMIT
80
+ The source (older) commit hash. Defaults to HEAD^ (the previous commit to target).
81
+ --repo=REPO_ROOT The commit hash to compare against.
82
+ --suppress-flaky-failures-exit-code
83
+ Return OK exit code if the only failures are flaky failures.
84
+ --no-save Do not save this run to the database of previous flakefighters runs.
85
+ -M LOAD_MAX_RUNS, --load-max-runs=LOAD_MAX_RUNS
86
+ The maximum number of previous runs to consider.
87
+ -D DATABASE_URL, --database-url=DATABASE_URL
88
+ The database URL. Defaults to 'flakefighter.db' in current working directory.
89
+ --store-max-runs=STORE_MAX_RUNS
90
+ The maximum number of previous flakefighters runs to store. Default is to store all.
91
+ --time-immemorial=TIME_IMMEMORIAL
92
+ How long to store flakefighters runs for, specified as `days:hours:minutes`. E.g. to store
93
+ tests for one week, use 7:0:0.
94
+ ```
95
+
96
+ ## Contributing
97
+
98
+ Contributions are very welcome.
99
+ Tests can be run with [pytest](https://pytest.readthedocs.io/en/latest/), please ensure the coverage at least stays the same before you submit a pull request.
100
+
101
+ ## Flake Fighters
102
+ Our plugin is made up of a collection of heuristics that come together to help inform whether a test failure is genuine or flaky.
103
+ These come in two "flavours": those which run live after each test, and those which run at the end of the entire test suite.
104
+ Both extend the base class `FlakeFighter` and implement the `flaky_failure` method, which returns `True` if the test is deemed to be flaky.
105
+
106
+ ## Issues
107
+
108
+ If you encounter any problems, please [file an issue](https://github.com/test-flare/pytest-flakefighters/issues) along with a detailed description.
109
+
110
+ ------------------------------------------------------------------------
111
+
112
+ This [pytest](https://github.com/pytest-dev/pytest) plugin was generated with [Cookiecutter](https://github.com/audreyr/cookiecutter) along with [\@hackebrot](https://github.com/hackebrot)\'s [cookiecutter-pytest-plugin](https://github.com/pytest-dev/cookiecutter-pytest-plugin) template.
@@ -0,0 +1,68 @@
1
+ # Pytest FlakeFighters
2
+ ### Pytest plugin implementing flaky test failure detection and classification.
3
+
4
+ [![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active)
5
+ [![PyPI version](https://img.shields.io/pypi/v/pytest-flakefighters.svg)](https://pypi.org/project/pytest-flakefighters)
6
+ [![Python versions](https://img.shields.io/pypi/pyversions/pytest-flakefighters.svg)](https://pypi.org/project/pytest-flakefighters)
7
+ ![Test status](https://github.com/test-flare/pytest-flakefighters/actions/workflows/ci-tests.yaml/badge.svg)
8
+ [![codecov](https://codecov.io/gh/test-flare/pytest-flakefighters/branch/main/graph/badge.svg?token=04ijFVrb4a)](https://codecov.io/gh/test-flare/pytest-flakefighters)
9
+ [![Documentation Status](https://readthedocs.org/projects/causal-testing-framework/badge/?version=latest)](https://causal-testing-framework.readthedocs.io/en/latest/?badge=latest)
10
+ ![GitHub License](https://img.shields.io/github/license/test-flare/pytest-flakefighters)
11
+
12
+
13
+
14
+ ## Features
15
+
16
+ - Implements the [DeFlaker algorithm](https://deflaker.com/) for pytest
17
+
18
+
19
+ ## Installation
20
+
21
+ You can install \"pytest-flakefighters\" by cloning this repo and running `pip install .` from the root directory.
22
+ If you intend to develop the plugin, run `pip install -e .[dev]` instead.
23
+
24
+ We eventually intend to distribute our tool on PyPI.
25
+
26
+ ## Usage
27
+
28
+ FlakeFighter is intended to run on git repositories that have test suites runnable with `pytest`.
29
+ Once you have installed FlakeFighter, you can run it from the root directory of your repo simply by running `pytest` in your usual way.
30
+ FlakeFighter has the following arguments.
31
+
32
+ ```
33
+ --target-commit=TARGET_COMMIT
34
+ The target (newer) commit hash. Defaults to HEAD (the most recent commit).
35
+ --source-commit=SOURCE_COMMIT
36
+ The source (older) commit hash. Defaults to HEAD^ (the previous commit to target).
37
+ --repo=REPO_ROOT The commit hash to compare against.
38
+ --suppress-flaky-failures-exit-code
39
+ Return OK exit code if the only failures are flaky failures.
40
+ --no-save Do not save this run to the database of previous flakefighters runs.
41
+ -M LOAD_MAX_RUNS, --load-max-runs=LOAD_MAX_RUNS
42
+ The maximum number of previous runs to consider.
43
+ -D DATABASE_URL, --database-url=DATABASE_URL
44
+ The database URL. Defaults to 'flakefighter.db' in current working directory.
45
+ --store-max-runs=STORE_MAX_RUNS
46
+ The maximum number of previous flakefighters runs to store. Default is to store all.
47
+ --time-immemorial=TIME_IMMEMORIAL
48
+ How long to store flakefighters runs for, specified as `days:hours:minutes`. E.g. to store
49
+ tests for one week, use 7:0:0.
50
+ ```
51
+
52
+ ## Contributing
53
+
54
+ Contributions are very welcome.
55
+ Tests can be run with [pytest](https://pytest.readthedocs.io/en/latest/), please ensure the coverage at least stays the same before you submit a pull request.
56
+
57
+ ## Flake Fighters
58
+ Our plugin is made up of a collection of heuristics that come together to help inform whether a test failure is genuine or flaky.
59
+ These come in two "flavours": those which run live after each test, and those which run at the end of the entire test suite.
60
+ Both extend the base class `FlakeFighter` and implement the `flaky_failure` method, which returns `True` if the test is deemed to be flaky.
61
+
62
+ ## Issues
63
+
64
+ If you encounter any problems, please [file an issue](https://github.com/test-flare/pytest-flakefighters/issues) along with a detailed description.
65
+
66
+ ------------------------------------------------------------------------
67
+
68
+ This [pytest](https://github.com/pytest-dev/pytest) plugin was generated with [Cookiecutter](https://github.com/audreyr/cookiecutter) along with [\@hackebrot](https://github.com/hackebrot)\'s [cookiecutter-pytest-plugin](https://github.com/pytest-dev/cookiecutter-pytest-plugin) template.
@@ -0,0 +1,20 @@
1
+ codecov:
2
+ require_ci_to_pass: yes
3
+
4
+ coverage:
5
+ precision: 2
6
+ round: down
7
+ range: "70...100"
8
+
9
+ parsers:
10
+ gcov:
11
+ branch_detection:
12
+ conditional: yes
13
+ loop: yes
14
+ method: no
15
+ macro: no
16
+
17
+ comment:
18
+ layout: "reach,diff,flags,files,footer"
19
+ behavior: default
20
+ require_changes: no
@@ -0,0 +1,20 @@
1
+ # Minimal makefile for Sphinx documentation
2
+ #
3
+
4
+ # You can set these variables from the command line, and also
5
+ # from the environment for the first two.
6
+ SPHINXOPTS ?=
7
+ SPHINXBUILD ?= sphinx-build
8
+ SOURCEDIR = source
9
+ BUILDDIR = build
10
+
11
+ # Put it first so that "make" without argument is like "make help".
12
+ help:
13
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14
+
15
+ .PHONY: help Makefile
16
+
17
+ # Catch-all target: route all unknown targets to Sphinx using the new
18
+ # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
19
+ %: Makefile
20
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)