pytest-flakefighters 0.0.0__tar.gz → 0.1.5__tar.gz

pytest_flakefighters-0.1.5/.github/workflows/publish-pypi.yaml

@@ -0,0 +1,32 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  build-release:
+    name: Build and publish PyPI
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install build tools
+        run: |
+          pip install --upgrade pip setuptools wheel build setuptools_scm
+
+      - name: Build Package
+        run: |
+          python -m build --no-isolation
+
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

{pytest_flakefighters-0.0.0 → pytest_flakefighters-0.1.5}/.gitignore

@@ -70,6 +70,9 @@ instance/
 
 # Sphinx documentation
 docs/_build/
+docs/build/
+docs/source/autoapi
+.ipynb_checkpoints/
 
 # PyBuilder
 .pybuilder/
@@ -209,3 +212,6 @@ __marimo__/
 megalinter-reports/
 
 *.db
+
+# Hide autogenerated sourcecode docs
+docs/source/autoapi

pytest_flakefighters-0.1.5/.readthedocs.yaml

@@ -0,0 +1,27 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.10"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+   configuration: docs/source/conf.py
+
+# If using Sphinx, optionally build your docs in additional formats such as PDF
+# formats:
+#    - pdf
+
+# Optionally declare the Python requirements required to build your docs
+python:
+   install:
+   - requirements: docs/source/requirements.txt
+   - method: pip
+     path: . # This should tell RTD to install the CTF from root to render the Estimators page correctly.

{pytest_flakefighters-0.0.0 → pytest_flakefighters-0.1.5}/PKG-INFO

@@ -1,32 +1,9 @@
 Metadata-Version: 2.4
 Name: pytest-flakefighters
-Version: 0.0.0
+Version: 0.1.5
 Summary: Pytest plugin implementing flaky test failure detection and classification.
 Author: TestFLARE Team
-License: 
-        The MIT License (MIT)
-        
-        Copyright (c) 2025 TestFLARE Team
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy
-        of this software and associated documentation files (the "Software"), to deal
-        in the Software without restriction, including without limitation the rights
-        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-        copies of the Software, and to permit persons to whom the Software is
-        furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in
-        all copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-        THE SOFTWARE.
-        
-Project-URL: Repository, https://github.com/test-flare/pytest-flakefighter
+Project-URL: Repository, https://github.com/test-flare/pytest-flakefighters
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -49,6 +26,8 @@ Requires-Dist: pre_commit; extra == "dev"
 Requires-Dist: sphinx-autoapi; extra == "dev"
 Requires-Dist: sphinx_rtd_theme; extra == "dev"
 Requires-Dist: tox>=4.31.0; extra == "dev"
+Requires-Dist: myst_parser; extra == "dev"
+Requires-Dist: nbsphinx; extra == "dev"
 Dynamic: license-file
 
 # Pytest FlakeFighters

pytest_flakefighters-0.1.5/docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

pytest_flakefighters-0.1.5/docs/source/_static/css/custom.css

@@ -0,0 +1,49 @@
+/* Custom CSS */
+
+/* Note: /*
+The CSS below will only work if 'logo_only':True in conf.py, but will require adjusting if False.
+ */
+
+.wy-side-nav-search a .logo {
+    max-width: 200px;
+    max-height: 170px;
+    height: auto;
+    width:auto;
+}
+
+.wy-nav-content {
+    background-color: white;
+}
+
+
+.wy-nav-top a {
+    color: #131E29;  /* Change the colour of the homepage navigation title on mobiles */
+}
+
+.wy-menu-vertical p.caption {
+
+    color: #9ADBE8; /* Change the colour of captions*/
+}
+
+
+.rst-content code.literal, .highlight .s2, .highlight .s1, .rst-content tt.literal{
+    color: #440099;
+    font-weight: bold;
+}
+
+
+.sig {
+    word-wrap: break-word;
+    word-break: break-word;
+    white-space: normal !important;
+}
+
+.sig-param {
+    display: inline-block;
+}
+
+dl.class > dt {
+    word-wrap: break-word;
+    overflow-wrap: break-word;
+    white-space: normal !important;
+}

pytest_flakefighters-0.1.5/docs/source/acknowlegements.rst

@@ -0,0 +1,10 @@
+Acknowledgements
+================
+
+The TestFLARE project is funded by EPSRC grant `EP/X024539/1 <https://gtr.ukri.org/projects?ref=EP%2FX024539%2F1>`_.
+
+
+The TestFLARE Team
+------------------
+
+- Phil McMinn, Owain Parry, Michael Foster, Rimsha Chaudhry, Gregory Kapfhammer, Michael Hilton

{pytest_flakefighters-0.0.0/docs → pytest_flakefighters-0.1.5/docs/source}/conf.py

@@ -11,14 +11,17 @@
 # serve to show the default.
 
 import os
-import shlex
 import sys
+from datetime import date
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 # sys.path.insert(0, os.path.abspath('.'))
 
+# Include root directory of project
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")))
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -28,9 +31,17 @@ import sys
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    "sphinx.ext.ifconfig",
+    "autoclasstoc",
+    "autoapi.extension",
+    "myst_parser",
+    "nbsphinx",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
 ]
 
+# Get autosummary to work with templates
+autosummary_generate = True
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 
@@ -46,25 +57,25 @@ source_suffix = ".rst"
 master_doc = "index"
 
 # General information about the project.
-project = "pytest-flakefighters"
-copyright = "2015, TestFLARE Team"
-author = "TestFLARE Team"
+project = "Pytest Flakefighters"
+copyright = f"{date.today().year}, TestFLARE Team"
+author = "Michael Foster, Owain Parry, Rimsha Choudhry, Phil McMinn"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = "0.0.1"
+# version = "0.0.1"
 # The full version, including alpha/beta/rc tags.
-release = "0.0.1"
+# release = "0.0.1"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+# language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -108,7 +119,13 @@ todo_include_todos = False
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = "alabaster"
+html_theme = "sphinx_rtd_theme"
+
+# Path to generate documentation from using Sphinx AutoAPI
+autoapi_dirs = [os.path.abspath(os.path.join("..", "..", "src"))]
+autoapi_generate_api_docs = True
+autoapi_keep_files = True
+
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -127,17 +144,23 @@ html_theme = "alabaster"
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-# html_logo = None
+html_logo = "_static/images/logo.png"
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-# html_favicon = None
+html_favicon = "_static/images/favicon.png"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
+html_css_files = ["css/custom.css"]
+
+html_theme_options = {
+    "style_nav_header_background": "#9ADBE8",  # Set the colour using CSS
+    "logo_only": True,
+}
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied

pytest_flakefighters-0.1.5/docs/source/configuration.rst

@@ -0,0 +1,26 @@
+Configuration
+================
+
+The flakefighters plugin implements several cutting edge flaky test detection tools from the research community.
+Each one is individually configurable and can be run individually or with other flakefighters.
+You can control which flakefighters to run and provide additional configuration options from your :code:`pyproject.toml` file by including sections of the following form for each flakefighter you want to run.
+Here, :code:`<FlakeFighterName>` is the class name of the flakefighter you wish to configure, assuming the class is in the :code:`pytest-flakefighters/flakefighters` directory.
+::
+[tool.pytest.ini_options.pytest_flakefighters.<FlakeFighterName>]
+run_live=[true/false]
+option1=value1
+option2=value2
+...
+
+Every flakefighter will have a :code:`run_live` option, which can be set to :code:`true` to classify each test execution as flaky immediately after it is run, or :code:`false` to clasify all tests at once at the end, although individual flakefighters may only support one particular option.
+Individual flakefighters have their own configurable options.
+These are detailed below.
+
+.. autoclass:: pytest_flakefighters.flakefighters.coverage_independence.CoverageIndependence
+  :noindex:
+.. autoclass:: pytest_flakefighters.flakefighters.deflaker.DeFlaker
+  :noindex:
+.. autoclass:: pytest_flakefighters.flakefighters.traceback_matching.TracebackMatching
+  :noindex:
+.. autoclass:: pytest_flakefighters.flakefighters.traceback_matching.CosineSimilarity
+  :noindex:

pytest_flakefighters-0.1.5/docs/source/dev/actions_and_webhooks.rst

@@ -0,0 +1,29 @@
+Github Actions and Webhooks
+===========================
+
+Actions
+--------------
+
+The Causal Testing Framework makes use of 5 `Github Actions <https://github.com/features/actions>`_,
+which can be found in the
+`.github/workflows <https://github.com/test-flare/pytest-flakefighters/tree/main/.github/workflows>`_ directory. These include:
+
+#.  ``ci-tests-drafts.yaml``, which runs continuous integration (CI) tests on each on each draft pull request.
+
+#.  ``ci-tests.yaml``, which runs continuous integration (CI) tests on each on each pull request.
+
+#.  ``ci-mega-linter.yaml``, which runs linting on each pull request.
+
+#.  ``publish-to-pypi.yaml``, runs when a new version tag is pushed and publishes that tag version to PyPI.
+
+Webhooks
+---------------
+
+We also use two `Webhooks <https://docs.github.com/en/webhooks-and-events/webhooks/about-webhooks>`_, which can
+be found in the `project settings <https://github.com/test-flare/pytest-flakefighters/settings>`_ on Github. These
+include:
+
+
+#.  `Codecov <https://github.com/codecov>`_
+
+#.  `Read the Docs <https://github.com/readthedocs>`_

pytest_flakefighters-0.1.5/docs/source/dev/documentation.rst

@@ -0,0 +1,58 @@
+Project Documentation
+=====================
+
+This page describes:
+
+#. Our project's documentation style,
+
+#. The tools used for documentation,
+
+#. Read the Docs where the documentation for this project is hosted.
+
+
+Style and Tools
+-------------------
+
+The `Sphinx docstring format <https://sphinx-rtd-tutorial.readthedocs.io/en/latest/docstrings.html#the-sphinx-docstring-format>`_ is used
+throughout our project's codebase to allow for the easy understanding of classes, methods and functions. This format
+allows for the easy generation of html documentation pages. It also checks for docstrings that have been added to the project's pyLint configuration.
+
+To install the packages required to work with the documentation, please ensure the projects **dev** dependencies are installed::
+
+    pip install pytest-flakefighters[dev]
+
+Sphinx
+******
+
+This project makes use of `Sphinx <https://www.sphinx-doc.org/en/master/>`_, to generate documentation. The documentation for the project sits within the `docs/` directory inside the project root.
+To manually build the docs, first navigate to the `docs/` directory and run::
+
+    make html
+
+This will populate `docs/build/` with static html pages containing the docs.
+To cleanup the compiled docs you can run::
+
+    make clean
+
+
+
+The source reStructuredText files (``.rst``) files can be found within the ``docs/source`` directory, which contains the
+handwritten documentation pages and get compiled by the ``make`` commands.
+
+Autodoc & AutoAPI
+*****************
+
+`Autodoc <https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html>`_ is an extension to Sphinx that can import
+code modules and compile documentation from their docstrings.
+
+`AutoAPI <https://sphinx-autoapi.readthedocs.io/en/latest/>`_ is a third party Sphinx tool for recursively discovering
+code modules and compiling them into a logical doctree structure
+
+The configuration for Sphinx, Autodoc and AutoAPI can all be found in `/docs/source/conf.py <https://github.com/test-flare/pytest-flakefighters/blob/main/docs/source/conf.py>`_.
+
+ReadTheDocs
+-----------
+`Read the Docs <https://readthedocs.org/>`_ is a documentation hosting site that hosts, versions and builds documentation
+for free for open source projects.
+
+This project makes use of a Github Webhook to trigger the build in ReadTheDocs, further reading on this can be found :doc:`here <../dev/actions_and_webhooks>`\.

pytest_flakefighters-0.1.5/docs/source/dev/version_release.rst

@@ -0,0 +1,31 @@
+Version Releases
+================
+
+This project follows the `Semantic Versioning 2.0.0 <https://semver.org/>`_ style for code releases.
+This page describes the release process for a new framework version on the `Project Github <https://github.com/test-flare/pytest-flakefighters>`_.
+
+How to release
+--------------
+
+#. Once your PR(s) are merged, navigate to https://github.com/test-flare/pytest-flakefighters/releases, which can be found on the right hand side of the projects Github main page by clicking on 'Releases'.
+
+#. Press the **Draft a new release** button in the top right of the releases page.
+
+#. Press the **Choose a tag** button and add the new version following the Semantic Version guidelines.
+   Please include the 'v' before the tag, e.g. **v0.0.0**.
+
+#. Enter the same tag name into the **Release Title** box.
+
+#. Press **Generate Release Notes** button.
+
+#. Add any additional information that may be helpful to the release notes. If there are breaking changes for example, which modules will they affect?
+
+#. Ensure the **Set as the latest release** checkbox is selected.
+
+#. Press publish release.
+
+#. Check that the Github Action worker, found in the `Actions tab <https://github.com/test-flare/pytest-flakefighters/actions>`_ has successfully completed. The typical time to publish to PyPI is around 2 minutes.
+
+#. Check on the projects `PyPI page <https://pypi.org/project/pytest-flakefighters/>`_ that the latest release is ready!
+
+From here the latest version can be installed using the common pip version commands. e.g. ``pip install pytest-flakefighters==0.0.0``

pytest_flakefighters-0.1.5/docs/source/glossary.rst

@@ -0,0 +1,7 @@
+Glossary
+########
+
+.. glossary::
+
+  Flaky tests
+    Tests that intermittently pass and fail without changes to test or project source code.

pytest_flakefighters-0.1.5/docs/source/index.rst

@@ -0,0 +1,114 @@
+.. pytest-flakefighters documentation master file, created by
+   sphinx-quickstart on Thu Oct  1 00:43:18 2015.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to pytest Flakefighters
+===============================================================
+
+|status|
+|ci-tests|
+|code-cov|
+|pypi|
+|docs|
+|python|
+|license|
+
+
+Motivation
+----------
+
+:term:`Flaky tests` intermittently pass and fail without changes to test or project source code, often without an obvious cause.
+When flaky tests proliferate, developers may loose faith in their test suites, potentially exposing end-users to the consequences of software failures.
+
+The Extension
+-------------
+
+Flakefighters is a pytest extension that provides a "Swiss army knife" of techniques to detect flaky tests.
+The extension incorporates several cutting edge flaky test detection techniques from research to automatically classify test failures as either genuine: indicating either a fault in the code or a mis-specified test case, or flaky: indicating a test with a nondeterministic outcome.
+Flaky tests are then reported separately in the test report, and can be optionally suppressed so they don't block CI/CD pipelines.
+
+
+.. toctree::
+   :hidden:
+   :caption: Home
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+   :caption: Introduction
+
+   installation
+   configuration
+
+
+.. toctree::
+   :maxdepth: 2
+   :caption: API
+   :hidden:
+   :titlesonly:
+
+   /autoapi/index
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+   :caption: Glossary
+
+   glossary
+
+.. toctree::
+  :hidden:
+  :maxdepth: 1
+  :caption: Development
+
+  /dev/version_release
+  /dev/documentation
+  /dev/actions_and_webhooks
+
+.. toctree::
+   :caption: Useful Links
+   :hidden:
+   :maxdepth: 2
+
+   Source code <https://github.com/test-flare/pytest-flakefighters/>
+   Documentation <https://causal-testing-framework.readthedocs.io/en/latest/>
+   PyPI <https://pypi.org/project/pytest-flakefighters/>
+   TestFLARE Homepage <https://test-flare.github.io/>
+
+.. toctree::
+   :hidden:
+   :maxdepth: 2
+   :caption: Acknowledgements
+
+   acknowlegements
+
+.. Define variables for our GH badges
+
+.. |ci-tests| image:: https://github.com/test-flare/pytest-flakefighters/actions/workflows/ci-tests.yaml/badge.svg
+   :target: https://github.com/test-flare/pytest-flakefighters/actions/workflows/ci-tests.yaml
+   :alt: Continuous Integration Tests
+
+.. |code-cov| image:: https://codecov.io/gh/test-flare/pytest-flakefighters/branch/main/graph/badge.svg?token=04ijFVrb4a
+   :target: https://codecov.io/gh/test-flare/pytest-flakefighters
+   :alt: Code coverage
+
+.. |docs| image:: https://readthedocs.org/projects/pytest-flakefighters/badge/?version=latest
+   :target: https://pytest-flakefighters.readthedocs.io/en/latest/?badge=latest
+   :alt: Documentation
+
+.. |python| image:: https://img.shields.io/badge/dynamic/toml?url=https%3A%2F%2Fraw.githubusercontent.com%2Ftest-flare%2Fpytest-flakefighters%2Fmain%2Fpyproject.toml&query=%24.project%5B'requires-python'%5D&label=python
+   :target: https://img.shields.io/badge/dynamic/toml?url=https%3A%2F%2Fraw.githubusercontent.com%2Ftest-flare%2Fpytest-flakefighters%2Fmain%2Fpyproject.toml&query=%24.project%5B'requires-python'%5D&label=python
+   :alt: Python
+
+.. |status| image:: https://www.repostatus.org/badges/latest/active.svg
+   :target: https://www.repostatus.org/#active
+   :alt: Status
+
+.. |pypi| image:: https://img.shields.io/pypi/v/pytest-flakefighters
+  :target: https://pypi.org/project/pytest-flakefighters
+  :alt: PyPI
+
+.. |license| image:: https://img.shields.io/github/license/test-flare/pytest-flakefighters
+   :target: https://github.com/test-flare/pytest-flakefighters
+   :alt: License

pytest_flakefighters-0.1.5/docs/source/installation.rst

@@ -0,0 +1,65 @@
+Getting Started
+================
+
+Installation
+-----------------
+* We currently support Python versions 3.10, 3.11, 3.12, and 3.13.
+
+* The Flakefighters plugin can be installed through the `Python Package Index (PyPI)`_ (recommended), or directly from source (recommended for contributors).
+
+.. _Python Package Index (PyPI): https://pypi.org/project/pytest-flakefighters
+
+Method 1: Installing via pip
+..............................
+
+To install the Causal Testing Framework using :code:`pip` for the latest stable version::
+
+    pip install pytest-flakefighters
+
+The plugin will then automatically run as part of `pytest`.
+
+If you also want to install the framework with (optional) development packages/tools::
+
+    pip install pytest-flakefighters[dev]
+
+
+Method 2: Installing via Source (For Developers/Contributors)
+...............................................................
+
+If you're planning to contribute to the project or need an editable installation for development, you can install directly from source::
+
+    git clone https://github.com/test-flare/pytest-flakefighters.git
+    cd pytest-flakefighters
+
+then, to install a specific release::
+
+    git fetch --all --tags --prune
+    git checkout tags/<tag> -b <branch>
+    pip install . # For core API only
+    pip install -e . # For editable install, useful for development work
+
+e.g. version `1.0.0`::
+
+    git fetch --all --tags --prune
+    git checkout tags/1.0.0 -b version
+    pip install .
+
+or to install the latest development version::
+
+    pip install .
+
+To also install developer tools::
+
+    pip install -e .[dev]
+
+Verifying Your Installation
+-----------------------------
+
+After installation, verify that the framework is installed correctly in your environment::
+
+    python -c "import pytest_flakefighters; print(pytest_flakefighters.__version__)"
+
+Next Steps
+-----------
+
+* Check out the :doc:`configuration` to learn how to configure the extension.