pytest-flakefighters 0.2.3__py3-none-any.whl → 0.3.1__py3-none-any.whl

_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.2.3'
-__version_tuple__ = version_tuple = (0, 2, 3)
+__version__ = version = '0.3.1'
+__version_tuple__ = version_tuple = (0, 3, 1)
 
 __commit_id__ = commit_id = None

pytest_flakefighters/config.py

@@ -0,0 +1,90 @@
+"""
+This module defines all the configuration options in a dictionary.
+Keys should be of the form `(--long-name, -L)` or just `(--long-name,)`.
+Options can then be specified on the commandline as `--long-name` or in a configuration file as `long_name`.
+Options specified on the commandline will override those specified in configuration files.
+"""
+
+import os
+
+from pytest_flakefighters.rerun_strategies import All, FlakyFailure, PreviouslyFlaky
+
+rerun_strategies = {"ALL": All, "FLAKY_FAILURE": FlakyFailure, "PREVIOUSLY_FLAKY": PreviouslyFlaky}
+
+
+options = {
+    ("--root",): {
+        "dest": "root",
+        "action": "store",
+        "default": os.getcwd(),
+        "help": "The root directory of the project. Defaults to the current working directory.",
+    },
+    ("--suppress-flaky-failures-exit-code",): {
+        "dest": "suppress_flaky",
+        "action": "store_true",
+        "default": False,
+        "help": "Return OK exit code if the only failures are flaky failures.",
+    },
+    ("--no-save",): {
+        "action": "store_true",
+        "default": False,
+        "help": "Do not save this run to the database of previous flakefighters runs.",
+    },
+    ("--function-coverage",): {
+        "action": "store_true",
+        "default": False,
+        "help": "Use function-level coverage instead of line coverage.",
+    },
+    ("--load-max-runs", "-M"): {
+        "action": "store",
+        "default": None,
+        "help": "The maximum number of previous runs to consider.",
+    },
+    ("--database-url", "-D"): {
+        "action": "store",
+        "default": "sqlite:///flakefighters.db",
+        "help": "The database URL. Defaults to 'flakefighters.db' in current working directory.",
+    },
+    ("--store-max-runs",): {
+        "action": "store",
+        "default": None,
+        "type": int,
+        "help": "The maximum number of previous flakefighters runs to store. Default is to store all.",
+    },
+    ("--max-reruns",): {
+        "action": "store",
+        "default": 0,
+        "type": int,
+        "help": "The maximum number of times to rerun tests. "
+        "By default, only failing tests marked as flaky will be rerun. "
+        "This can be changed with the --rerun-strategy parameter.",
+    },
+    ("--rerun-strategy",): {
+        "action": "store",
+        "type": str,
+        "choices": list(rerun_strategies),
+        "default": "FLAKY_FAILURE",
+        "help": "The strategy used to determine which tests to rerun. Supported options are:\n  "
+        + "\n  ".join(f"{name} - {strat.help()}" for name, strat in rerun_strategies.items()),
+    },
+    ("--time-immemorial",): {
+        "action": "store",
+        "default": None,
+        "help": "How long to store flakefighters runs for, specified as `days:hours:minutes`. "
+        "E.g. to store tests for one week, use 7:0:0.",
+    },
+    ("--display-outcomes", "-O"): {
+        "action": "store",
+        "type": int,
+        "nargs": "?",  # Allows 0 or 1 arguments
+        "const": 0,  # Value used if -O is present but no value is provided
+        "default": 0,  # Value used if -O is not present at all
+        "help": "Display historical test outcomes of the specified number of previous runs."
+        "If no value is specified, then display only the current verdict.",
+    },
+    ("--display-verdicts",): {
+        "action": "store_true",
+        "default": False,
+        "help": "Display the flaky classification verdicts alongside test outcomes.",
+    },
+}

pytest_flakefighters/database_management.py

@@ -34,10 +34,12 @@ from sqlalchemy.orm import (
 class Base(DeclarativeBase):
     """
     Declarative base class for data objects.
+
+    :ivar id: Unique autoincrementing ID for the object.
     """
 
     id: Mapped[int] = Column(Integer, primary_key=True)  # pylint: disable=C0103
-    # @pytest, these are not the tests you're looking for...
+    # Explicitly flag that we don't want pytest to collect our Test, TestExecution, etc. classes.
     __test__ = False  # pylint: disable=C0103
 
     @declared_attr
@@ -49,8 +51,16 @@ class Base(DeclarativeBase):
 class Run(Base):
     """
     Class to store attributes of a flakefighters run.
+    :ivar start_time: The time the test run was begun.
+    :ivar created_at: The time the entry was added to the database.
+    This is not necessarily equivalent to start_time if the test suite took a long time to run or
+    if the entry was migrated from a separate database.
+    :ivar root: The root directory of the project.
+    :ivar tests: The test suite.
+    :ivar active_flakefighters: The flakefighters that are active on the run.
     """
 
+    start_time = Column(DateTime)
     created_at = Column(DateTime, default=func.now())
     root: Mapped[str] = Column(String)
     tests = relationship("Test", backref="run", lazy="subquery", cascade="all, delete", passive_deletes=True)
@@ -63,6 +73,10 @@ class Run(Base):
 class ActiveFlakeFighter(Base):
     """
     Store relevant information about the active flakefighters.
+
+    :ivar run_id: Foreign key of the related run.
+    :ivar name: Class name of the flakefighter.
+    :ivar params: The parameterss of the flakefighter.
     """
 
     run_id: Mapped[int] = Column(Integer, ForeignKey("run.id"), nullable=False)
@@ -74,6 +88,17 @@ class ActiveFlakeFighter(Base):
 class Test(Base):
     """
     Class to store attributes of a test case.
+
+    :ivar run_id: Foreign key of the related run.
+    :ivar fspath: File system path of the file containing the test definition.
+    :ivar line_no: Line number of the test definition.
+    :ivar name: Name of the test case.
+    :ivar skipped: Boolean true if the test was skipped, else false.
+    :ivar executions: List of execution attempts.
+    :ivar flakefighter_results: List of test-level flakefighter results.
+
+    .. note::
+      Execution-level flakefighter results will be stored inside the individual TestExecution objects
     """
 
     run_id: Mapped[int] = Column(Integer, ForeignKey("run.id"), nullable=False)
@@ -104,6 +129,16 @@ class Test(Base):
 class TestExecution(Base):  # pylint: disable=R0902
     """
     Class to store attributes of a test outcome.
+
+    :ivar test_id: Foreign key of the related test.
+    :ivar outcome: Outcome of the test. One of "passed", "failed", or "skipped".
+    :ivar stdout: The captured stdout string.
+    :ivar stedrr: The captured stderr string.
+    :ivar start_time: The start time of the test.
+    :ivar end_time: The end time of the test.
+    :ivar coverage: The line coverage of the test.
+    :ivar flakefighter_results: The execution-level flakefighter results.
+    :ivar exception: The exception associated with the test if one was thrown.
     """
 
     __tablename__ = "test_execution"
@@ -140,6 +175,10 @@ class TestExecution(Base):  # pylint: disable=R0902
 class TestException(Base):  # pylint: disable=R0902
     """
     Class to store information about the exceptions that cause tests to fail.
+
+    :ivar execution_id: Foreign key of the related execution.
+    :ivar name: Name of the exception.
+    :traceback: The full stack of traceback entries.
     """
 
     __tablename__ = "test_exception"
@@ -155,6 +194,13 @@ class TestException(Base):  # pylint: disable=R0902
 class TracebackEntry(Base):  # pylint: disable=R0902
     """
     Class to store attributes of entries in the stack trace.
+
+    :ivar exception_id: Foreign key of the related exception.
+    :ivar path: Filepath of the source file.
+    :ivar lineno: Line number of the executed statement.
+    :ivar colno: Column number of the executed statement.
+    :ivar statement: The executed statement.
+    :ivar source: The surrounding source code.
     """
 
     exception_id: Mapped[int] = Column(Integer, ForeignKey("test_exception.id"), nullable=False)
@@ -169,6 +215,11 @@ class TracebackEntry(Base):  # pylint: disable=R0902
 class FlakefighterResult(Base):  # pylint: disable=R0902
     """
     Class to store flakefighter results.
+
+    :ivar test_execution_id: Foreign key of the related test execution. Should not be set if test_id is present.
+    :ivar test_id: Foreign key of the related test. Should not be set if test_execution_id is present.
+    :ivar name: Name of the flakefighter.
+    :ivar flaky: Boolean true if the test (execution) was classified as flaky.
     """
 
     __tablename__ = "flakefighter_result"
@@ -182,10 +233,25 @@ class FlakefighterResult(Base):  # pylint: disable=R0902
         CheckConstraint("NOT (test_execution_id IS NULL AND test_id IS NULL)", name="check_test_id_not_null"),
     )
 
+    @property
+    def classification(self):
+        """
+        Return the classification as a string.
+        "flaky" if the test was classified as flaky, else "genuine".
+        """
+        return "flaky" if self.flaky else "genuine"
+
 
 class Database:
     """
     Class to handle database setup and interaction.
+
+    :ivar engine: The database engine.
+    :ivar store_max_runs: The maximum number of previous runs that should be stored. If the database exceeds this size,
+                          older runs will be pruned to make space for newer ones.
+    :ivar time_immemorial: Time before which runs should not be considered. Runs before this date will be pruned when
+                           saving new runs.
+    :ivar previous_runs: List of previous flakefighter runs with most recent first.
     """
 
     def __init__(
@@ -230,4 +296,4 @@ class Database:
         :param limit: The maximum number of runs to return (these will be most recent runs).
         """
         with Session(self.engine) as session:
-            return session.scalars(select(Run).order_by(desc(Run.id)).limit(limit)).all()
+            return session.scalars(select(Run).order_by(desc(Run.start_time)).limit(limit)).all()

pytest_flakefighters/flakefighters/deflaker.py

@@ -68,7 +68,10 @@ class DeFlaker(FlakeFighter):
         self.method_declarations = {}
         for file, lines in self.lines_changed.items():
             with open(file) as f:
-                tree = ast.parse(f.read())
+                try:
+                    tree = ast.parse(f.read())
+                except SyntaxError:
+                    continue
 
             self.method_declarations[file] = [
                 node.lineno
@@ -133,9 +136,5 @@ class DeFlaker(FlakeFighter):
         :param run: Run object representing the pytest run, with tests accessible through run.tests.
         """
         for test in run.tests:
-            test.flakefighter_results.append(
-                FlakefighterResult(
-                    name=self.__class__.__name__,
-                    flaky=any(self._flaky_execution(execution) for execution in test.executions),
-                )
-            )
+            for execution in test.executions:
+                self.flaky_test_live(execution)

pytest_flakefighters/flakefighters/traceback_matching.py

@@ -49,7 +49,7 @@ class TracebackMatching(FlakeFighter):
         Convert the key parameters into a dictionary so that the object can be replicated.
         :return A dictionary of the parameters used to create the object.
         """
-        return {"root": self.root}
+        return {"run_live": self.run_live, "root": self.root}
 
     def _flaky_execution(self, execution, previous_executions) -> bool:
         """
@@ -66,15 +66,13 @@ class TracebackMatching(FlakeFighter):
         ]
         return any(e == current_traceback for e in previous_executions)
 
-    def previous_flaky_executions(self, runs: list[Run] = None) -> list:
+    def previous_flaky_executions(self, runs: list[Run]) -> list:
         """
         Extract the relevant information from previous flaky executions and collapse into a single list.
         :param runs: The runs to consider. Defaults to self.previous_runs.
         :return: List containing the relative path, line number, column number, and code statement of all previous
         test executions.
         """
-        if runs is None:
-            runs = self.previous_runs
         return [
             [
                 (os.path.relpath(elem.path, run.root), elem.lineno, elem.colno, elem.statement)
@@ -87,17 +85,20 @@ class TracebackMatching(FlakeFighter):
             if execution.exception
         ]
 
-    def flaky_test_live(self, execution: TestExecution):
+    def flaky_test_live(self, execution: TestExecution, previous_runs: list[Run] = None):
         """
         Classify executions as flaky if they have the same failure logs as a flaky execution.
         :param execution: Test execution to consider.
+        :param previous_runs: The previous runs to which the execution will be compared.
         """
+        if previous_runs is None:
+            previous_runs = self.previous_runs
         execution.flakefighter_results.append(
             FlakefighterResult(
                 name=self.__class__.__name__,
                 flaky=self._flaky_execution(
                     execution,
-                    self.previous_flaky_executions(),
+                    self.previous_flaky_executions(previous_runs),
                 ),
             )
         )
@@ -109,14 +110,7 @@ class TracebackMatching(FlakeFighter):
         """
         for test in run.tests:
             for execution in test.executions:
-                execution.flakefighter_results.append(
-                    FlakefighterResult(
-                        name=self.__class__.__name__,
-                        flaky=self._flaky_execution(
-                            execution, self.previous_flaky_executions(self.previous_runs + [run])
-                        ),
-                    )
-                )
+                self.flaky_test_live(execution, self.previous_runs + [run])
 
 
 class CosineSimilarity(TracebackMatching):

pytest_flakefighters/main.py

@@ -3,13 +3,13 @@ This module adds all the FlakeFighter configuration options to pytest.
 """
 
 import logging
-import os
 from importlib.metadata import entry_points
 
 import coverage
 import pytest
 import yaml
 
+from pytest_flakefighters.config import options
 from pytest_flakefighters.database_management import Database
 from pytest_flakefighters.flakefighters.deflaker import DeFlaker
 from pytest_flakefighters.function_coverage import Profiler
@@ -35,79 +35,45 @@ def pytest_addoption(parser: pytest.Parser):
     Add extra pytest options.
     :param parser: The argument parser.
     """
+    # Allows users to specify flakefighter configurations in their pyproject.toml file without pytest throwing out
+    # "unknown configuration option" warnings
+    parser.addini("pytest_flakefighters", type="args", help="Configuration for the pytest-flakefighters extension")
+
+    def datatype(details):
+        if "type" not in details:
+            return None
+        if details["type"] is str:
+            return "string"
+        return str(details["type"].__name__)
+
     group = parser.getgroup("flakefighters")
-    group.addoption(
-        "--root",
-        dest="root",
-        action="store",
-        default=os.getcwd(),
-        help="The root directory of the project. Defaults to the current working directory.",
-    )
-    group.addoption(
-        "--suppress-flaky-failures-exit-code",
-        dest="suppress_flaky",
-        action="store_true",
-        default=False,
-        help="Return OK exit code if the only failures are flaky failures.",
-    )
-    group.addoption(
-        "--no-save",
-        action="store_true",
-        default=False,
-        help="Do not save this run to the database of previous flakefighters runs.",
-    )
-    group.addoption(
-        "--function-coverage",
-        action="store_true",
-        default=False,
-        help="Use function-level coverage instead of line coverage.",
-    )
-    group.addoption(
-        "--load-max-runs",
-        "-M",
-        action="store",
-        default=None,
-        help="The maximum number of previous runs to consider.",
-    )
-    group.addoption(
-        "--database-url",
-        "-D",
-        action="store",
-        default="sqlite:///flakefighters.db",
-        help="The database URL. Defaults to 'flakefighters.db' in current working directory.",
-    )
-    group.addoption(
-        "--store-max-runs",
-        action="store",
-        default=None,
-        type=int,
-        help="The maximum number of previous flakefighters runs to store. Default is to store all.",
-    )
-    group.addoption(
-        "--max-reruns",
-        action="store",
-        default=0,
-        type=int,
-        help="The maximum number of times to rerun tests. "
-        "By default, only failing tests marked as flaky will be rerun. "
-        "This can be changed with the --rerun-strategy parameter.",
-    )
-    group.addoption(
-        "--rerun-strategy",
-        action="store",
-        type=str,
-        choices=list(rerun_strategies),
-        default="FLAKY_FAILURE",
-        help="The strategy used to determine which tests to rerun. Supported options are:\n  "
-        + "\n  ".join(f"{name} - {strat.help()}" for name, strat in rerun_strategies.items()),
-    )
-    group.addoption(
-        "--time-immemorial",
-        action="store",
-        default=None,
-        help="How long to store flakefighters runs for, specified as `days:hours:minutes`. "
-        "E.g. to store tests for one week, use 7:0:0.",
-    )
+    for name, details in options.items():
+        # Add a commandline option with short name if provided, e.g. "--custom-option"
+        # We need the default to be None here so that we can test if the user has provided it
+        group.addoption(*name, **(details | {"default": None}))
+        # Add configuration file option with no "--" and "-" replaced by "_"
+        parser.addini(
+            name[0][2:].replace("-", "_"),
+            help=details["help"],
+            default=details.get("default"),
+            type=datatype(details),
+        )
+
+
+def get_config_value(config, name):
+    """
+    Get the configuration value.
+    Options specified on the commandline will override those specified in configuration files.
+    If neither is specified, the default value specified in `options.py` will be used.
+    """
+    cli_val = config.getoption(name)
+    if cli_val is not None:
+        return cli_val
+
+    try:
+        return config.getini(name)
+    except ValueError:
+        return None
 
 
 def pytest_configure(config: pytest.Config):
@@ -116,52 +82,66 @@ def pytest_configure(config: pytest.Config):
     :param config: The config options.
     """
     database = Database(
-        config.option.database_url,
-        config.option.load_max_runs,
-        config.option.store_max_runs,
-        config.option.time_immemorial,
+        get_config_value(config, "database_url"),
+        get_config_value(config, "load_max_runs"),
+        get_config_value(config, "store_max_runs"),
+        get_config_value(config, "time_immemorial"),
     )
 
-    if config.option.function_coverage:
+    if get_config_value(config, "function_coverage"):
         cov = Profiler()
     else:
         cov = coverage.Coverage()
 
-    algorithms = entry_points(group="pytest_flakefighters")
+    algorithms = {ff.name: ff for ff in entry_points(group="pytest_flakefighters")}
     flakefighter_configs = config.inicfg.get("pytest_flakefighters")
 
     flakefighters = []
     if flakefighter_configs is not None:
-        if isinstance(flakefighter_configs, str):
-            flakefighter_configs = yaml.safe_load(flakefighter_configs)
-        elif hasattr(flakefighter_configs, "value"):
-            flakefighter_configs = yaml.safe_load(flakefighter_configs.value)
-        else:
-            raise TypeError(f"Unexpected type for config: {type(flakefighter_configs)}")
-        for flakefighter in algorithms:
-            if flakefighter.name in flakefighter_configs:
-                flakefighters.append(
-                    flakefighter.load().from_config(
-                        vars(config.option) | {"database": database} | flakefighter_configs[flakefighter.name]
+        # Can't measure coverage since the branch taken depends on the python version
+        if isinstance(flakefighter_configs, str):  # pragma: no cover
+            flakefighter_configs = yaml.safe_load(flakefighter_configs)  # pragma: no cover
+        elif hasattr(flakefighter_configs, "value"):  # pragma: no cover
+            flakefighter_configs = yaml.safe_load(flakefighter_configs.value)  # pragma: no cover
+        else:  # pragma: no cover
+            raise TypeError(f"Unexpected type for config: {type(flakefighter_configs)}")  # pragma: no cover
+        for module, classes in flakefighter_configs["flakefighters"].items():
+            for class_name, params in classes.items():
+                if class_name in algorithms:
+                    flakefighters.append(
+                        algorithms[class_name]
+                        .load()
+                        .from_config(
+                            {k: get_config_value(config, k) for k in vars(config.option)}
+                            | {"database": database}
+                            | params
+                        )
+                    )
+                else:
+                    raise ValueError(
+                        f"Could not load flakefighter {module}:{class_name}. Did you register its entry point?"
                     )
-                )
 
     else:
         logger.warning("No flakefighters specified. Using basic DeFlaker only.")
         flakefighters.append(
             DeFlaker(
                 run_live=True,
-                root=config.option.root,
+                root=get_config_value(config, "root"),
             )
         )
 
     config.pluginmanager.register(
         FlakeFighterPlugin(
-            root=config.option.root,
+            root=get_config_value(config, "root"),
             database=database,
             cov=cov,
             flakefighters=flakefighters,
-            rerun_strategy=rerun_strategy(config.option.rerun_strategy, config.option.max_reruns, database=database),
-            save_run=not config.option.no_save,
+            rerun_strategy=rerun_strategy(
+                get_config_value(config, "rerun_strategy"), get_config_value(config, "max_reruns"), database=database
+            ),
+            save_run=not get_config_value(config, "no_save"),
+            display_outcomes=get_config_value(config, "display_outcomes"),
+            display_verdicts=get_config_value(config, "display_verdicts"),
         )
     )

pytest_flakefighters/plugin.py

@@ -5,6 +5,7 @@ This module implements the DeFlaker algorithm [Bell et al. 10.1145/3180155.31801
 from datetime import datetime
 from enum import Enum
 from typing import Union
+from xml.etree import ElementTree as ET
 
 import coverage
 import pytest
@@ -49,6 +50,8 @@ class FlakeFighterPlugin:  # pylint: disable=R0902
         flakefighters: list[FlakeFighter],
         save_run: bool = True,
         rerun_strategy: RerunStrategy = RerunStrategy.FLAKY_FAILURE,
+        display_outcomes: int = 0,
+        display_verdicts: bool = False,
     ):
         self.root = root
         self.database = database
@@ -56,12 +59,16 @@ class FlakeFighterPlugin:  # pylint: disable=R0902
         self.flakefighters = flakefighters
         self.save_run = save_run
         self.rerun_strategy = rerun_strategy
+        self.test_reports = {}
+        self.display_verdicts = display_verdicts
+        self.display_outcomes = display_outcomes
 
         self.run = Run(  # pylint: disable=E1123
             root=root,
             active_flakefighters=[
                 ActiveFlakeFighter(name=f.__class__.__name__, params=f.params()) for f in flakefighters
             ],
+            start_time=datetime.now(),
         )
 
     def pytest_sessionstart(self, session: pytest.Session):  # pylint: disable=unused-argument
@@ -176,9 +183,54 @@ class FlakeFighterPlugin:  # pylint: disable=R0902
                     test.executions.append(test_execution)
                     for ff in filter(lambda ff: ff.run_live, self.flakefighters):
                         ff.flaky_test_live(test_execution)
+                    self.test_reports[item.nodeid] = report
                     report.flaky = any(result.flaky for result in test_execution.flakefighter_results)
+                    # Limited pytest-json support
+                    report.stage_metadata = {
+                        "executions": [
+                            {
+                                "start_time": x.start_time.isoformat(),
+                                "end_time": x.end_time.isoformat(),
+                                "outcome": test_execution.outcome,
+                                "flakefighter_results": {r.name: r.classification for r in x.flakefighter_results},
+                            }
+                            for x in test.executions
+                        ],
+                    }
+                    # html
+                    if hasattr(report, "extras"):
+                        report.extras.append(
+                            {
+                                "content": f"""
+                                <h4>Flakefighter Results</h4>
+                                <div id="ff-{report.nodeid.replace("::", "_")}"></div>
+                                <table style="width:100%"><tbody><tr>"""
+                                + "".join(
+                                    [
+                                        f"""
+                                        <td>
+                                        <p><strong>Start time:</strong> {execution.start_time}</p>
+                                        <p><strong>End time:</strong> {execution.end_time}</p>
+                                        <p><strong>Outcome:</strong> {execution.outcome}</p>
+                                        <p><strong>Flakefighter Results:</strong></p>
+                                        <ul>
+                                        {''.join(['<li><strong>'+result.name+':</strong> '+result.classification+'</li>'
+                                        for result in execution.flakefighter_results])}
+                                        </ul>
+                                        </td>
+                                        """
+                                        for execution in test.executions
+                                    ]
+                                )
+                                + "</tr></tbody></table>",
+                                "extension": "html",
+                                "format_type": "html",
+                                "mime_type": "text/html",
+                            }
+                        )
                     if item.execution_count <= self.rerun_strategy.max_reruns and self.rerun_strategy.rerun(report):
                         break  # trigger rerun
+
                 item.ihook.pytest_runtest_logreport(report=report)
             else:
                 break  # Skip further reruns
@@ -199,6 +251,128 @@ class FlakeFighterPlugin:  # pylint: disable=R0902
             return report.outcome, "F", ("FLAKY", {"yellow": True})
         return None
 
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_runtestloop(self, session: pytest.Session):  # pylint: disable=unused-argument
+        """
+        Run postprocessing flakefighters.
+        :param session: The pytest session object.
+        """
+        yield
+        for ff in filter(lambda ff: not ff.run_live, self.flakefighters):
+            ff.flaky_tests_post(self.run)
+        for test in self.run.tests:
+            if test.name in self.test_reports:
+                self.test_reports[test.name].flakefighter_results = {
+                    r.name: r.classification for r in test.flakefighter_results
+                }
+
+    @pytest.hookimpl(optionalhook=True)
+    def pytest_json_modifyreport(self, json_report: dict):
+        """
+        Add flakefighter results to the pytest-json-report report.
+
+        :param json_report: The report dict.
+        """
+        for t in json_report.get("tests", []):
+            t["call"]["metadata"] = self.test_reports[t["nodeid"]].stage_metadata
+
+            t["metadata"] = t.get("metadata", {}) | {
+                "flakefighter_results": self.test_reports[t["nodeid"]].flakefighter_results
+            }
+
+    def build_outcome_string(self, test: Test) -> str:
+        """
+        Construct a string to represent previous flakefighter outcomes for a given test and its associated executions.
+
+        :param test: The test case.
+        """
+        result_string = []
+        if test.flakefighter_results:
+            if self.display_verdicts:
+                result_string.append(
+                    "Overall\n" + "\n".join(f"  {f.name}: {f.classification}" for f in test.flakefighter_results) + "\n"
+                )
+        for i, execution in enumerate(test.executions):
+            if execution.flakefighter_results:
+                if self.display_verdicts:
+                    result_string.append(
+                        f"Execution {i}: {execution.outcome}\n"
+                        + "\n".join(
+                            f"  {f.name}: {'flaky' if f.flaky else 'genuine'}" for f in execution.flakefighter_results
+                        )
+                    )
+                else:
+                    result_string.append(f"Execution {i}: {execution.outcome}")
+        return "\n".join(result_string)
+
+    def modify_xml(self, xml_path: str):
+        """
+        Modify the JUnitXML file to add the flakefighter results for each test.
+
+        :param xml_path: The path of the saved XML file.
+        """
+        tree = ET.parse(xml_path)
+        for testsuite in tree.getroot().findall("testsuite"):
+            for testcase in testsuite.findall("testcase"):
+                splitname = testcase.get("classname").split(".")
+                splitname[0] = f"{splitname[0]}.py"
+                nodeid = "::".join(splitname + [testcase.get("name")])
+                flakefighter_results = ET.SubElement(testcase, "flakefighterresults")
+                if nodeid in self.test_reports:
+                    for execution in self.test_reports[nodeid].stage_metadata["executions"]:
+                        execution_results = ET.Element(
+                            "execution",
+                            {
+                                "outcome": execution["outcome"],
+                                "starttime": execution["start_time"],
+                                "endtime": execution["end_time"],
+                            },
+                        )
+                        flakefighter_results.append(execution_results)
+                        for name, classification in execution["flakefighter_results"].items():
+                            ET.SubElement(execution_results, name).text = classification
+                    test_results = ET.SubElement(flakefighter_results, "test")
+                    for name, classification in self.test_reports[nodeid].flakefighter_results.items():
+                        ET.SubElement(test_results, name).text = classification
+
+        tree.write(xml_path)
+
+    @pytest.hookimpl(optionalhook=True)
+    def pytest_html_results_summary(
+        self, prefix: list, summary: list, postfix: list
+    ):  # pylint: disable=unused-argument
+        """
+        Add the test-level flakefighter results.
+        :param prefix: The prefix content. UNUSED.
+        :param summary: The summary content. UNUSED.
+        :param postfix: The postfix content.
+        """
+        postfix.extend(
+            [
+                "<h2>Test-level flakefighter results</h2>",
+                "<table>",
+                "<thead><tr><td>Test</td><td>Flakefighter results</td></tr></thead>",
+                "<tbody>",
+            ]
+            + [
+                f"<tr><td>{nodeid}</td><td>"
+                + "".join(
+                    [
+                        f"""<ul>
+                            {''.join(['<li><strong>'+result['name']+':</strong> '+result['classification']+'</li>'
+                                for result in report.flakefighter_results])}
+                            </ul>"""
+                    ]
+                )
+                + "</td></tr>"
+                for nodeid, report in self.test_reports.items()
+            ]
+            + [
+                "</tbody>",
+                "</table>",
+            ]
+        )
+
     def pytest_sessionfinish(
         self,
         session: pytest.Session,
@@ -208,8 +382,23 @@ class FlakeFighterPlugin:  # pylint: disable=R0902
         :param session: The pytest session object.
         :param exitstatus: The status which pytest will return to the system.
         """
-        for ff in filter(lambda ff: not ff.run_live, self.flakefighters):
-            ff.flaky_tests_post(self.run)
+
+        if session.config.option.xmlpath:
+            self.modify_xml(session.config.option.xmlpath)
+
+        runs = [self.run]
+        if self.display_outcomes:
+            runs += self.database.load_runs(self.display_outcomes)
+        if self.display_verdicts or self.display_outcomes:
+            for run in runs:
+                for test in run.tests:
+                    if test.name in self.test_reports:
+                        self.test_reports[test.name].sections.append(
+                            (
+                                f"Flakefighter Verdicts {run.start_time if run != self.run else '(Current)'}",
+                                self.build_outcome_string(test),
+                            )
+                        )
 
         genuine_failure_observed = any(
             not test.flaky for test in self.run.tests if any(e.outcome != "passed" for e in test.executions)

{pytest_flakefighters-0.2.3.dist-info → pytest_flakefighters-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytest-flakefighters
-Version: 0.2.3
+Version: 0.3.1
 Summary: Pytest plugin implementing flaky test failure detection and classification.
 Author: TestFLARE Team
 Project-URL: Documentation, https://pytest-flakefighters.readthedocs.io
@@ -38,22 +38,26 @@ Requires-Dist: psycopg2>2.9; extra == "pg"
 Dynamic: license-file
 
 # Pytest FlakeFighters
-### Pytest plugin implementing flaky test failure detection and classification.
 
 [![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active)
 [![PyPI version](https://img.shields.io/pypi/v/pytest-flakefighters.svg)](https://pypi.org/project/pytest-flakefighters)
 [![Python versions](https://img.shields.io/badge/python-3.10_--_3.13-blue)](https://pypi.org/project/pytest-flakefighters)
 ![Test status](https://github.com/test-flare/pytest-flakefighters/actions/workflows/ci-tests.yaml/badge.svg)
 [![codecov](https://codecov.io/gh/test-flare/pytest-flakefighters/branch/main/graph/badge.svg?token=04ijFVrb4a)](https://codecov.io/gh/test-flare/pytest-flakefighters)
-[![Documentation Status](https://readthedocs.org/projects/causal-testing-framework/badge/?version=latest)](https://causal-testing-framework.readthedocs.io/en/latest/?badge=latest)
+[![Documentation Status](https://readthedocs.org/projects/pytest-flakefighters/badge/?version=latest)](https://pytest-flakefighters.readthedocs.io/en/latest/?badge=latest)
 ![GitHub License](https://img.shields.io/github/license/test-flare/pytest-flakefighters)
 
-
+### Pytest plugin implementing flaky test failure detection and classification.
+Read more about flaky tests [here](https://docs.pytest.org/en/stable/explanation/flaky.html).
 
 ## Features
 
--   Implements the [DeFlaker algorithm](https://deflaker.com/) for pytest
-
+- Implements the [DeFlaker algorithm](http://www.deflaker.org/get-rid-of-your-flakes/) for pytest
+- Implements two traceback-matching classifiers from [Alshammari et al. (2024)](https://doi.org/10.1109/ICST60714.2024.00031).
+- Implements a novel coverage-independence classifier that classifies tests as flaky if they fail independently of passing test cases that exercise overlapping code.
+- Optionally rerun flaky failures
+- Output results to JSON, HTML, or JUnitXML
+- Save test outcome history to a remote or local database
 
 ## Installation
 

pytest_flakefighters-0.3.1.dist-info/RECORD

@@ -0,0 +1,19 @@
+_version.py,sha256=gGLpQUQx-ty9SEy9PYw9OgJWWzJLBnCpfJOfzL7SjlI,704
+pytest_flakefighters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pytest_flakefighters/config.py,sha256=NBqbonht0O4L_P3hV_g__LiD6Fo5aGIau7D84hBA5kE,3460
+pytest_flakefighters/database_management.py,sha256=FZTDYG9uVlPY2j874I_YDFQ9kuNXhm4P9fL3ckrg8MY,10427
+pytest_flakefighters/function_coverage.py,sha256=IgzWHMFgkVrc68ZjQ1o27tGxKa8WvcMcGW8RWnpFLVw,2187
+pytest_flakefighters/main.py,sha256=QxhaCdrbJmMNgekUDRsLIwNDhnd_vaGQ56wpbYKcV1U,5541
+pytest_flakefighters/plugin.py,sha256=enB2bXXE76Swpn7aHrb2j8fx2oCHi2bcQAx9kLWrTOc,17489
+pytest_flakefighters/rerun_strategies.py,sha256=YI-spvm0dhB-gGM9IEgDz19NHvzD_XesZ8lDcBDGSJM,2536
+pytest_flakefighters/flakefighters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pytest_flakefighters/flakefighters/abstract_flakefighter.py,sha256=bchRwoNzQaFhSknT9wKDpbf3EiReHPsHsDhm5qi8y_o,1953
+pytest_flakefighters/flakefighters/coverage_independence.py,sha256=5swoY1jOaaGFlZNX6PdejBFVd7Gpi3zsaW5W6toIzk8,4480
+pytest_flakefighters/flakefighters/deflaker.py,sha256=eqqfhvKrxYIbbtm_pdqTw2AwmcHo5vBh3HfqyscusdU,5911
+pytest_flakefighters/flakefighters/traceback_matching.py,sha256=L-3vVdB9aTjGHrsCAh9R8L51YQoSfRUfhMioIywoJqM,6808
+pytest_flakefighters-0.3.1.dist-info/licenses/LICENSE,sha256=tTzR2CWQMPOp-mQIQqi0cTRkaogeBUmW06blQsBLdQg,1082
+pytest_flakefighters-0.3.1.dist-info/METADATA,sha256=qJ0VwrCDLgYhWnZASbHn17hivEt1XJVeszRfB49Ko6I,6044
+pytest_flakefighters-0.3.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+pytest_flakefighters-0.3.1.dist-info/entry_points.txt,sha256=xockvN1AszN2XqaET77JDIRdOafgm3DdvOtHgVw-aDU,424
+pytest_flakefighters-0.3.1.dist-info/top_level.txt,sha256=bWwe0xVZ_l2CP8KSnztW6FafZKQZ7zUTMa1U32geY28,30
+pytest_flakefighters-0.3.1.dist-info/RECORD,,

{pytest_flakefighters-0.2.3.dist-info → pytest_flakefighters-0.3.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.10.1)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

pytest_flakefighters-0.2.3.dist-info/RECORD

@@ -1,18 +0,0 @@
-_version.py,sha256=kBRz0P2plw1eVdIpt70W6m1LMbEIhLY3RyOfVGdubaI,704
-pytest_flakefighters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pytest_flakefighters/database_management.py,sha256=mSdrP4enD6_oT941y5KAP-Jae84FihVJEvqCUFPhrss,7105
-pytest_flakefighters/function_coverage.py,sha256=IgzWHMFgkVrc68ZjQ1o27tGxKa8WvcMcGW8RWnpFLVw,2187
-pytest_flakefighters/main.py,sha256=N0yGSGA-ZLtsKRCtSCIdnw7RPVEtLnXUhnftQGHjujA,5430
-pytest_flakefighters/plugin.py,sha256=kdagObAK93ab6-5lckwkBR846RPspxXen4qlEcsAjB8,8866
-pytest_flakefighters/rerun_strategies.py,sha256=YI-spvm0dhB-gGM9IEgDz19NHvzD_XesZ8lDcBDGSJM,2536
-pytest_flakefighters/flakefighters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pytest_flakefighters/flakefighters/abstract_flakefighter.py,sha256=bchRwoNzQaFhSknT9wKDpbf3EiReHPsHsDhm5qi8y_o,1953
-pytest_flakefighters/flakefighters/coverage_independence.py,sha256=5swoY1jOaaGFlZNX6PdejBFVd7Gpi3zsaW5W6toIzk8,4480
-pytest_flakefighters/flakefighters/deflaker.py,sha256=GjmAA5i7YFJKbNaheWihqUqyu9_z2NZwqy6U--ztj9c,5989
-pytest_flakefighters/flakefighters/traceback_matching.py,sha256=LheOEqCvrBDrZJm4RK4ZFFcebWs4ggP7qRiUPg8HQpM,6926
-pytest_flakefighters-0.2.3.dist-info/licenses/LICENSE,sha256=tTzR2CWQMPOp-mQIQqi0cTRkaogeBUmW06blQsBLdQg,1082
-pytest_flakefighters-0.2.3.dist-info/METADATA,sha256=k4LadBPIqRWtTIkjOtj1R_KQE8_zUh7ldLPvcETfvOI,5507
-pytest_flakefighters-0.2.3.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
-pytest_flakefighters-0.2.3.dist-info/entry_points.txt,sha256=xockvN1AszN2XqaET77JDIRdOafgm3DdvOtHgVw-aDU,424
-pytest_flakefighters-0.2.3.dist-info/top_level.txt,sha256=bWwe0xVZ_l2CP8KSnztW6FafZKQZ7zUTMa1U32geY28,30
-pytest_flakefighters-0.2.3.dist-info/RECORD,,