pytest-flakefighters 0.0.0__py3-none-any.whl

pytest_flakefighters/database_management.py

@@ -0,0 +1,231 @@
+"""
+This module manages all interaction with the test run database.
+"""
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Union
+
+from sqlalchemy import (
+    Boolean,
+    CheckConstraint,
+    Column,
+    DateTime,
+    ForeignKey,
+    Integer,
+    PickleType,
+    String,
+    Text,
+    create_engine,
+    desc,
+    func,
+    select,
+)
+from sqlalchemy.orm import (
+    DeclarativeBase,
+    Mapped,
+    Session,
+    declared_attr,
+    relationship,
+)
+
+
+@dataclass
+class Base(DeclarativeBase):
+    """
+    Declarative base class for data objects.
+    """
+
+    id: Mapped[int] = Column(Integer, primary_key=True)  # pylint: disable=C0103
+    # @pytest, these are not the tests you're looking for...
+    __test__ = False  # pylint: disable=C0103
+
+    @declared_attr
+    def __tablename__(self):
+        return self.__name__.lower()
+
+
+@dataclass
+class Run(Base):
+    """
+    Class to store attributes of a flakefighters run.
+    """
+
+    created_at = Column(DateTime, default=func.now())
+    root: Mapped[str] = Column(String)
+    tests = relationship("Test", backref="run", lazy="subquery", cascade="all, delete", passive_deletes=True)
+    active_flakefighters = relationship(
+        "ActiveFlakeFighter", backref="run", lazy="subquery", cascade="all, delete", passive_deletes=True
+    )
+
+
+@dataclass
+class ActiveFlakeFighter(Base):
+    """
+    Store relevant information about the active flakefighters.
+    """
+
+    run_id: Mapped[int] = Column(Integer, ForeignKey("run.id"), nullable=False)
+    name: Mapped[str] = Column(String)
+    params: Mapped[dict] = Column(PickleType)
+
+
+@dataclass
+class Test(Base):
+    """
+    Class to store attributes of a test case.
+    """
+
+    run_id: Mapped[int] = Column(Integer, ForeignKey("run.id"), nullable=False)
+    name: Mapped[str] = Column(String)
+    skipped: Mapped[bool] = Column(Boolean, default=False)
+    executions = relationship(
+        "TestExecution", backref="test", lazy="subquery", cascade="all, delete", passive_deletes=True
+    )
+    flakefighter_results = relationship(
+        "FlakefighterResult", backref="test", lazy="subquery", cascade="all, delete", passive_deletes=True
+    )
+
+    @property
+    def flaky(self) -> bool:
+        """
+        Return whether a test (or any of its executions) has been marked as flaky by any flakefighter.
+        """
+        if not self.executions and not self.flakefighter_results:
+            return None
+        return any(result.flaky for result in self.flakefighter_results) or any(
+            execution.flaky for execution in self.executions
+        )
+
+
+@dataclass
+class TestExecution(Base):  # pylint: disable=R0902
+    """
+    Class to store attributes of a test outcome.
+    """
+
+    __tablename__ = "test_execution"
+
+    test_id: Mapped[int] = Column(Integer, ForeignKey("test.id"), nullable=False)
+    outcome: Mapped[str] = Column(String)
+    stdout: Mapped[str] = Column(Text)
+    stderr: Mapped[str] = Column(Text)
+    report: Mapped[str] = Column(Text)
+    start_time: Mapped[datetime] = Column(DateTime(timezone=True))
+    end_time: Mapped[datetime] = Column(DateTime(timezone=True))
+    coverage: Mapped[dict] = Column(PickleType)
+    flakefighter_results = relationship(
+        "FlakefighterResult", backref="test_execution", lazy="subquery", cascade="all, delete", passive_deletes=True
+    )
+    exception = relationship(
+        "TestException",
+        uselist=False,
+        backref="test_execution",
+        lazy="subquery",
+        cascade="all, delete",
+        passive_deletes=True,
+    )
+
+    @property
+    def flaky(self) -> bool:
+        """
+        Return whether a test (or any of its executions) has been marked as flaky by any flakefighter.
+        """
+        return any(result.flaky for result in self.flakefighter_results)
+
+
+@dataclass
+class TestException(Base):  # pylint: disable=R0902
+    """
+    Class to store information about the exceptions that cause tests to fail.
+    """
+
+    __tablename__ = "test_exception"
+
+    execution_id: Mapped[int] = Column(Integer, ForeignKey("test_execution.id"), nullable=False)
+    name: Mapped[str] = Column(String)
+    traceback = relationship(
+        "TracebackEntry", backref="exception", lazy="subquery", cascade="all, delete", passive_deletes=True
+    )
+
+
+@dataclass
+class TracebackEntry(Base):  # pylint: disable=R0902
+    """
+    Class to store attributes of entries in the stack trace.
+    """
+
+    exception_id: Mapped[int] = Column(Integer, ForeignKey("test_exception.id"), nullable=False)
+    path: Mapped[str] = Column(String)
+    lineno: Mapped[int] = Column(Integer)
+    colno: Mapped[int] = Column(Integer)
+    statement: Mapped[str] = Column(String)
+    source: Mapped[str] = Column(Text)
+
+
+@dataclass
+class FlakefighterResult(Base):  # pylint: disable=R0902
+    """
+    Class to store flakefighter results.
+    """
+
+    __tablename__ = "flakefighter_result"
+
+    test_execution_id: Mapped[int] = Column(Integer, ForeignKey("test_execution.id"), nullable=True)
+    test_id: Mapped[int] = Column(Integer, ForeignKey("test.id"), nullable=True)
+    name: Mapped[str] = Column(String)
+    flaky: Mapped[bool] = Column(Boolean)
+
+    __table_args__ = (
+        CheckConstraint("(test_execution_id IS NOT NULL) + (test_id IS NOT NULL) = 1", name="check_test_id_not_null"),
+    )
+
+
+class Database:
+    """
+    Class to handle database setup and interaction.
+    """
+
+    def __init__(
+        self,
+        url: str,
+        load_max_runs: int = None,
+        store_max_runs: int = None,
+        time_immemorial: Union[timedelta, str] = None,
+    ):
+        if isinstance(time_immemorial, str) and time_immemorial:
+            days, hours, minutes = [int(x) for x in time_immemorial.split(":")]
+            time_immemorial = timedelta(days=days, hours=hours, minutes=minutes)
+
+        self.engine = create_engine(url)
+        Base.metadata.create_all(self.engine)
+
+        self.store_max_runs = store_max_runs
+        self.time_immemorial = time_immemorial
+        self.previous_runs = self.load_runs(load_max_runs)
+
+    def save(self, run: Run):
+        """
+        Save the given run into the database.
+        """
+        with Session(self.engine) as session:
+            session.add(run)
+            if self.time_immemorial is not None:
+                expiry_date = datetime.now() - self.time_immemorial
+                for r in session.query(Run).filter(Run.created_at < (expiry_date - self.time_immemorial)):
+                    session.delete(r)
+
+            if self.store_max_runs is not None:
+                for r in self.load_runs()[self.store_max_runs - 1 :]:
+                    session.delete(r)
+            session.commit()
+            session.flush()
+
+    def load_runs(self, limit: int = None):
+        """
+        Load runs from the database.
+
+        :param limit: The maximum number of runs to return (these will be most recent runs).
+        """
+        with Session(self.engine) as session:
+            return session.scalars(select(Run).order_by(desc(Run.id)).limit(limit)).all()

pytest_flakefighters/flakefighters/abstract_flakefighter.py

@@ -0,0 +1,50 @@
+"""
+This module implements the FlakeFighter abstract class to be extended by concrete flakefighter classes.
+Each of these is a microservice which takes a failed test and classifies that failure as either genuine or flaky.
+This classification can be configured to either run "live" after each test, or as a postprocessing step on the entire
+test suite after it completes.
+If running live, detectors are run at the end of pytest_runtest_makereport.
+If running as a postprocessing step, detectors are run at the start of pytest_sessionfinish.
+"""
+
+from abc import ABC, abstractmethod
+
+from pytest_flakefighters.database_management import Run, TestExecution
+
+
+class FlakeFighter(ABC):  # pylint: disable=R0903
+    """
+    Abstract base class for a FlakeFighter
+    :ivar run_live: Run detection "live" after each test. Otherwise run as a postprocessing step after the test suite.
+    """
+
+    def __init__(self, run_live: bool):
+        self.run_live = run_live
+
+    @classmethod
+    @abstractmethod
+    def from_config(cls, config: dict):
+        """
+        Factory method to create a new instance from a pytest configuration.
+        """
+
+    @abstractmethod
+    def flaky_test_live(self, execution: TestExecution):
+        """
+        Detect whether a given test execution is flaky and append the result to its `flakefighter_results` attribute.
+        :param execution: The test execution to classify.
+        """
+
+    @abstractmethod
+    def flaky_tests_post(self, run: Run):
+        """
+        Go through each test in the test suite and append the result to its `flakefighter_results` attribute.
+        :param run: Run object representing the pytest run, with tests accessible through run.tests.
+        """
+
+    @abstractmethod
+    def params(self) -> dict:
+        """
+        Convert the key parameters into a dictionary so that the object can be replicated.
+        :return A dictionary of the parameters used to create the object.
+        """

pytest_flakefighters/flakefighters/coverage_independence.py

@@ -0,0 +1,99 @@
+"""
+This module implements the CoverageIndependence FlakeFighter.
+"""
+
+import pandas as pd
+from scipy.cluster.hierarchy import fcluster, linkage
+from scipy.spatial.distance import pdist
+
+from pytest_flakefighters.database_management import (
+    FlakefighterResult,
+    Run,
+    TestExecution,
+)
+from pytest_flakefighters.flakefighters.abstract_flakefighter import FlakeFighter
+
+
+class CoverageIndependence(FlakeFighter):
+    """
+    Classify tests as flaky if they fail independently of passing test cases that exercise overlapping code.
+
+    :ivar threshold: The minimum distance to consider as "similar", expressed as a proportion 0 <= threshold < 1 where 0
+    represents no difference and 1 represents complete difference.
+    :ivar metric: From `scipy.spatial.distance`: ['braycurtis', 'canberra', 'chebyshev', 'correlation', 'dice',
+    'hamming', 'jaccard', 'kulsinski', 'mahalanobis', 'minkowski', 'rogerstanimoto', 'russellrao', 'seuclidean',
+    'sokalmichener', 'sokalsneath', 'sqeuclidean', 'yule'].
+    :ivar linkage_method: From `scipy.cluster.hierarchy.linkage`: ['single', 'complete', 'average', 'weighted',
+    'centroid', 'median', 'ward']
+    """
+
+    def __init__(self, threshold: float = 0, metric: str = "jaccard", linkage_method="single"):
+        super().__init__(False)
+        self.threshold = threshold
+        self.metric = metric
+        self.linkage_method = linkage_method
+
+    @classmethod
+    def from_config(cls, config: dict):
+        """
+        Factory method to create a new instance from a pytest configuration.
+        """
+        return CoverageIndependence(
+            threshold=config.get("threshold", 0),
+            metric=config.get("metric", "jaccard"),
+            linkage_method=config.get("linkage_method", "single"),
+        )
+
+    def params(self):
+        """
+        Convert the key parameters into a dictionary so that the object can be replicated.
+        :return A dictionary of the parameters used to create the object.
+        """
+        return {"threshold": self.threshold, "metric": self.metric, "linkage_method": self.linkage_method}
+
+    def flaky_test_live(self, execution: TestExecution):
+        """
+        NOT SUPPORTED.
+        Detect whether a given test execution is flaky and append the result to its `flakefighter_results` attribute.
+        :param execution: The test execution to classify.
+        """
+        raise NotImplementedError("Coverage independence cannot be measured live")
+
+    def flaky_tests_post(self, run: Run):
+        """
+        Go through each test in the test suite and append the result to its `flakefighter_results` attribute.
+        :param run: Run object representing the pytest run, with tests accessible through run.tests.
+        """
+        coverage = []
+        # Enumerating tests and executions since they won't have IDs if they are not yet in the database
+        for test in run.tests:
+            for execution in test.executions:
+                coverage.append(
+                    {"test": test, "execution": execution}
+                    | {f"{file}:{line}": True for file in execution.coverage for line in execution.coverage[file]}
+                )
+
+        # Can't compute the pairwise distance of a single execution
+        if len(coverage) < 2:
+            return
+
+        coverage = pd.DataFrame(coverage)
+        coverage[coverage.columns.drop(["test", "execution"])] = (
+            coverage[coverage.columns.drop(["test", "execution"])].astype(pd.BooleanDtype()).fillna(False).astype(bool)
+        )
+        # Calculate the distance between each pair of test executions
+        raw_coverage = coverage.drop(["test", "execution"], axis=1).to_numpy()
+        distances = pdist(raw_coverage, metric=self.metric)
+        # Assign each test execution to a cluster
+        coverage["cluster"] = fcluster(
+            linkage(distances, method=self.linkage_method), t=self.threshold, criterion="distance"
+        )
+
+        for _, group in coverage.groupby("cluster"):
+            for test in group["test"]:
+                result = FlakefighterResult(
+                    name=self.__class__.__name__,
+                    flaky=len(set(map(lambda x: x.outcome, group["execution"]))) > 1,
+                )
+                if result not in test.flakefighter_results:
+                    test.flakefighter_results.append(result)

pytest_flakefighters/flakefighters/deflaker.py

@@ -0,0 +1,126 @@
+"""
+This module implements the DeFlaker FlakeFighter.
+"""
+
+import os
+
+import git
+from unidiff import PatchSet
+
+from pytest_flakefighters.database_management import (
+    FlakefighterResult,
+    Run,
+    TestExecution,
+)
+from pytest_flakefighters.flakefighters.abstract_flakefighter import FlakeFighter
+
+
+class DeFlaker(FlakeFighter):
+    """
+    A python equivalent of the DeFlaker algorithm from Bell et al. 2019 [10.1145/3180155.3180164].
+    Given the subtle differences between JUnit and pytest, this is not intended to be an exact port, but it follows
+    the same general methodology of checking whether covered code has been changed between commits.
+
+    :ivar root: The root directory of the Git repository.
+    :ivar source_commit: The source (older) commit hash. Defaults to HEAD^ (the previous commit to target).
+    :ivar target_commit: The target (newer) commit hash. Defaults to HEAD (the most recent commit).
+    """
+
+    def __init__(self, run_live: bool, root: str = ".", source_commit: str = None, target_commit: str = None):
+        super().__init__(run_live)
+
+        self.repo_root = git.Repo(root)
+        if target_commit is None and not self.repo_root.is_dirty():
+            # No uncommitted changes, so use most recent commit
+            self.target_commit = self.repo_root.commit().hexsha
+        else:
+            self.target_commit = target_commit
+        if source_commit is None:
+            if self.target_commit is None:
+                # If uncommitted changes, use most recent commit as source
+                self.source_commit = self.repo_root.commit().hexsha
+            else:
+                # If no uncommitted changes, use previous commit as source
+                parents = [
+                    commit.hexsha
+                    for commit in self.repo_root.commit(source_commit).iter_parents()
+                    if commit.hexsha != self.target_commit
+                ]
+                self.source_commit = parents[0]
+        else:
+            self.source_commit = source_commit
+
+        patches = PatchSet(self.repo_root.git.diff(self.source_commit, self.target_commit, "-U0", "--no-prefix"))
+        self.lines_changed = {}
+        for patch in patches:
+            if patch.target_file == patch.source_file:
+                abspath = os.path.join(self.repo_root.working_dir, patch.source_file)
+                self.lines_changed[abspath] = []
+                for hunk in patch:
+                    # Add each line in the hunk to lines_changed
+                    self.lines_changed[abspath] += list(
+                        range(hunk.target_start, hunk.target_start + hunk.target_length)
+                    )
+
+    @classmethod
+    def from_config(cls, config: dict):
+        """
+        Factory method to create a new instance from a pytest configuration.
+        """
+        return DeFlaker(
+            run_live=config.get("run_live", True),
+            root=config.get("root", "."),
+            source_commit=config.get("source_commit"),
+            target_commit=config.get("target_commit"),
+        )
+
+    def params(self):
+        """
+        Convert the key parameters into a dictionary so that the object can be replicated.
+        :return A dictionary of the parameters used to create the object.
+        """
+        return {"root": self.repo_root, "source_commit": self.source_commit, "target_commit": self.target_commit}
+
+    def line_modified_by_target_commit(self, file_path: str, line_number: int) -> bool:
+        """
+        Returns true if the given line in the file has been modified by the present commit.
+
+        :param file_path: The file to check.
+        :param line_number: The line number to check.
+        """
+        return line_number in self.lines_changed.get(file_path, [])
+
+    def _flaky_execution(self, execution):
+        """
+        Classify an execution as flaky or not.
+        :return: Boolean True of the test is classed as flaky and False otherwise.
+        """
+        return not any(
+            execution.outcome == "failed" and self.line_modified_by_target_commit(file_path, line_number)
+            for file_path in execution.coverage
+            for line_number in execution.coverage[file_path]
+            if file_path in self.lines_changed
+        )
+
+    def flaky_test_live(self, execution: TestExecution):
+        """
+        Classify a failing test as flaky if it does not cover any code which has been changed between the source and
+        target commits.
+        :param execution: The test execution to classify.
+        """
+        execution.flakefighter_results.append(
+            FlakefighterResult(name=self.__class__.__name__, flaky=self._flaky_execution(execution))
+        )
+
+    def flaky_tests_post(self, run: Run) -> list[bool | None]:
+        """
+        Classify failing tests as flaky if any of their executions are flaky.
+        :param run: Run object representing the pytest run, with tests accessible through run.tests.
+        """
+        for test in run.tests:
+            test.flakefighter_results.append(
+                FlakefighterResult(
+                    name=self.__class__.__name__,
+                    flaky=any(self._flaky_execution(execution) for execution in test.executions),
+                )
+            )

pytest_flakefighters/flakefighters/traceback_matching.py

@@ -0,0 +1,171 @@
+"""
+This module implements three FlakeFighters based on failure de-duplication from Alshammari et. al.
+[https://arxiv.org/pdf/2401.15788].
+"""
+
+import os
+import re
+
+import pandas as pd
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+from pytest_flakefighters.database_management import (
+    FlakefighterResult,
+    Run,
+    TestExecution,
+)
+from pytest_flakefighters.flakefighters.abstract_flakefighter import FlakeFighter
+
+
+class TracebackMatching(FlakeFighter):
+    """
+    Simple text-based matching classifier from Section II.A of [Alshammari et. al.].
+    We implement text-based matching on the failure logs for each test. Each failure log is represented by its failure
+    exception and stacktrace.
+    """
+
+    def __init__(self, run_live: bool, previous_runs: list[Run], root: str = "."):
+        super().__init__(run_live)
+        self.root = os.path.abspath(root)
+        self.previous_runs = previous_runs
+        print("TracebackMatching")
+
+    @classmethod
+    def from_config(cls, config: dict):
+        """
+        Factory method to create a new instance from a pytest configuration.
+        """
+        return TracebackMatching(
+            run_live=config.get("run_live", True),
+            previous_runs=config["database"].previous_runs,
+            root=config.get("root", "."),
+        )
+
+    def params(self):
+        """
+        Convert the key parameters into a dictionary so that the object can be replicated.
+        :return A dictionary of the parameters used to create the object.
+        """
+        return {"root": self.root}
+
+    def _flaky_execution(self, execution, previous_executions) -> bool:
+        """
+        Classify an execution as flaky if any of its failing executions has a traceback that matches a test previously
+        classed as flaky.
+        :return: Boolean True if the test is classed as flaky and False otherwise.
+        """
+        if not execution.exception:
+            return False
+        current_traceback = [
+            (os.path.relpath(e.path, self.root), e.lineno, e.colno, e.statement)
+            for e in execution.exception.traceback
+            if os.path.commonpath([self.root, e.path]) == self.root
+        ]
+        return any(e == current_traceback for e in previous_executions)
+
+    def previous_flaky_executions(self, runs: list[Run] = None) -> list:
+        """
+        Extract the relevant information from previous flaky executions and collapse into a single list.
+        :param runs: The runs to consider. Defaults to self.previous_runs.
+        :return: List containing the relative path, line number, column number, and code statement of all previous
+        test executions.
+        """
+        if runs is None:
+            runs = self.previous_runs
+        return [
+            [
+                (os.path.relpath(elem.path, run.root), elem.lineno, elem.colno, elem.statement)
+                for elem in execution.exception.traceback
+            ]
+            for run in runs
+            for test in run.tests
+            if test.flaky
+            for execution in test.executions
+            if execution.exception
+        ]
+
+    def flaky_test_live(self, execution: TestExecution):
+        """
+        Classify executions as flaky if they have the same failure logs as a flaky execution.
+        :param execution: Test execution to consider.
+        """
+        execution.flakefighter_results.append(
+            FlakefighterResult(
+                name=self.__class__.__name__,
+                flaky=self._flaky_execution(
+                    execution,
+                    self.previous_flaky_executions(),
+                ),
+            )
+        )
+
+    def flaky_tests_post(self, run: Run) -> list[bool | None]:
+        """
+        Classify failing executions as flaky if any if their executions are flaky.
+        :param run: Run object representing the pytest run, with tests accessible through run.tests.
+        """
+        for test in run.tests:
+            for execution in test.executions:
+                execution.flakefighter_results.append(
+                    FlakefighterResult(
+                        name=self.__class__.__name__,
+                        flaky=self._flaky_execution(
+                            execution, self.previous_flaky_executions(self.previous_runs + [run])
+                        ),
+                    )
+                )
+
+
+class CosineSimilarity(TracebackMatching):
+    """
+    TF-IDF cosine similarity matching classifier from Section II.C of [Alshammari et. al.].
+    Test executions are classified as flaky if the stack trace is sufficiently similar to a previous flaky execution.
+    """
+
+    def __init__(self, run_live: bool, previous_runs: list[Run], root: str = ".", threshold: float = 1):
+        super().__init__(run_live, previous_runs, root)
+        self.root = os.path.abspath(root)
+        self.previous_runs = previous_runs
+        self.threshold = threshold
+
+    @classmethod
+    def from_config(cls, config: dict):
+        """
+        Factory method to create a new instance from a pytest configuration.
+        """
+        return CosineSimilarity(
+            run_live=config.get("run_live", True),
+            previous_runs=config["database"].previous_runs,
+            root=config.get("root", "."),
+        )
+
+    def _tf_idf_matrix(self, executions):
+        corpus = [
+            re.sub(r"[^\w\s]", " ", "\n".join([" ".join(map(str, tuple)) for tuple in execution]))
+            for execution in executions
+        ]
+        vectorizer = TfidfVectorizer()
+        tfidf_matrix = vectorizer.fit_transform(corpus)
+
+        feature_names = vectorizer.get_feature_names_out()
+        return pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names)
+
+    def _flaky_execution(self, execution, previous_executions) -> bool:
+        """
+        Classify an execution as flaky if the test execution is sufficiently cosine-similar to any of the previous
+        executions.
+        :return: Boolean True if the test is classed as flaky and False otherwise.
+        """
+        if not execution.exception or not previous_executions:
+            return False
+
+        execution = [
+            (os.path.relpath(elem.path, self.root), elem.lineno, elem.colno, elem.statement)
+            for elem in execution.exception.traceback
+        ]
+
+        tf_idf_matrix = self._tf_idf_matrix([execution] + previous_executions)
+
+        similarity = cosine_similarity(tf_idf_matrix.iloc[0].values.reshape(1, -1), tf_idf_matrix.iloc[1:].values)
+        return (similarity >= self.threshold).any()