sflkit 0.2.18__tar.gz → 0.2.20__tar.gz

{sflkit-0.2.18/src/sflkit.egg-info → sflkit-0.2.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sflkit
-Version: 0.2.18
+Version: 0.2.20
 Summary: SFLKit: : A Workbench for Statistical Fault Localization
 Author-email: Marius Smytzek <marius.smytzek@cispa.de>
 Project-URL: Homepage, https://github.com/uds-se/sflkit/

{sflkit-0.2.18 → sflkit-0.2.20}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sflkit"
-version = "0.2.18"
+version = "0.2.20"
 authors = [
     { name = "Marius Smytzek", email = "marius.smytzek@cispa.de" },
 ]

{sflkit-0.2.18 → sflkit-0.2.20}/src/sflkit/language/python/factory.py

@@ -849,14 +849,17 @@ class LenEventFactory(DefEventFactory):
                 keywords=[],
             )
         )
-        return If(
-            test=Call(
-                func=Name(id="hasattr"),
-                args=[Name(id=event.var), Constant(value="__len__")],
-                keywords=[],
-            ),
+        return Try(
             body=[call],
+            handlers=[
+                ExceptHandler(
+                    type=Name(id="TypeError"),
+                    name=None,
+                    body=[Pass()],
+                )
+            ],
             orelse=[],
+            finalbody=[],
         )
 
     def get_event_call(self, event: LenEvent):

{sflkit-0.2.18 → sflkit-0.2.20}/src/sflkit/runners/run.py

@@ -9,6 +9,8 @@ import subprocess
 from pathlib import Path
 from typing import List, Dict, Optional, Tuple
 
+from sflkit.logger import LOGGER
+
 Environment = Dict[str, str]
 
 PYTEST_RESULT_PATTERN = re.compile(
@@ -51,6 +53,7 @@ class Runner(abc.ABC):
         files: Optional[List[os.PathLike] | os.PathLike] = None,
         base: Optional[os.PathLike] = None,
         environ: Environment = None,
+        k: str = None,
     ) -> List[str]:
         return []
 
@@ -99,6 +102,7 @@ class Runner(abc.ABC):
         files: Optional[List[os.PathLike] | os.PathLike] = None,
         base: Optional[os.PathLike] = None,
         environ: Environment = None,
+        k: str = None,
     ):
         self.passing_tests.clear()
         self.failing_tests.clear()
@@ -107,7 +111,7 @@ class Runner(abc.ABC):
             directory,
             output,
             self.filter_tests(
-                self.get_tests(directory, files=files, base=base, environ=environ)
+                self.get_tests(directory, files=files, base=base, environ=environ, k=k)
             ),
             environ=environ,
         )
@@ -117,7 +121,130 @@ class VoidRunner(Runner):
     pass
 
 
+class PytestStructure:
+    def __init__(self, name: str, parent: Optional["PytestStructure"] = None):
+        self.name = name
+        self.children = []
+        self.parent = parent
+        self.depth = 0
+
+    def add_child(self, child):
+        self.children.append(child)
+
+    def set_parent(self, parent):
+        self.parent = parent
+
+    def get_seperator(self):
+        return ""
+
+    def get_tests(self) -> List[str]:
+        if self.children:
+            return [
+                self.name + self.get_seperator() + test
+                for child in self.children
+                for test in child.get_tests()
+            ]
+        else:
+            return [self.name]
+
+    DIR = "Dir"
+    PACKAGE = "Package"
+    MODULE = "Module"
+    CLASS = "Class"
+    FUNCTION = "Function"
+    UNIT_TEST_CASE = "UnitTestCase"
+    TEST_CASE_FUNCTION = "TestCaseFunction"
+
+    @staticmethod
+    def get_pattern(obj):
+        return re.compile(rf"<{obj} ['\"]?(?P<name>[^>'\"]*)['\"]?>")
+
+    @staticmethod
+    def parse_tests(output: str) -> List[str]:
+        trees = []
+        current = None
+        for original_line in output.split("\n"):
+            line = original_line.lstrip()
+            level = len(original_line) - len(line)
+            if f"<{PytestStructure.DIR}" in line:
+                structure = Directory
+                pattern = PytestStructure.get_pattern(PytestStructure.DIR)
+            elif f"<{PytestStructure.PACKAGE}" in line:
+                structure = Directory
+                pattern = PytestStructure.get_pattern(PytestStructure.PACKAGE)
+            elif f"<{PytestStructure.MODULE}" in line:
+                structure = Module
+                pattern = PytestStructure.get_pattern(PytestStructure.MODULE)
+            elif f"<{PytestStructure.CLASS}" in line:
+                structure = Class
+                pattern = PytestStructure.get_pattern(PytestStructure.CLASS)
+            elif f"<{PytestStructure.FUNCTION}" in line:
+                structure = Function
+                pattern = PytestStructure.get_pattern(PytestStructure.FUNCTION)
+            elif f"<{PytestStructure.UNIT_TEST_CASE}" in line:
+                structure = Class
+                pattern = PytestStructure.get_pattern(PytestStructure.UNIT_TEST_CASE)
+            elif f"<{PytestStructure.TEST_CASE_FUNCTION}" in line:
+                structure = Function
+                pattern = PytestStructure.get_pattern(
+                    PytestStructure.TEST_CASE_FUNCTION
+                )
+            else:
+                continue
+            match = pattern.search(line)
+            if match:
+                obj = structure(match.group("name"))
+                obj.depth = level
+                if current is None:
+                    trees.append(obj)
+                elif current.depth < obj.depth:
+                    obj.set_parent(current)
+                    current.add_child(obj)
+                elif current.depth >= obj.depth:
+                    while current.depth >= obj.depth and current:
+                        current = current.parent
+                    if current is None:
+                        trees.append(obj)
+                    else:
+                        obj.set_parent(current)
+                        current.add_child(obj)
+                current = obj
+        return [test for tree in trees for test in tree.get_tests()]
+
+    def __repr__(self):
+        return f"<{self.__class__.__name__} {self.name}>"
+
+
+class Directory(PytestStructure):
+    def get_seperator(self):
+        return os.sep
+
+
+class Module(PytestStructure):
+    def get_seperator(self):
+        return "::"
+
+
+class Class(PytestStructure):
+    def get_seperator(self):
+        return "::"
+
+
+class Function(PytestStructure):
+    def get_seperator(self):
+        return "::"
+
+
 class PytestRunner(Runner):
+    def __init__(
+        self,
+        re_filter: str = r".*",
+        timeout=DEFAULT_TIMEOUT,
+        set_python_path: bool = False,
+    ):
+        super().__init__(re_filter, timeout)
+        self.set_python_path = set_python_path
+
     @staticmethod
     def _common_base(directory: Path, tests: List[str]) -> Path:
         parts = directory.parts
@@ -160,29 +287,39 @@ class PytestRunner(Runner):
         files: Optional[List[os.PathLike] | os.PathLike] = None,
         base: Optional[os.PathLike] = None,
         environ: Environment = None,
+        k: str = None,
     ) -> List[str]:
         c = []
         directory = directory.absolute()
+        if k:
+            c.append("-k")
+            c.append(k)
         if files:
             if isinstance(files, (str, os.PathLike)):
-                c.append(files)
+                c.append(str(files))
             else:
-                c += files
+                c += [str(f) for f in files]
         if base:
             if not files:
-                c.append(base)
+                c.append(str(base))
             root_dir = directory / base
         else:
             root_dir = directory
-        output = subprocess.run(
-            ["python", "-m", "pytest", "--collect-only", "-q"] + c,
+        process = subprocess.run(
+            [
+                "python3",
+                "-m",
+                "pytest",
+                "--collect-only",
+            ]
+            + c,
             stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
             env=environ,
             cwd=directory,
-        ).stdout.decode("utf-8")
-        tests = output.split("\n\n", 1)[0].split("\n")
-        if tests[-1] == "":
-            tests = tests[:-1]
+        )
+        LOGGER.info(f"pytest collection finished with {process.returncode}")
+        tests = PytestStructure.parse_tests(process.stdout.decode("utf8"))
         return self._normalize_paths(tests, directory, root_dir)
 
     @staticmethod
@@ -207,7 +344,7 @@ class PytestRunner(Runner):
     ) -> TestResult:
         try:
             output = subprocess.run(
-                ["python", "-m", "pytest", test],
+                ["python3", "-m", "pytest", test],
                 stdout=subprocess.PIPE,
                 env=environ,
                 cwd=directory,
@@ -274,6 +411,7 @@ class InputRunner(Runner):
         files: Optional[List[os.PathLike] | os.PathLike] = None,
         base: Optional[os.PathLike] = None,
         environ: Environment = None,
+        k: str = None,
     ) -> List[str]:
         return list(self.passing.keys()) + list(self.failing.keys())
 
@@ -288,7 +426,7 @@ class InputRunner(Runner):
             result = TestResult.FAILING
         try:
             process = subprocess.run(
-                ["python", self.access] + test,
+                ["python3", self.access] + test,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
                 env=environ,

{sflkit-0.2.18 → sflkit-0.2.20/src/sflkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sflkit
-Version: 0.2.18
+Version: 0.2.20
 Summary: SFLKit: : A Workbench for Statistical Fault Localization
 Author-email: Marius Smytzek <marius.smytzek@cispa.de>
 Project-URL: Homepage, https://github.com/uds-se/sflkit/

{sflkit-0.2.18 → sflkit-0.2.20}/src/sflkit.egg-info/SOURCES.txt

@@ -53,6 +53,7 @@ tests/test_instrumentation.py
 tests/test_language.py
 tests/test_predicates.py
 tests/test_runner.py
+tests/test_runner_pytest_versions.py
 tests/test_scope.py
 tests/test_spectra.py
 tests/test_suggestions.py

{sflkit-0.2.18 → sflkit-0.2.20}/tests/test_instrumentation.py

@@ -2,8 +2,7 @@ import atexit
 import os
 from pathlib import Path
 
-from parameterized import parameterized
-from sflkitlib.events import event, EventType
+from sflkitlib.events import EventType
 
 from sflkit import instrument_config, Config
 from utils import BaseTest
@@ -121,6 +120,61 @@ class TestInstrumentation(BaseTest):
         self.assertTrue(mapping.exists())
         self.assertTrue(mapping.is_file())
 
+    def test_instrument_exclude(self):
+        src = Path(BaseTest.TEST_RESOURCES, "test_exclude")
+        dst = Path(BaseTest.TEST_DIR)
+        instrument_config(
+            Config.create(
+                path=str(src),
+                language="python",
+                events="line",
+                predicates="line",
+                working=BaseTest.TEST_DIR,
+                include="included",
+                exclude=os.path.join("included", "excluded"),
+            )
+        )
+        included = dst / "included"
+        excluded = included / "excluded"
+        a = included / "a.py"
+        b = excluded / "b.py"
+        c = dst / "c.py"
+
+        included_src = src / "included"
+        excluded_src = included_src / "excluded"
+        a_src = included_src / "a.py"
+        b_src = excluded_src / "b.py"
+        c_src = src / "c.py"
+
+        self.assertTrue(included.exists())
+        self.assertTrue(included.is_dir())
+        self.assertTrue(excluded.exists())
+        self.assertTrue(excluded.is_dir())
+        self.assertTrue(a.exists())
+        self.assertTrue(a.is_file())
+        self.assertTrue(b.exists())
+        self.assertTrue(b.is_file())
+        self.assertTrue(c.exists())
+        self.assertTrue(c.is_file())
+
+        with open(a, "r") as fp:
+            a_content = fp.read()
+        with open(a_src, "r") as fp:
+            a_src_content = fp.read()
+        self.assertNotEqual(a_src_content, a_content)
+
+        with open(b, "r") as fp:
+            b_content = fp.read()
+        with open(b_src, "r") as fp:
+            b_src_content = fp.read()
+        self.assertEqual(b_src_content, b_content)
+
+        with open(c, "r") as fp:
+            c_content = fp.read()
+        with open(c_src, "r") as fp:
+            c_src_content = fp.read()
+        self.assertEqual(c_src_content, c_content)
+
 
 class TestLib(BaseTest):
     @classmethod
@@ -146,10 +200,6 @@ class TestLib(BaseTest):
     def setUp(self) -> None:
         self.events = None
 
-    def load(self) -> None:
-        self.lib.dump_events()
-        self.events = event.load(self.TEST_PATH)
-
     def tearDown(self) -> None:
         # noinspection PyBroadException
         try:

{sflkit-0.2.18 → sflkit-0.2.20}/tests/test_runner.py

@@ -37,7 +37,7 @@ class RunnerTests(BaseTest):
             exclude="tests",
         )
         instrument_config(config)
-        runner = PytestRunner()
+        runner = PytestRunner(set_python_path=True)
         output = Path(BaseTest.TEST_DIR, "events").absolute()
         runner.run(
             Path(BaseTest.TEST_DIR), output, files=[Path("tests", "test_middle.py")]

sflkit-0.2.20/tests/test_runner_pytest_versions.py

@@ -0,0 +1,121 @@
+import os
+import shutil
+import subprocess
+import sys
+import unittest
+from pathlib import Path
+
+from sflkit import Config, instrument_config, Analyzer
+from sflkit.analysis.analysis_type import AnalysisType
+from sflkit.analysis.suggestion import Location
+from sflkit.mapping import EventMapping
+from sflkit.model import EventFile
+from sflkit.runners.run import (
+    PytestRunner,
+)
+from tests.utils import BaseTest
+
+
+@unittest.skip
+class RunnerTests(BaseTest):
+    VENV = Path("test_venv")
+    PYTHON_BASE = os.path.join(
+        os.path.expanduser("~"), ".pyenv", "versions", "3.8.4", "bin", "python3"
+    )
+    PYTHON = "python3"
+
+    def assertPathExists(self, path: os.PathLike):
+        self.assertTrue(os.path.exists(path), f"{path} does not exists.")
+
+    def create_venv(self, environ):
+        shutil.rmtree(self.VENV, ignore_errors=True)
+        subprocess.check_call([self.PYTHON_BASE, "-m", "venv", self.VENV], env=environ)
+
+    def activate_venv(self, environ):
+        environ["VIRTUAL_ENV"] = str(self.VENV.absolute())
+        environ["_OLD_VIRTUAL_PATH"] = environ["PATH"]
+        if sys.platform.startswith("win"):
+            environ["PATH"] = f'{(self.VENV / "Scripts").absolute()};{environ["PATH"]}'
+        else:
+            environ["PATH"] = f'{(self.VENV / "bin").absolute()}:{environ["PATH"]}'
+        if "PYTHONHOME" in environ:
+            environ["_OLD_VIRTUAL_PYTHONHOME"] = environ["PYTHONHOME"]
+            del environ["PYTHONHOME"]
+        if "VIRTUAL_ENV_DISABLE_PROMPT" not in environ:
+            environ["_OLD_VIRTUAL_PS1"] = environ["PS1"] if "PS1" in environ else ""
+            environ[
+                "PS1"
+            ] = f'({self.VENV}) {environ["PS1"] if "PS1" in environ else ""}'
+            environ["VIRTUAL_ENV_PROMPT"] = f"({self.VENV})"
+        return environ
+
+    def install_pytest(self, environ, pytest_version):
+        subprocess.check_call(
+            [self.PYTHON, "-m", "pip", "install", f"pytest=={pytest_version}"],
+            env=environ,
+        )
+        subprocess.check_call(
+            [self.PYTHON, "-m", "pip", "install", f"sflkitlib"],
+            env=environ,
+        )
+
+    def tearDown(self):
+        shutil.rmtree(self.VENV, ignore_errors=True)
+
+    def _runner_test(self, pytest_version):
+        environ = os.environ.copy()
+        self.create_venv(environ)
+        environ = self.activate_venv(environ)
+        self.install_pytest(environ, pytest_version)
+        config = Config.create(
+            path=os.path.join(self.TEST_RESOURCES, BaseTest.TEST_RUNNER),
+            language="python",
+            events="line",
+            predicates="line",
+            working=BaseTest.TEST_DIR,
+            exclude="tests",
+        )
+        instrument_config(config)
+        runner = PytestRunner(set_python_path=True)
+        output = Path(BaseTest.TEST_DIR, "events").absolute()
+        runner.run(
+            Path(BaseTest.TEST_DIR),
+            output,
+            files=[Path("tests", "test_middle.py")],
+            environ=environ,
+        )
+        self.assertPathExists(output)
+        self.assertPathExists(output / "passing")
+        self.assertPathExists(output / "failing")
+        self.assertPathExists(output / "undefined")
+        mapping = EventMapping.load(config)
+        analyzer = Analyzer(
+            [
+                EventFile(
+                    output / "failing" / os.listdir(output / "failing")[0],
+                    0,
+                    mapping,
+                    failing=True,
+                )
+            ],
+            [
+                EventFile(output / "passing" / path, run_id, mapping)
+                for run_id, path in enumerate(os.listdir(output / "passing"), start=1)
+            ],
+            config.factory,
+        )
+        analyzer.analyze()
+        predicates = analyzer.get_analysis_by_type(AnalysisType.LINE)
+        suggestions = sorted(map(lambda p: p.get_suggestion(), predicates))
+        self.assertEqual(1, suggestions[-1].suspiciousness)
+        self.assertEqual(1, len(suggestions[-1].lines))
+        self.assertEqual(Location("middle.py", 7), suggestions[-1].lines[0])
+
+    def test_pytest_3_10_1(self):
+        self._runner_test("3.10.1")
+
+    def test_pytest_5_4_3(self):
+        self._runner_test("5.4.3")
+
+    def test_pytest_7_0_1(self):
+        self._runner_test("7.0.1")