sflkit 0.2.21__tar.gz → 0.2.22__tar.gz

{sflkit-0.2.21/src/sflkit.egg-info → sflkit-0.2.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sflkit
-Version: 0.2.21
+Version: 0.2.22
 Summary: SFLKit: : A Workbench for Statistical Fault Localization
 Author-email: Marius Smytzek <marius.smytzek@cispa.de>
 Project-URL: Homepage, https://github.com/uds-se/sflkit/
@@ -13,7 +13,7 @@ Classifier: Topic :: Software Development :: Testing
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: sflkitlib>=0.0.3
+Requires-Dist: sflkitlib>=0.0.5
 Requires-Dist: numpy>=1.23.5
 Requires-Dist: matplotlib>=3.7.2
 Requires-Dist: sortedcollections>=2.1.0

{sflkit-0.2.21 → sflkit-0.2.22}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sflkit"
-version = "0.2.21"
+version = "0.2.22"
 authors = [
     { name = "Marius Smytzek", email = "marius.smytzek@cispa.de" },
 ]
@@ -23,7 +23,7 @@ classifiers = [
     "Topic :: Software Development :: Testing"
 ]
 dependencies = [
-    "sflkitlib>=0.0.3",
+    "sflkitlib>=0.0.5",
     "numpy>=1.23.5",
     "matplotlib>=3.7.2",
     "sortedcollections>=2.1.0"

{sflkit-0.2.21 → sflkit-0.2.22}/src/sflkit/analysis/analyzer.py

@@ -19,6 +19,9 @@ class Analyzer(object):
         self.irrelevant_event_files = irrelevant_event_files
         self.model = Model(factory)
         self.paths: Dict[int, os.PathLike] = dict()
+        self.max_suspiciousness = 0
+        self.min_suspiciousness = 0
+        self.avg_suspiciousness = 0
 
     def _analyze(self, event_file):
         self.model.prepare(event_file)
@@ -48,20 +51,35 @@ class Analyzer(object):
 
     def get_sorted_suggestions(
         self, base_dir, metric: Callable = None, type_: AnalysisType = None
-    ):
+    ) -> List[Suggestion]:
         if type_:
             objects = self.get_analysis_by_type(type_)
         else:
             objects = self.get_analysis()
+        return self.get_sorted_suggestions_from_analysis(base_dir, objects, metric)
+
+    def get_sorted_suggestions_from_analysis(
+        self, base_dir, analysis: Set[AnalysisObject], metric: Callable = None
+    ) -> List[Suggestion]:
         suggestions = dict()
+        max_suspiciousness = float("-inf")
+        min_suspiciousness = float("inf")
+        avg_suspiciousness = 0
         for suggestion in map(
-            lambda p: p.get_suggestion(metric=metric, base_dir=base_dir), objects
+            lambda p: p.get_suggestion(metric=metric, base_dir=base_dir), analysis
         ):
+            max_suspiciousness = max(max_suspiciousness, suggestion.suspiciousness)
+            min_suspiciousness = min(min_suspiciousness, suggestion.suspiciousness)
+            avg_suspiciousness += suggestion.suspiciousness
             if suggestion.suspiciousness not in suggestions:
                 suggestions[suggestion.suspiciousness] = set(suggestion.lines)
             else:
                 suggestions[suggestion.suspiciousness] |= set(suggestion.lines)
 
+        self.max_suspiciousness = max_suspiciousness
+        self.min_suspiciousness = min_suspiciousness
+        self.avg_suspiciousness = avg_suspiciousness / len(analysis)
+
         return sorted(
             [
                 Suggestion(list(lines), suspiciousness)

sflkit-0.2.22/src/sflkit/evaluation.py

@@ -0,0 +1,116 @@
+import enum
+import random
+from typing import List, Dict, Callable, Set, Optional
+
+from sflkit.analysis.suggestion import Suggestion, Location
+
+
+class Average:
+    def __init__(self):
+        self.number_of_locations = 0
+
+    def average(self, suspiciousness: float, current_suspiciousness: float):
+        current_suspiciousness *= self.number_of_locations
+        self.number_of_locations += 1
+        return (current_suspiciousness + suspiciousness) / self.number_of_locations
+
+
+class Scenario(enum.Enum):
+    BEST_CASE = "best_case"
+    AVG_CASE = "avg_case"
+    WORST_CASE = "worst_case"
+
+
+class Rank:
+    def __init__(
+        self,
+        suggestions: List[Suggestion],
+        metric: Callable[[float, float], float] = max,
+        default_suspiciousness: float = float("-inf"),
+    ):
+        self.suggestions = sorted(suggestions, reverse=True)
+        self.suspiciousness: Dict[Location, float] = dict()
+        self.ranks: Dict[float, List[Location]] = dict()
+        self.locations: Dict[Location, float] = dict()
+        current_rank = 1
+        for i, suggestion in enumerate(self.suggestions):
+            lines = suggestion.lines
+            if len(lines) == 0:
+                continue
+            elif len(lines) == 1:
+                rank = current_rank
+                current_rank += 1
+            else:
+                rank = (len(lines)) / 2 + (current_rank - 1)
+                current_rank += len(lines)
+            self.ranks[rank] = lines
+            for line in lines:
+                self.suspiciousness[line] = metric(
+                    suggestion.suspiciousness,
+                    self.locations.get(line, default_suspiciousness),
+                )
+                self.locations[line] = rank
+        self.number_of_locations = len(self.locations)
+
+    def top_n(
+        self,
+        faulty: Set[Location],
+        n: int,
+        scenario: Optional[Scenario] = None,
+        repeat: int = 1000,
+    ) -> float:
+        top_n_locations = list()
+        for suggestion in self.suggestions:
+            if len(top_n_locations) >= n:
+                break
+            for line in suggestion.lines:
+                if line not in top_n_locations:
+                    top_n_locations.append(line)
+        if len(top_n_locations) <= n:
+            return self._top_n(faulty, top_n_locations, scenario)
+        else:
+            sum_ = 0
+            for _ in range(repeat):
+                sum_ += self._top_n(
+                    faulty, random.sample(top_n_locations, k=n), scenario
+                )
+            return sum_ / repeat
+
+    @staticmethod
+    def _top_n(
+        faulty: Set[Location],
+        top_n_locations: List[Location],
+        scenario: Optional[Scenario] = None,
+    ) -> float:
+        found = len(faulty.intersection(top_n_locations))
+        if scenario == Scenario.BEST_CASE:
+            return 1 if found > 0 else 0
+        elif scenario == Scenario.WORST_CASE:
+            return found / len(faulty)
+        elif scenario == Scenario.AVG_CASE:
+            return min(found / (len(faulty) / 2), 1)
+        else:
+            return found / len(top_n_locations)
+
+    def get_rank(
+        self, faulty: Set[Location], scenario: Optional[Scenario] = None
+    ) -> float:
+        if scenario == Scenario.BEST_CASE:
+            rank = min(self.locations[location] for location in faulty)
+        elif scenario == Scenario.WORST_CASE:
+            rank = max(self.locations[location] for location in faulty)
+        elif scenario == Scenario.AVG_CASE:
+            rank = sorted([self.locations[location] for location in faulty])[
+                max(len(faulty) // 2 - 1, 0)
+            ]
+        else:
+            rank = sum(self.locations[location] for location in faulty) / len(faulty)
+        return rank
+
+    def exam(self, faulty: Set[Location], scenario: Optional[Scenario] = None) -> float:
+        return self.get_rank(faulty, scenario) / self.number_of_locations
+
+    def wasted_effort(
+        self, faulty: Set[Location], scenario: Optional[Scenario] = None
+    ) -> float:
+        return self.get_rank(faulty, scenario)

{sflkit-0.2.21 → sflkit-0.2.22}/src/sflkit/language/python/factory.py

@@ -853,7 +853,19 @@ class LenEventFactory(DefEventFactory):
             body=[call],
             handlers=[
                 ExceptHandler(
-                    type=Name(id="TypeError"),
+                    type=Tuple(
+                        elts=[
+                            Name(
+                                id="AttributeError",
+                            ),
+                            Name(
+                                id="TypeError",
+                            ),
+                            Name(
+                                id="NameError",
+                            ),
+                        ],
+                    ),
                     name=None,
                     body=[Pass()],
                 )

{sflkit-0.2.21 → sflkit-0.2.22}/src/sflkit/runners/run.py

@@ -7,7 +7,7 @@ import shutil
 import string
 import subprocess
 from pathlib import Path
-from typing import List, Dict, Optional, Tuple
+from typing import List, Dict, Optional, Tuple, Set
 
 from sflkit.logger import LOGGER
 
@@ -17,7 +17,7 @@ PYTEST_RESULT_PATTERN = re.compile(
     rb"=( |\x1b\[\d+m)+(("
     rb"((?P<f>\d+) failed)|"
     rb"((?P<p>\d+) passed)|"
-    rb"(\d+ (skipped|warning(s?)|"
+    rb"(\d+ (skipped|warning(s?)|deselected|"
     rb"(error(s?)))))"
     rb"(( |\x1b\[\d+m)*,( |\x1b\[\d+m)+)?)+( |\x1b\[\d+m)+in( |\x1b\[\d+m)+"
 )
@@ -157,7 +157,17 @@ class PytestStructure:
 
     @staticmethod
     def get_pattern(obj):
-        return re.compile(rf"<{obj} ['\"]?(?P<name>[^>'\"]*)['\"]?>")
+        return re.compile(
+            rf"<{obj} "
+            r"('(?P<name_single>([^']|\\')*)'"
+            r"|\"(?P<name_double>([^\"]|\\\")*)\""
+            r"|(?P<name>[^\"'].*))"
+            r">"
+        )
+
+    @staticmethod
+    def clean_line(line: str) -> str:
+        return line.replace("\\\\", "\\")
 
     @staticmethod
     def parse_tests(output: str) -> List[str]:
@@ -193,7 +203,18 @@ class PytestStructure:
                 continue
             match = pattern.search(line)
             if match:
-                obj = structure(match.group("name"))
+                name = (
+                    match.group("name_single")
+                    or match.group("name_double")
+                    or match.group("name")
+                )
+                if match.group("name_single"):
+                    name = PytestStructure.clean_line(name)
+                    name = name.replace("\\'", "'")
+                elif match.group("name_double"):
+                    name = PytestStructure.clean_line(name)
+                    name = name.replace('\\"', '"')
+                obj = structure(name)
                 obj.depth = level
                 if current is None:
                     trees.append(obj)
@@ -246,10 +267,10 @@ class PytestRunner(Runner):
         self.set_python_path = set_python_path
 
     @staticmethod
-    def _common_base(directory: Path, tests: List[str]) -> Path:
+    def common_base(directory: Path, tests: List[str]) -> Path:
         parts = directory.parts
         common_bases = {Path(*parts[:i]) for i in range(1, len(parts) + 1)}
-        leaves_paths = {Path(r.split("::")[0]) for r in tests}
+        leaves_paths = {Path(r.split("::", 1)[0] if "::" in r else r) for r in tests}
         common_bases = set(
             filter(
                 lambda p: all(map(lambda r: Path(p, *r.parts).exists(), leaves_paths)),
@@ -261,24 +282,82 @@ class PytestRunner(Runner):
         else:
             return None
 
-    def _normalize_paths(
-        self,
+    @staticmethod
+    def common_path(files: List[str]) -> Optional[Path]:
+        paths = []
+        for f in files:
+            if "::" in f:
+                paths.append(f.split("::")[0])
+            else:
+                paths.append(f)
+        common_path = os.path.commonpath(paths)
+        if common_path:
+            return Path(common_path)
+        return None
+
+    @staticmethod
+    def get_files(files: List[os.PathLike]) -> Set[str]:
+        paths = set()
+        for f in files:
+            f = str(f)
+            if "::" in f:
+                paths.add(f.split("::", 1)[0])
+            else:
+                paths.add(f)
+        return paths
+
+    @staticmethod
+    def get_absolute_files(
+        files: Set[str], directory: Optional[Path] = None
+    ) -> Set[Path]:
+        result = set()
+        for f in files:
+            f = Path(f)
+            if f.is_absolute() or directory is None:
+                result.add(f)
+            else:
+                result.add(directory / f)
+        return result
+
+    @staticmethod
+    def normalize_paths(
         tests: List[str],
+        files: Set[Path] = None,
         directory: Optional[Path] = None,
         root_dir: Optional[Path] = None,
     ):
         result = tests
+
         if directory:
-            base = self._common_base(directory, tests)
-            if base is None and root_dir:
-                base = self._common_base(root_dir, tests)
-            if base is not None:
+            if files:
                 result = []
                 for r in tests:
-                    path, test = r.split("::", 1)
-                    result.append(
-                        str((base / path).relative_to(directory)) + "::" + test
-                    )
+                    for f in files:
+                        base = PytestRunner.common_base(f, [r])
+                        if base is not None:
+                            path, test = r.split("::", 1)
+                            result.append(
+                                str((base / path).relative_to(directory)) + "::" + test
+                            )
+                            break
+            else:
+                base = PytestRunner.common_base(directory, tests)
+                if base is None and root_dir:
+                    base = PytestRunner.common_base(root_dir, tests)
+                if base is None and root_dir is not None:
+                    result = []
+                    for r in tests:
+                        path, test = r.split("::", 1)
+                        result.append(
+                            str((root_dir / path).relative_to(directory)) + "::" + test
+                        )
+                elif base is not None:
+                    result = []
+                    for r in tests:
+                        path, test = r.split("::", 1)
+                        result.append(
+                            str((base / path).relative_to(directory)) + "::" + test
+                        )
         return result
 
     def get_tests(
@@ -294,17 +373,20 @@ class PytestRunner(Runner):
         if k:
             c.append("-k")
             c.append(k)
-        if files:
-            if isinstance(files, (str, os.PathLike)):
-                c.append(str(files))
-            else:
-                c += [str(f) for f in files]
         if base:
             if not files:
                 c.append(str(base))
             root_dir = directory / base
         else:
             root_dir = directory
+        file_bases = None
+        if files:
+            if isinstance(files, (str, os.PathLike)):
+                str_files = [str(files)]
+            else:
+                str_files = [str(f) for f in files]
+            file_bases = self.get_absolute_files(self.get_files(str_files), directory)
+            c += str_files
         process = subprocess.run(
             [
                 "python3",
@@ -320,7 +402,7 @@ class PytestRunner(Runner):
         )
         LOGGER.info(f"pytest collection finished with {process.returncode}")
         tests = PytestStructure.parse_tests(process.stdout.decode("utf8"))
-        return self._normalize_paths(tests, directory, root_dir)
+        return self.normalize_paths(tests, file_bases, directory, root_dir)
 
     @staticmethod
     def __get_pytest_result__(

{sflkit-0.2.21 → sflkit-0.2.22/src/sflkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sflkit
-Version: 0.2.21
+Version: 0.2.22
 Summary: SFLKit: : A Workbench for Statistical Fault Localization
 Author-email: Marius Smytzek <marius.smytzek@cispa.de>
 Project-URL: Homepage, https://github.com/uds-se/sflkit/
@@ -13,7 +13,7 @@ Classifier: Topic :: Software Development :: Testing
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: sflkitlib>=0.0.3
+Requires-Dist: sflkitlib>=0.0.5
 Requires-Dist: numpy>=1.23.5
 Requires-Dist: matplotlib>=3.7.2
 Requires-Dist: sortedcollections>=2.1.0

{sflkit-0.2.21 → sflkit-0.2.22}/src/sflkit.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ src/sflkit/__init__.py
 src/sflkit/cli.py
 src/sflkit/color.py
 src/sflkit/config.py
+src/sflkit/evaluation.py
 src/sflkit/logger.py
 src/sflkit/mapping.py
 src/sflkit.egg-info/PKG-INFO
@@ -47,6 +48,7 @@ tests/test_cli.py
 tests/test_color.py
 tests/test_config.py
 tests/test_coverage.py
+tests/test_evaluation.py
 tests/test_events.py
 tests/test_execution.py
 tests/test_instrumentation.py

{sflkit-0.2.21 → sflkit-0.2.22}/src/sflkit.egg-info/requires.txt

@@ -1,4 +1,4 @@
-sflkitlib>=0.0.3
+sflkitlib>=0.0.5
 numpy>=1.23.5
 matplotlib>=3.7.2
 sortedcollections>=2.1.0

sflkit-0.2.22/tests/test_evaluation.py

@@ -0,0 +1,166 @@
+import random
+from typing import Optional
+
+from sflkit.analysis.suggestion import Suggestion, Location
+from sflkit.evaluation import Rank, Scenario
+from utils import BaseTest
+
+
+class TestEvaluation(BaseTest):
+    def setUp(self):
+        random.seed(0)
+        self.suggestions = [
+            Suggestion([Location("a.py", 1)], 0.5),
+            Suggestion([Location("a.py", 2)], 0.3),
+            Suggestion([Location("a.py", 3)], 0.7),
+            Suggestion([Location("a.py", 4)], 0.1),
+            Suggestion([Location("a.py", 5)], 0.9),
+            Suggestion([Location("a.py", 6)], 0.2),
+            Suggestion([Location("a.py", 7)], 0.8),
+            Suggestion([Location("a.py", 8)], 0.4),
+            Suggestion([Location("a.py", 9)], 0.6),
+            Suggestion([Location("a.py", 10)], 0.0),
+        ]
+
+    def get_rank(self, multi: bool = False):
+        suggestions = (
+            self.suggestions
+            if not multi
+            else self.suggestions
+            + [
+                Suggestion(
+                    [Location("a.py", 11), Location("a.py", 12), Location("a.py", 13)],
+                    0.55,
+                )
+            ]
+        )
+        return (
+            Rank(suggestions),
+            {
+                Location("a.py", 2),
+                Location("a.py", 6),
+                Location("a.py", 7),
+                Location("a.py", 8),
+            },
+        )
+
+    def get_top_n(
+        self, n: int, scenario: Optional[Scenario] = None, multi: bool = False
+    ):
+        rank, locations = self.get_rank(multi=multi)
+        return rank.top_n(
+            locations,
+            n,
+            scenario=scenario,
+            repeat=10000,
+        )
+
+    def get_exam(self, scenario: Optional[Scenario] = None):
+        rank, locations = self.get_rank(multi=True)
+        return rank.exam(
+            locations,
+            scenario=scenario,
+        )
+
+    def get_wasted_effort(self, scenario: Optional[Scenario] = None):
+        rank, locations = self.get_rank(multi=True)
+        return rank.wasted_effort(
+            locations,
+            scenario=scenario,
+        )
+
+    def test_top_1(self):
+        top_1 = self.get_top_n(1)
+        self.assertAlmostEqual(0, top_1, delta=self.DELTA)
+
+    def test_top_5(self):
+        top_5 = self.get_top_n(5)
+        self.assertAlmostEqual(0.2, top_5, delta=self.DELTA)
+
+    def test_top_10(self):
+        top_10 = self.get_top_n(10)
+        self.assertAlmostEqual(0.4, top_10, delta=self.DELTA)
+
+    def test_top_5_multi(self):
+        top_5_multi = self.get_top_n(5, multi=True)
+        self.assertAlmostEqual(0.14285, top_5_multi, delta=0.05)
+
+    def test_top_1_avg(self):
+        top_1_avg = self.get_top_n(1, scenario=Scenario.AVG_CASE)
+        self.assertAlmostEqual(0, top_1_avg, delta=self.DELTA)
+
+    def test_top_5_avg(self):
+        top_5_avg = self.get_top_n(5, scenario=Scenario.AVG_CASE)
+        self.assertAlmostEqual(0.5, top_5_avg, delta=self.DELTA)
+
+    def test_top_10_avg(self):
+        top_10_avg = self.get_top_n(10, scenario=Scenario.AVG_CASE)
+        self.assertAlmostEqual(1, top_10_avg, delta=self.DELTA)
+
+    def test_top_5_avg_multi(self):
+        top_5_avg_multi = self.get_top_n(5, scenario=Scenario.AVG_CASE, multi=True)
+        self.assertAlmostEqual(0.35714, top_5_avg_multi, delta=0.005)
+
+    def test_top_1_best(self):
+        top_1_best = self.get_top_n(1, scenario=Scenario.BEST_CASE)
+        self.assertAlmostEqual(0, top_1_best, delta=self.DELTA)
+
+    def test_top_5_best(self):
+        top_5_best = self.get_top_n(5, scenario=Scenario.BEST_CASE)
+        self.assertAlmostEqual(1, top_5_best, delta=self.DELTA)
+
+    def test_top_10_best(self):
+        top_10_best = self.get_top_n(10, scenario=Scenario.BEST_CASE)
+        self.assertAlmostEqual(1, top_10_best, delta=self.DELTA)
+
+    def test_top_5_best_multi(self):
+        top_5_best_multi = self.get_top_n(5, scenario=Scenario.BEST_CASE, multi=True)
+        self.assertAlmostEqual(0.71429, top_5_best_multi, delta=0.005)
+
+    def test_top_1_worst(self):
+        top_1_worst = self.get_top_n(1, scenario=Scenario.WORST_CASE)
+        self.assertAlmostEqual(0, top_1_worst, delta=self.DELTA)
+
+    def test_top_5_worst(self):
+        top_5_worst = self.get_top_n(5, scenario=Scenario.WORST_CASE)
+        self.assertAlmostEqual(0.25, top_5_worst, delta=self.DELTA)
+
+    def test_top_10_worst(self):
+        top_10_worst = self.get_top_n(10, scenario=Scenario.WORST_CASE)
+        self.assertAlmostEqual(1, top_10_worst, delta=self.DELTA)
+
+    def test_top_5_worst_multi(self):
+        top_5_worst_multi = self.get_top_n(5, scenario=Scenario.WORST_CASE, multi=True)
+        self.assertAlmostEqual(0.17857, top_5_worst_multi, delta=0.005)
+
+    def test_exam_avg(self):
+        exam_avg = self.get_exam(scenario=Scenario.AVG_CASE)
+        self.assertAlmostEqual(9 / 13, exam_avg, delta=self.DELTA)
+
+    def test_exam_best(self):
+        exam_best = self.get_exam(scenario=Scenario.BEST_CASE)
+        self.assertAlmostEqual(2 / 13, exam_best, delta=self.DELTA)
+
+    def test_exam_worst(self):
+        exam_worst = self.get_exam(scenario=Scenario.WORST_CASE)
+        self.assertAlmostEqual(11 / 13, exam_worst, delta=self.DELTA)
+
+    def test_exam(self):
+        exam = self.get_exam()
+        self.assertAlmostEqual((2 + 9 + 10 + 11) / (13 * 4), exam, delta=self.DELTA)
+
+    def test_wasted_effort_avg(self):
+        wasted_effort_avg = self.get_wasted_effort(scenario=Scenario.AVG_CASE)
+        self.assertAlmostEqual(9, wasted_effort_avg, delta=self.DELTA)
+
+    def test_wasted_effort_best(self):
+        wasted_effort_best = self.get_wasted_effort(scenario=Scenario.BEST_CASE)
+        self.assertAlmostEqual(2, wasted_effort_best, delta=self.DELTA)
+
+    def test_wasted_effort_worst(self):
+        wasted_effort_worst = self.get_wasted_effort(scenario=Scenario.WORST_CASE)
+        self.assertAlmostEqual(11, wasted_effort_worst, delta=self.DELTA)
+
+    def test_wasted_effort(self):
+        wasted_effort = self.get_wasted_effort()
+        self.assertAlmostEqual((2 + 9 + 10 + 11) / 4, wasted_effort, delta=self.DELTA)

{sflkit-0.2.21 → sflkit-0.2.22}/tests/test_runner.py

@@ -9,6 +9,7 @@ from sflkit.model import EventFile
 from sflkit.runners.run import (
     PytestRunner,
     InputRunner,
+    PytestStructure,
 )
 from tests.utils import BaseTest
 
@@ -112,3 +113,67 @@ class RunnerTests(BaseTest):
         self.assertEqual(1, suggestions[-1].suspiciousness)
         self.assertEqual(1, len(suggestions[-1].lines))
         self.assertEqual(Location("main.py", 10), suggestions[-1].lines[0])
+
+    def test_parse_and_paths(self):
+        collect = (
+            "\n"
+            "<Package b>\n"
+            "  <Module test_b.py>\n"
+            "    <Function test_d>\n"
+            "<Package tests>\n"
+            "  <Module test_a.py>\n"
+            "    <Function test_a>\n"
+            "    <Function test_b>\n"
+            "    <Function test_c>\n"
+        )
+        tests = PytestStructure.parse_tests(collect)
+        self.assertEqual(4, len(tests))
+        self.assertIn(os.path.join("b", "test_b.py::test_d"), tests)
+        self.assertIn(os.path.join("tests", "test_a.py::test_a"), tests)
+        self.assertIn(os.path.join("tests", "test_a.py::test_b"), tests)
+        self.assertIn(os.path.join("tests", "test_a.py::test_c"), tests)
+        files = PytestRunner.get_files(
+            [
+                Path("structure", "tests", "b", "test_b.py"),
+                Path("structure", "tests", "test_a.py::test_a"),
+                Path("structure", "tests", "test_a.py::test_b"),
+                Path("structure", "tests", "test_a.py::test_c"),
+            ]
+        )
+        self.assertEqual(2, len(files))
+        self.assertIn(os.path.join("structure", "tests", "b", "test_b.py"), files)
+        self.assertIn(os.path.join("structure", "tests", "test_a.py"), files)
+        directory = Path(BaseTest.TEST_RESOURCES, "structure")
+        files = PytestRunner.get_absolute_files(
+            files,
+            directory,
+        )
+        self.assertEqual(2, len(files))
+        self.assertIn(
+            directory / "structure" / "tests" / "b" / "test_b.py",
+            files,
+        )
+        self.assertIn(directory / "structure" / "tests" / "test_a.py", files)
+        tests = PytestRunner.normalize_paths(
+            tests,
+            files=files,
+            directory=directory,
+            root_dir=directory / "structure" / "tests",
+        )
+        self.assertEqual(4, len(tests))
+        self.assertIn(
+            os.path.join("structure", "tests", "b", "test_b.py::test_d"),
+            tests,
+        )
+        self.assertIn(
+            os.path.join("structure", "tests", "test_a.py::test_a"),
+            tests,
+        )
+        self.assertIn(
+            os.path.join("structure", "tests", "test_a.py::test_b"),
+            tests,
+        )
+        self.assertIn(
+            os.path.join("structure", "tests", "test_a.py::test_c"),
+            tests,
+        )