differential-coverage 1.2.1__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/PKG-INFO +8 -2
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/README.md +7 -1
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/pyproject.toml +6 -1
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/__init__.py +7 -8
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/api.py +13 -5
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/approach_data.py +3 -1
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/cli.py +30 -3
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/output.py +36 -10
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/readers/libfuzzer_merge.py +1 -3
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage.egg-info/PKG-INFO +8 -2
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/tests/test_api.py +11 -0
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/tests/test_approach_data.py +7 -5
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/tests/test_cli.py +44 -0
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/tests/test_docs.py +5 -1
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/tests/test_llvm_cov.py +1 -1
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/tests/test_output.py +32 -2
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/tests/test_reducers.py +2 -1
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/setup.cfg +0 -0
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/fs.py +0 -0
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/readers/__init__.py +0 -0
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/readers/afl_showmap.py +0 -0
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/readers/llvm_cov.py +0 -0
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/readers/registry.py +0 -0
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/types.py +1 -1
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage.egg-info/SOURCES.txt +0 -0
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage.egg-info/dependency_links.txt +0 -0
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage.egg-info/entry_points.txt +0 -0
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage.egg-info/requires.txt +0 -0
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage.egg-info/top_level.txt +0 -0
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/tests/test_libfuzzer_merge.py +0 -0
- {differential_coverage-1.2.1 → differential_coverage-1.3.0}/tests/test_readers.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: differential_coverage
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Compute differential coverage, a better way of comparing testing tools.
|
|
5
5
|
Author-email: Valentin Huber <contact@valentinhuber.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -38,6 +38,7 @@ In principle, differential coverage measures how much of approach $a_2$'s covera
|
|
|
38
38
|
|
|
39
39
|
- `relcov` does exactly this. It is an *asymetrical* measure between coverage of two approaches. If the approach relies on randomness (such as most fuzzers), it can aggregate over multiple trials of these.
|
|
40
40
|
- `relscore` aggregates all this data to an order of all approaches. This measure, compared to simple coverage numbers, takes into account and values higher approaches that can cover code that no other approach covers. However, compared to `relcov`, this hides a lot of information in this aggregation.
|
|
41
|
+
- `count` just reports absolute coverage counts (union of covered edges/blocks/branches across trials) per approach — useful as a baseline, but without the differential comparison.
|
|
41
42
|
|
|
42
43
|
For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEFINITIONS.md).
|
|
43
44
|
|
|
@@ -146,13 +147,14 @@ pip install differential-coverage[latex]
|
|
|
146
147
|
Generally, the command line interface follows the following structure:
|
|
147
148
|
|
|
148
149
|
```bash
|
|
149
|
-
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
150
|
+
differential-coverage {relcov,relscore,count} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
150
151
|
```
|
|
151
152
|
|
|
152
153
|
Examples:
|
|
153
154
|
|
|
154
155
|
```bash
|
|
155
156
|
differential-coverage relscore ./coverage_data
|
|
157
|
+
differential-coverage count ./coverage_data
|
|
156
158
|
differential-coverage --input-format llvm-cov relcov ./coverage_data
|
|
157
159
|
differential-coverage --output csv relscore ./coverage_data
|
|
158
160
|
```
|
|
@@ -206,6 +208,10 @@ relscores: dict[str, float] = dc.relscores()
|
|
|
206
208
|
for approach_name, relscore in relscores.items():
|
|
207
209
|
print(f"relscore {approach_name}: {relscore}")
|
|
208
210
|
|
|
211
|
+
counts: dict[str, int] = dc.coverage_counts()
|
|
212
|
+
for approach_name, count in counts.items():
|
|
213
|
+
print(f"coverage {approach_name}: {count}")
|
|
214
|
+
|
|
209
215
|
```
|
|
210
216
|
|
|
211
217
|
### Hints
|
|
@@ -14,6 +14,7 @@ In principle, differential coverage measures how much of approach $a_2$'s covera
|
|
|
14
14
|
|
|
15
15
|
- `relcov` does exactly this. It is an *asymetrical* measure between coverage of two approaches. If the approach relies on randomness (such as most fuzzers), it can aggregate over multiple trials of these.
|
|
16
16
|
- `relscore` aggregates all this data to an order of all approaches. This measure, compared to simple coverage numbers, takes into account and values higher approaches that can cover code that no other approach covers. However, compared to `relcov`, this hides a lot of information in this aggregation.
|
|
17
|
+
- `count` just reports absolute coverage counts (union of covered edges/blocks/branches across trials) per approach — useful as a baseline, but without the differential comparison.
|
|
17
18
|
|
|
18
19
|
For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEFINITIONS.md).
|
|
19
20
|
|
|
@@ -122,13 +123,14 @@ pip install differential-coverage[latex]
|
|
|
122
123
|
Generally, the command line interface follows the following structure:
|
|
123
124
|
|
|
124
125
|
```bash
|
|
125
|
-
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
126
|
+
differential-coverage {relcov,relscore,count} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
126
127
|
```
|
|
127
128
|
|
|
128
129
|
Examples:
|
|
129
130
|
|
|
130
131
|
```bash
|
|
131
132
|
differential-coverage relscore ./coverage_data
|
|
133
|
+
differential-coverage count ./coverage_data
|
|
132
134
|
differential-coverage --input-format llvm-cov relcov ./coverage_data
|
|
133
135
|
differential-coverage --output csv relscore ./coverage_data
|
|
134
136
|
```
|
|
@@ -182,6 +184,10 @@ relscores: dict[str, float] = dc.relscores()
|
|
|
182
184
|
for approach_name, relscore in relscores.items():
|
|
183
185
|
print(f"relscore {approach_name}: {relscore}")
|
|
184
186
|
|
|
187
|
+
counts: dict[str, int] = dc.coverage_counts()
|
|
188
|
+
for approach_name, count in counts.items():
|
|
189
|
+
print(f"coverage {approach_name}: {count}")
|
|
190
|
+
|
|
185
191
|
```
|
|
186
192
|
|
|
187
193
|
### Hints
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "differential_coverage"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.3.0"
|
|
8
8
|
description = "Compute differential coverage, a better way of comparing testing tools."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -38,5 +38,10 @@ where = ["src"]
|
|
|
38
38
|
[tool.pytest.ini_options]
|
|
39
39
|
pythonpath = ["src"]
|
|
40
40
|
|
|
41
|
+
[tool.ruff.lint]
|
|
42
|
+
ignore = [
|
|
43
|
+
"UP045", # Optional[X] -> X | None
|
|
44
|
+
]
|
|
45
|
+
|
|
41
46
|
[project.scripts]
|
|
42
47
|
differential-coverage = "differential_coverage:main"
|
{differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/__init__.py
RENAMED
|
@@ -1,24 +1,23 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
from differential_coverage.cli import main
|
|
3
1
|
from differential_coverage.api import DifferentialCoverage
|
|
4
2
|
from differential_coverage.approach_data import ApproachData
|
|
3
|
+
from differential_coverage.cli import main
|
|
5
4
|
from differential_coverage.types import (
|
|
6
|
-
EdgeId,
|
|
7
5
|
ApproachId,
|
|
6
|
+
CollectionReducer,
|
|
7
|
+
EdgeId,
|
|
8
8
|
TrialId,
|
|
9
9
|
ValueReducer,
|
|
10
|
-
CollectionReducer,
|
|
11
10
|
)
|
|
12
11
|
|
|
13
12
|
__all__ = [
|
|
14
|
-
"main",
|
|
15
|
-
"DifferentialCoverage",
|
|
16
13
|
"ApproachData",
|
|
17
|
-
"EdgeId",
|
|
18
14
|
"ApproachId",
|
|
15
|
+
"CollectionReducer",
|
|
16
|
+
"DifferentialCoverage",
|
|
17
|
+
"EdgeId",
|
|
19
18
|
"TrialId",
|
|
20
19
|
"ValueReducer",
|
|
21
|
-
"
|
|
20
|
+
"main",
|
|
22
21
|
]
|
|
23
22
|
|
|
24
23
|
if __name__ == "__main__":
|
{differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/api.py
RENAMED
|
@@ -1,14 +1,15 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
from collections.abc import Collection, Mapping
|
|
1
3
|
from functools import reduce
|
|
2
4
|
from pathlib import Path
|
|
3
|
-
from typing import
|
|
4
|
-
import warnings
|
|
5
|
+
from typing import Generic
|
|
5
6
|
|
|
6
|
-
from differential_coverage.readers import GranularityArg, InputFormat
|
|
7
|
-
from differential_coverage.fs import read_campaign_dir
|
|
8
7
|
from differential_coverage.approach_data import ApproachData
|
|
8
|
+
from differential_coverage.fs import read_campaign_dir
|
|
9
|
+
from differential_coverage.readers import GranularityArg, InputFormat
|
|
9
10
|
from differential_coverage.types import (
|
|
10
|
-
EdgeId,
|
|
11
11
|
ApproachId,
|
|
12
|
+
EdgeId,
|
|
12
13
|
TrialId,
|
|
13
14
|
)
|
|
14
15
|
|
|
@@ -76,6 +77,13 @@ class DifferentialCoverage(Generic[ApproachId, TrialId, EdgeId]):
|
|
|
76
77
|
def approaches(self) -> Mapping[ApproachId, ApproachData[TrialId, EdgeId]]:
|
|
77
78
|
return self._approaches
|
|
78
79
|
|
|
80
|
+
def coverage_counts(self) -> dict[ApproachId, int]:
|
|
81
|
+
"""Absolute coverage count per approach: size of the union of all trial edges."""
|
|
82
|
+
return {
|
|
83
|
+
approach_name: len(approach_data.edges_union)
|
|
84
|
+
for approach_name, approach_data in self._approaches.items()
|
|
85
|
+
}
|
|
86
|
+
|
|
79
87
|
def relscores(self) -> dict[ApproachId, float]:
|
|
80
88
|
all_edges = reduce(
|
|
81
89
|
lambda x, y: x.union(y),
|
{differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/cli.py
RENAMED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import argparse
|
|
2
2
|
import re
|
|
3
|
-
from collections.abc import Callable
|
|
3
|
+
from collections.abc import Callable, Sequence
|
|
4
4
|
from importlib import metadata
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import Optional
|
|
6
|
+
from typing import Optional
|
|
7
7
|
|
|
8
8
|
from differential_coverage.api import DifferentialCoverage
|
|
9
9
|
from differential_coverage.fs import read_campaign_dir
|
|
10
|
-
from differential_coverage.output import
|
|
10
|
+
from differential_coverage.output import (
|
|
11
|
+
print_counts,
|
|
12
|
+
print_relcov_corpus_table,
|
|
13
|
+
print_scores,
|
|
14
|
+
)
|
|
11
15
|
|
|
12
16
|
# All subcommands expect this directory layout (one campaign dir, no moving files).
|
|
13
17
|
CAMPAIGN_DIR_HELP = (
|
|
@@ -94,6 +98,19 @@ def cmd_relscore(args: argparse.Namespace) -> int:
|
|
|
94
98
|
return 0
|
|
95
99
|
|
|
96
100
|
|
|
101
|
+
def cmd_count(args: argparse.Namespace) -> int:
|
|
102
|
+
campaign = _load_campaign(args)
|
|
103
|
+
dc = DifferentialCoverage(campaign)
|
|
104
|
+
counts = dc.coverage_counts()
|
|
105
|
+
print_counts(
|
|
106
|
+
counts,
|
|
107
|
+
output=args.output,
|
|
108
|
+
colormap=args.colormap,
|
|
109
|
+
latex_enable_color=args.latex_enable_color,
|
|
110
|
+
)
|
|
111
|
+
return 0
|
|
112
|
+
|
|
113
|
+
|
|
97
114
|
def cmd_relcov_performance_over_approach(args: argparse.Namespace) -> int:
|
|
98
115
|
campaign = _load_campaign(args)
|
|
99
116
|
|
|
@@ -252,6 +269,16 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
252
269
|
p_relscore.add_argument("dir", type=Path, help=CAMPAIGN_DIR_HELP)
|
|
253
270
|
p_relscore.set_defaults(func=cmd_relscore)
|
|
254
271
|
|
|
272
|
+
p_count = subparsers.add_parser(
|
|
273
|
+
"count",
|
|
274
|
+
help=(
|
|
275
|
+
"Print absolute coverage counts (union of covered edges/blocks/branches "
|
|
276
|
+
"across trials) for each approach. Does not compute differential coverage."
|
|
277
|
+
),
|
|
278
|
+
)
|
|
279
|
+
p_count.add_argument("dir", type=Path, help=CAMPAIGN_DIR_HELP)
|
|
280
|
+
p_count.set_defaults(func=cmd_count)
|
|
281
|
+
|
|
255
282
|
return parser
|
|
256
283
|
|
|
257
284
|
|
{differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/output.py
RENAMED
|
@@ -6,7 +6,6 @@ import csv
|
|
|
6
6
|
from collections.abc import Callable, Mapping, Sequence
|
|
7
7
|
from typing import Literal, Optional
|
|
8
8
|
|
|
9
|
-
|
|
10
9
|
from differential_coverage.types import ApproachId
|
|
11
10
|
|
|
12
11
|
OutputFormat = Literal["stdout", "csv", "latex"]
|
|
@@ -23,8 +22,7 @@ class _PrinterIO:
|
|
|
23
22
|
# and avoid printing empty chunks.
|
|
24
23
|
if not s:
|
|
25
24
|
return 0
|
|
26
|
-
|
|
27
|
-
s = s[:-1]
|
|
25
|
+
s = s.removesuffix("\n")
|
|
28
26
|
if s:
|
|
29
27
|
self._printer(s)
|
|
30
28
|
return len(s)
|
|
@@ -46,28 +44,52 @@ def print_scores(
|
|
|
46
44
|
colormap: str = "viridis",
|
|
47
45
|
latex_enable_color: bool = False,
|
|
48
46
|
digits: int = _DEFAULT_DIGITS,
|
|
47
|
+
value_label: str = "score",
|
|
49
48
|
printer: Callable[[str], None] = print,
|
|
50
49
|
) -> None:
|
|
51
|
-
"""Print one
|
|
50
|
+
"""Print one numeric value per approach, in the requested output format."""
|
|
52
51
|
name_sorted = sorted(scores.items(), key=lambda x: x[0])
|
|
53
52
|
performance_sorted = sorted(name_sorted, key=lambda x: x[1], reverse=True)
|
|
54
53
|
|
|
55
54
|
if output == "stdout":
|
|
56
55
|
_print_scores_plain(performance_sorted, digits=digits, printer=printer)
|
|
57
56
|
elif output == "csv":
|
|
58
|
-
_print_scores_csv(
|
|
57
|
+
_print_scores_csv(
|
|
58
|
+
performance_sorted, digits=digits, value_label=value_label, printer=printer
|
|
59
|
+
)
|
|
59
60
|
elif output == "latex":
|
|
60
61
|
_print_scores_latex(
|
|
61
62
|
performance_sorted,
|
|
62
63
|
enable_color=latex_enable_color,
|
|
63
64
|
colormap=colormap,
|
|
64
65
|
digits=digits,
|
|
66
|
+
value_label=value_label,
|
|
65
67
|
printer=printer,
|
|
66
68
|
)
|
|
67
69
|
else:
|
|
68
70
|
raise ValueError(f"Invalid output format: {output}")
|
|
69
71
|
|
|
70
72
|
|
|
73
|
+
def print_counts(
|
|
74
|
+
counts: Mapping[ApproachId, int],
|
|
75
|
+
*,
|
|
76
|
+
output: OutputFormat = "stdout",
|
|
77
|
+
colormap: str = "viridis",
|
|
78
|
+
latex_enable_color: bool = False,
|
|
79
|
+
printer: Callable[[str], None] = print,
|
|
80
|
+
) -> None:
|
|
81
|
+
"""Print one absolute coverage count per approach, in the requested output format."""
|
|
82
|
+
print_scores(
|
|
83
|
+
{name: float(count) for name, count in counts.items()},
|
|
84
|
+
output=output,
|
|
85
|
+
colormap=colormap,
|
|
86
|
+
latex_enable_color=latex_enable_color,
|
|
87
|
+
digits=0,
|
|
88
|
+
value_label="coverage",
|
|
89
|
+
printer=printer,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
71
93
|
def _print_scores_plain(
|
|
72
94
|
performance_sorted: Sequence[tuple[ApproachId, float]],
|
|
73
95
|
*,
|
|
@@ -82,10 +104,11 @@ def _print_scores_csv(
|
|
|
82
104
|
performance_sorted: Sequence[tuple[ApproachId, float]],
|
|
83
105
|
*,
|
|
84
106
|
digits: int,
|
|
107
|
+
value_label: str,
|
|
85
108
|
printer: Callable[[str], None],
|
|
86
109
|
) -> None:
|
|
87
110
|
writer = csv.writer(_PrinterIO(printer))
|
|
88
|
-
writer.writerow(("approach",
|
|
111
|
+
writer.writerow(("approach", value_label))
|
|
89
112
|
for approach, score in performance_sorted:
|
|
90
113
|
writer.writerow((approach, _format_float(score, digits)))
|
|
91
114
|
|
|
@@ -110,11 +133,12 @@ def _print_scores_latex(
|
|
|
110
133
|
enable_color: bool = False,
|
|
111
134
|
colormap: str = "viridis",
|
|
112
135
|
digits: int,
|
|
136
|
+
value_label: str,
|
|
113
137
|
printer: Callable[[str], None],
|
|
114
138
|
) -> None:
|
|
115
|
-
"""LaTeX table (optionally with
|
|
139
|
+
"""LaTeX table (optionally with value cells colored by value)."""
|
|
116
140
|
printer(r"\begin{tabular}{lr}")
|
|
117
|
-
printer(
|
|
141
|
+
printer(rf"approach & {value_label} \\")
|
|
118
142
|
if not performance_sorted:
|
|
119
143
|
printer(r"\end{tabular}")
|
|
120
144
|
return
|
|
@@ -281,7 +305,9 @@ def _print_relcov_corpus_table_latex(
|
|
|
281
305
|
printer: Callable[[str], None],
|
|
282
306
|
) -> None:
|
|
283
307
|
try:
|
|
284
|
-
from pylatex.utils import
|
|
308
|
+
from pylatex.utils import ( # type: ignore[import-untyped] # does not provide types
|
|
309
|
+
escape_latex,
|
|
310
|
+
)
|
|
285
311
|
except ImportError:
|
|
286
312
|
raise ImportError('latex support requires the "latex" optional dependencies')
|
|
287
313
|
|
|
@@ -336,7 +362,7 @@ def _colormap_light_hex(t: float, *, colormap: str = "viridis") -> str:
|
|
|
336
362
|
if colormap not in colormaps:
|
|
337
363
|
raise ValueError(f"Invalid colormap: {colormap}")
|
|
338
364
|
cmap = colormaps[colormap]
|
|
339
|
-
[r, g, b,
|
|
365
|
+
[r, g, b, _a] = cmap(t)
|
|
340
366
|
[r, g, b] = [1 - ((1 - e) * 0.3) for e in [r, g, b]]
|
|
341
367
|
|
|
342
368
|
return mcolors.to_hex((r, g, b), keep_alpha=False)[1:].upper()
|
|
@@ -32,9 +32,7 @@ def detect(path: Path) -> bool:
|
|
|
32
32
|
return False
|
|
33
33
|
if num_files <= 0 or num_first > num_files:
|
|
34
34
|
return False
|
|
35
|
-
has_marker = any(
|
|
36
|
-
line.startswith("STARTED ") or line.startswith("COV ") for line in lines
|
|
37
|
-
)
|
|
35
|
+
has_marker = any(line.startswith(("STARTED ", "COV ")) for line in lines)
|
|
38
36
|
return has_marker
|
|
39
37
|
|
|
40
38
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: differential_coverage
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Compute differential coverage, a better way of comparing testing tools.
|
|
5
5
|
Author-email: Valentin Huber <contact@valentinhuber.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -38,6 +38,7 @@ In principle, differential coverage measures how much of approach $a_2$'s covera
|
|
|
38
38
|
|
|
39
39
|
- `relcov` does exactly this. It is an *asymetrical* measure between coverage of two approaches. If the approach relies on randomness (such as most fuzzers), it can aggregate over multiple trials of these.
|
|
40
40
|
- `relscore` aggregates all this data to an order of all approaches. This measure, compared to simple coverage numbers, takes into account and values higher approaches that can cover code that no other approach covers. However, compared to `relcov`, this hides a lot of information in this aggregation.
|
|
41
|
+
- `count` just reports absolute coverage counts (union of covered edges/blocks/branches across trials) per approach — useful as a baseline, but without the differential comparison.
|
|
41
42
|
|
|
42
43
|
For more precise definitions, including formulas, look at [DEFINITIONS.md](./DEFINITIONS.md).
|
|
43
44
|
|
|
@@ -146,13 +147,14 @@ pip install differential-coverage[latex]
|
|
|
146
147
|
Generally, the command line interface follows the following structure:
|
|
147
148
|
|
|
148
149
|
```bash
|
|
149
|
-
differential-coverage {relcov,relscore} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
150
|
+
differential-coverage {relcov,relscore,count} [--input-format {auto,afl-showmap,llvm-cov,libfuzzer-merge}] [--granularity {auto,branch,block,edge}] <campaign-dir>
|
|
150
151
|
```
|
|
151
152
|
|
|
152
153
|
Examples:
|
|
153
154
|
|
|
154
155
|
```bash
|
|
155
156
|
differential-coverage relscore ./coverage_data
|
|
157
|
+
differential-coverage count ./coverage_data
|
|
156
158
|
differential-coverage --input-format llvm-cov relcov ./coverage_data
|
|
157
159
|
differential-coverage --output csv relscore ./coverage_data
|
|
158
160
|
```
|
|
@@ -206,6 +208,10 @@ relscores: dict[str, float] = dc.relscores()
|
|
|
206
208
|
for approach_name, relscore in relscores.items():
|
|
207
209
|
print(f"relscore {approach_name}: {relscore}")
|
|
208
210
|
|
|
211
|
+
counts: dict[str, int] = dc.coverage_counts()
|
|
212
|
+
for approach_name, count in counts.items():
|
|
213
|
+
print(f"coverage {approach_name}: {count}")
|
|
214
|
+
|
|
209
215
|
```
|
|
210
216
|
|
|
211
217
|
### Hints
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
|
+
|
|
2
3
|
import pytest
|
|
3
4
|
|
|
4
5
|
from differential_coverage.api import DifferentialCoverage
|
|
@@ -68,3 +69,13 @@ def test_relscores_multiple_approaches() -> None:
|
|
|
68
69
|
"approach_c": 0.5, # 3 is not hit by approach_a, but only hit in half of the trials
|
|
69
70
|
}
|
|
70
71
|
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_coverage_counts() -> None:
|
|
75
|
+
dc = DifferentialCoverage(SAMPLE_CAMPAIGN_CONTENT)
|
|
76
|
+
assert dc.coverage_counts() == {
|
|
77
|
+
"approach_a": 3, # union of {1,2} and {1,3}
|
|
78
|
+
"approach_b": 2, # union of {1,3}, {1,3}, {1}
|
|
79
|
+
"approach_c": 3, # union of {1,2,3} and {1,2,3}
|
|
80
|
+
"seeds": 1, # {1}
|
|
81
|
+
}
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
from copy import deepcopy
|
|
2
2
|
from typing import Any
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
from test_api import SAMPLE_CAMPAIGN_CONTENT
|
|
6
|
+
|
|
3
7
|
from differential_coverage.approach_data import ApproachData
|
|
4
8
|
from differential_coverage.types import CollectionReducer, ValueReducer
|
|
5
|
-
from test_api import SAMPLE_CAMPAIGN_CONTENT
|
|
6
|
-
import pytest
|
|
7
9
|
|
|
8
10
|
|
|
9
11
|
def test_constructor() -> None:
|
|
@@ -52,7 +54,7 @@ def test_edges_by_trial(trials: dict[str, set[int]]) -> None:
|
|
|
52
54
|
|
|
53
55
|
def test_eq() -> None:
|
|
54
56
|
approach_data = ApproachData({"t1": {1, 2}})
|
|
55
|
-
assert approach_data == approach_data # eq to self
|
|
57
|
+
assert approach_data == approach_data # eq to self # noqa: PLR0124
|
|
56
58
|
assert approach_data == deepcopy(approach_data) # eq to deepcopy
|
|
57
59
|
assert approach_data == ApproachData({"t1": {1, 2}}) # eq to same data
|
|
58
60
|
assert approach_data != ApproachData({"t1": {1, 2, 3}}) # different data
|
|
@@ -162,7 +164,7 @@ def test_relcov_value_reducer() -> None:
|
|
|
162
164
|
else:
|
|
163
165
|
return original_getattr(self, name)
|
|
164
166
|
|
|
165
|
-
setattr(ValueReducer, "__getattribute__", spy_getattr)
|
|
167
|
+
setattr(ValueReducer, "__getattribute__", spy_getattr) # noqa: B010
|
|
166
168
|
|
|
167
169
|
res = lhs.relcov(rhs, value_reducer=ValueReducer.MEDIAN)
|
|
168
170
|
assert res == 0.5 # default median from superclass
|
|
@@ -189,7 +191,7 @@ def test_relcov_collection_reducer() -> None:
|
|
|
189
191
|
else:
|
|
190
192
|
return original_getattr(self, name)
|
|
191
193
|
|
|
192
|
-
setattr(CollectionReducer, "__getattribute__", spy_getattr)
|
|
194
|
+
setattr(CollectionReducer, "__getattribute__", spy_getattr) # noqa: B010
|
|
193
195
|
|
|
194
196
|
res = lhs.relcov(rhs, collection_reducer=CollectionReducer.UNION)
|
|
195
197
|
assert res == 0.5 # default: union of rhs, so half is reached
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import io
|
|
4
4
|
import sys
|
|
5
5
|
from pathlib import Path
|
|
6
|
+
|
|
6
7
|
import pytest
|
|
7
8
|
|
|
8
9
|
from differential_coverage.cli import main
|
|
@@ -287,3 +288,46 @@ def test_cli_digits_relscore() -> None:
|
|
|
287
288
|
score = line.split(":", 1)[1].strip()
|
|
288
289
|
assert "." in score
|
|
289
290
|
assert len(score.split(".", 1)[1]) == 4
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def test_cli_count() -> None:
|
|
294
|
+
"""CLI count prints absolute coverage counts sorted descending."""
|
|
295
|
+
code, out, _ = _run_cli(["count", str(SAMPLE_SHOWMAP_DIR)])
|
|
296
|
+
assert code == 0
|
|
297
|
+
lines = [s.strip() for s in out.strip().splitlines()]
|
|
298
|
+
assert len(lines) == 4
|
|
299
|
+
# Sample: approach_a=3, approach_c=3, approach_b=2, seeds=1
|
|
300
|
+
counts = {}
|
|
301
|
+
for line in lines:
|
|
302
|
+
name, count = line.split(":", 1)
|
|
303
|
+
counts[name.strip()] = int(count.strip())
|
|
304
|
+
assert counts == {
|
|
305
|
+
"approach_a": 3,
|
|
306
|
+
"approach_b": 2,
|
|
307
|
+
"approach_c": 3,
|
|
308
|
+
"seeds": 1,
|
|
309
|
+
}
|
|
310
|
+
# Sorted by count descending; ties broken by name ascending
|
|
311
|
+
assert lines[0].startswith("approach_a:") or lines[0].startswith("approach_c:")
|
|
312
|
+
assert lines[-1].startswith("seeds:")
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def test_cli_csv_count() -> None:
|
|
316
|
+
"""CLI --output csv with count outputs CSV with header approach,coverage."""
|
|
317
|
+
code, out, _ = _run_cli(["--output", "csv", "count", str(SAMPLE_SHOWMAP_DIR)])
|
|
318
|
+
assert code == 0
|
|
319
|
+
lines = out.strip().splitlines()
|
|
320
|
+
assert lines[0] == "approach,coverage"
|
|
321
|
+
assert len(lines) == 5 # header + 4 approaches
|
|
322
|
+
assert any(line.startswith("approach_a,3") for line in lines[1:])
|
|
323
|
+
assert any(line.startswith("seeds,1") for line in lines[1:])
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def test_cli_latex_count() -> None:
|
|
327
|
+
"""CLI --output latex with count outputs LaTeX tabular."""
|
|
328
|
+
code, out, _ = _run_cli(["--output", "latex", "count", str(SAMPLE_SHOWMAP_DIR)])
|
|
329
|
+
assert code == 0
|
|
330
|
+
lines = out.strip().splitlines()
|
|
331
|
+
assert any(line.startswith(r"\begin{tabular}") for line in lines)
|
|
332
|
+
assert any("coverage" in line for line in lines)
|
|
333
|
+
assert lines[-1] == r"\end{tabular}"
|
|
@@ -4,9 +4,9 @@ from typing import Any
|
|
|
4
4
|
|
|
5
5
|
def test_docs() -> None:
|
|
6
6
|
from differential_coverage import (
|
|
7
|
-
DifferentialCoverage,
|
|
8
7
|
ApproachData,
|
|
9
8
|
CollectionReducer,
|
|
9
|
+
DifferentialCoverage,
|
|
10
10
|
ValueReducer,
|
|
11
11
|
)
|
|
12
12
|
|
|
@@ -44,3 +44,7 @@ def test_docs() -> None:
|
|
|
44
44
|
relscores: dict[str, float] = dc.relscores()
|
|
45
45
|
for approach_name, relscore in relscores.items():
|
|
46
46
|
print(f"relscore {approach_name}: {relscore}")
|
|
47
|
+
|
|
48
|
+
counts: dict[str, int] = dc.coverage_counts()
|
|
49
|
+
for approach_name, count in counts.items():
|
|
50
|
+
print(f"coverage {approach_name}: {count}")
|
|
@@ -95,7 +95,7 @@ def test_read_llvm_cov_block_reads_calc_export(calc_coverage_dir: Path) -> None:
|
|
|
95
95
|
export = _calc_export(calc_coverage_dir, "approach_a", "t1.json")
|
|
96
96
|
edges = llvm_cov.read(export, granularity="block")
|
|
97
97
|
assert edges
|
|
98
|
-
assert not any(edge.endswith(":true"
|
|
98
|
+
assert not any(edge.endswith((":true", ":false")) for edge in edges)
|
|
99
99
|
|
|
100
100
|
|
|
101
101
|
def test_read_llvm_cov_expansion_branches(llvm_exports: dict[str, Path]) -> None:
|
|
@@ -2,9 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
import csv
|
|
4
4
|
import io
|
|
5
|
-
from
|
|
5
|
+
from collections.abc import Callable
|
|
6
6
|
|
|
7
|
-
from differential_coverage.output import
|
|
7
|
+
from differential_coverage.output import (
|
|
8
|
+
print_counts,
|
|
9
|
+
print_relcov_corpus_table,
|
|
10
|
+
print_scores,
|
|
11
|
+
)
|
|
8
12
|
|
|
9
13
|
# Deliberately unsorted order for both corpus (columns) and table (rows)
|
|
10
14
|
_UNSORTED_CORPUS = ["z_approach", "a_approach", "m_approach"]
|
|
@@ -109,6 +113,32 @@ def test_print_scores_respects_digits() -> None:
|
|
|
109
113
|
assert captured == ["a: 0.3333"]
|
|
110
114
|
|
|
111
115
|
|
|
116
|
+
def test_print_counts_uses_printer_stdout() -> None:
|
|
117
|
+
"""print_counts writes integer coverage counts via the provided printer."""
|
|
118
|
+
counts = {"b": 2, "a": 5}
|
|
119
|
+
captured, printer = _capture_printer()
|
|
120
|
+
|
|
121
|
+
print_counts(counts, output="stdout", printer=printer)
|
|
122
|
+
|
|
123
|
+
assert captured == ["a: 5", "b: 2"]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def test_print_counts_uses_printer_csv() -> None:
|
|
127
|
+
"""print_counts CSV output is routed through the provided printer."""
|
|
128
|
+
counts = {"b": 2, "a": 5}
|
|
129
|
+
captured, printer = _capture_printer()
|
|
130
|
+
|
|
131
|
+
print_counts(counts, output="csv", printer=printer)
|
|
132
|
+
|
|
133
|
+
csv_text = "\n".join(captured)
|
|
134
|
+
reader = csv.reader(io.StringIO(csv_text))
|
|
135
|
+
rows = list(reader)
|
|
136
|
+
|
|
137
|
+
assert rows[0] == ["approach", "coverage"]
|
|
138
|
+
assert rows[1] == ["a", "5"]
|
|
139
|
+
assert rows[2] == ["b", "2"]
|
|
140
|
+
|
|
141
|
+
|
|
112
142
|
def test_relcov_table_respects_digits_csv() -> None:
|
|
113
143
|
"""Relcov CSV cells use the requested digit precision."""
|
|
114
144
|
captured, printer = _capture_printer()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{differential_coverage-1.2.1 → differential_coverage-1.3.0}/src/differential_coverage/types.py
RENAMED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from collections.abc import Callable, Iterable
|
|
2
2
|
from enum import Enum
|
|
3
|
-
from typing import TYPE_CHECKING, Any, Protocol, TypeVar
|
|
4
3
|
from statistics import mean, median
|
|
4
|
+
from typing import TYPE_CHECKING, Any, Protocol, TypeVar
|
|
5
5
|
|
|
6
6
|
if TYPE_CHECKING:
|
|
7
7
|
from differential_coverage.approach_data import ApproachData
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|